feat: ui improvements

e2e/projects.spec.ts

@@ -25,6 +25,6 @@ test("create new project, rename it and delete it", async ({ page }) => {
   await page.getByTestId("delete-project-button").click()
   await page.getByTestId("delete-project-popover-button").click()
 
-  // If the project was deleted successfully, it redirects to the analytics page
+  // // If the project was deleted successfully, it redirects to the analytics page
   await page.waitForURL("**/analytics")
 })

packages/backend/package.json

@@ -35,6 +35,7 @@
     "p-queue": "^8.0.1",
     "postgres": "^3.4.3",
     "prexit": "^2.2.0",
+    "queue-promise": "^2.2.1",
     "rouge": "^1.0.3",
     "samlify": "^2.8.11",
     "shared": "*",

packages/backend/src/api/v1/evaluations/index.ts

@@ -6,7 +6,8 @@ import sql from "@/src/utils/db"
 import Context from "@/src/utils/koa"
 import Router from "koa-router"
 import { RunEvent } from "lunary/types"
-import PQueue from "p-queue"
+
+import Queue from "queue-promise"
 import { PassThrough } from "stream"
 import { runEval } from "./utils"
 
@@ -31,9 +32,9 @@ evaluations.post(
       Connection: "keep-alive",
     })
 
-    const queue = new PQueue({
-      concurrency: MAX_PARALLEL_EVALS,
-      timeout: 10000,
+    const queue = new Queue({
+      concurrent: MAX_PARALLEL_EVALS,
+      start: true,
     })
 
     const [{ plan }] =
@@ -70,16 +71,17 @@ evaluations.post(
       for (const variation of variations) {
         for (const provider of evaluation.providers) {
           count++
-          queue.add(() =>
-            runEval({
+          queue.enqueue(async () => {
+            await runEval({
               evaluationId: evaluation.id,
               promptId: prompt.id,
               variation,
               provider,
               prompt: prompt.messages,
               checklistId,
-            }),
-          )
+            })
+            console.log(`Task ${count} don with model ${provider.model} done`)
+          })
         }
       }
     }
@@ -89,17 +91,24 @@ evaluations.post(
     ctx.status = 200
     ctx.body = stream
 
-    queue.on("active", () => {
-      const percentDone = ((count - queue.size) / count) * 100
-      console.log(`Active: ${queue.size} of ${count} (${percentDone}%)`)
+    let done = 0
+
+    queue.on("dequeue", () => {
+      done++
+      const percentDone = (1 - (count - done) / count) * 100
+      console.log(`Active: ${done} of ${count} (${percentDone}%)`)
       stream.write(JSON.stringify({ percentDone }) + "\n")
     })
 
-    await queue.onIdle()
+    console.log(`Queue started with ${count} tasks`)
 
-    stream.write(JSON.stringify({ id: evaluation?.id }) + "\n")
+    queue.on("end", () => {
+      console.log("Queue is empty now")
 
-    stream.end()
+      stream.write(JSON.stringify({ id: evaluation?.id }) + "\n")
+
+      stream.end()
+    })
   },
 )
 

packages/backend/src/api/v1/evaluations/utils.ts

@@ -117,6 +117,8 @@ export async function runEval({
         duration,
       })}
       `
+
+      console.log(`Eval for ${provider.model} passed: ${passed}`)
     } catch (error: any) {
       await sql`
         insert into evaluation_result ${sql({

packages/backend/src/api/v1/radars.ts

@@ -36,55 +36,55 @@ const DEFAULT_RADARS = [
       },
     ],
   },
-  {
-    description:
-      "Answer potentially contains PII (Personal Identifiable Information)",
-    negative: true,
-    view: [
-      "AND",
-      {
-        id: "type",
-        params: {
-          type: "llm",
-        },
-      },
-    ],
-    checks: [
-      "AND",
-      {
-        id: "pii",
-        params: {
-          field: "input",
-          type: "contains",
-          entities: ["person", "location", "email", "cc", "phone", "ssn"],
-        },
-      },
-    ],
-  },
-  {
-    description: "Prompt contains PII (Personal Identifiable Information)",
-    negative: true,
-    view: [
-      "AND",
-      {
-        id: "type",
-        params: {
-          type: "llm",
-        },
-      },
-    ],
-    checks: [
-      "AND",
-      {
-        id: "pii",
-        params: {
-          field: "input",
-          type: "contains",
-          entities: ["person", "location", "email", "cc", "phone", "ssn"],
-        },
-      },
-    ],
-  },
+  // {
+  //   description:
+  //     "Answer potentially contains PII (Personal Identifiable Information)",
+  //   negative: true,
+  //   view: [
+  //     "AND",
+  //     {
+  //       id: "type",
+  //       params: {
+  //         type: "llm",
+  //       },
+  //     },
+  //   ],
+  //   checks: [
+  //     "AND",
+  //     {
+  //       id: "pii",
+  //       params: {
+  //         field: "input",
+  //         type: "contains",
+  //         entities: ["person", "location", "email", "cc", "phone", "ssn"],
+  //       },
+  //     },
+  //   ],
+  // },
+  // {
+  //   description: "Prompt contains PII (Personal Identifiable Information)",
+  //   negative: true,
+  //   view: [
+  //     "AND",
+  //     {
+  //       id: "type",
+  //       params: {
+  //         type: "llm",
+  //       },
+  //     },
+  //   ],
+  //   checks: [
+  //     "AND",
+  //     {
+  //       id: "pii",
+  //       params: {
+  //         field: "input",
+  //         type: "contains",
+  //         entities: ["person", "location", "email", "cc", "phone", "ssn"],
+  //       },
+  //     },
+  //   ],
+  // },
   {
     description: "Contains profanity or toxic language",
     negative: true,

packages/backend/src/checks/index.ts

@@ -348,6 +348,7 @@ export const CHECK_RUNNERS: CheckRunner[] = [
       return sql`${field} ${operator} ${textParam}`
     },
   },
+
   {
     id: "assertion",
     async evaluator(run, params) {

packages/backend/src/utils/db.ts

@@ -14,8 +14,8 @@ const sql = postgres(process.env.DATABASE_URL!, {
   connection: {
     application_name: `backend-${isProduction ? "production" : "development"}-${new Date().getTime()}`,
   },
-  debug: process.env.DEBUG ? debugFn : () => {}, 
-  onnotice: process.env.DEUG ? console.log : () => {},// TODO: replace `() => {}` by false when porsager/postgres PR is merged
+  // debug: process.env.DEBUG ? debugFn : () => {},
+  // onnotice: process.env.DEUG ? console.log : () => {},// TODO: replace `() => {}` by false when porsager/postgres PR is merged
 })
 
 function debugFn(

packages/backend/src/utils/license.ts

@@ -22,7 +22,6 @@ async function licenseMiddleware(ctx: Context, next: Next) {
 
   try {
     if (Date.now() - cache.lastFetch > TWO_HOURS) {
-      console.log("Fetching")
       const licenseData = await fetch(
         `https://license.lunary.ai/v1/licenses/${LICENSE_KEY}`,
       ).then((res) => res.json())

packages/frontend/components/blocks/SettingsCard.tsx

@@ -1,17 +1,41 @@
 import { Card, Stack, Title } from "@mantine/core"
+import Paywall from "../layout/Paywall"
 
 // so we can have an harmonized title for all cards
 export function SettingsCard({
   title,
   children,
   align,
+  paywallConfig,
   gap = "lg",
 }: {
   title
   children: React.ReactNode
+  paywallConfig?: any
   align?: string
   gap?: string
 }) {
+  if (paywallConfig?.enabled) {
+    return (
+      <Card withBorder p="lg">
+        <Stack gap={gap} align={align}>
+          <Title order={4}>{title}</Title>
+          <Stack
+            style={{
+              position: "relative",
+              minHeight: 400,
+              width: "100%",
+              borderRadius: 8,
+              overflow: "hidden",
+            }}
+          >
+            <Paywall {...paywallConfig}>{children}</Paywall>
+          </Stack>
+        </Stack>
+      </Card>
+    )
+  }
+
   return (
     <Card withBorder p="lg" style={{ overflow: "visible" }}>
       <Stack gap={gap} align={align}>

packages/frontend/components/checks/ChecksUIData.tsx

@@ -3,15 +3,18 @@ import {
   IconBiohazard,
   IconBraces,
   IconBracketsContainStart,
+  IconBrain,
   IconBrandOpenai,
   IconCalendar,
   IconCheck,
+  IconCheckbox,
   IconCircleLetterT,
   IconClock,
   IconCoin,
   IconCreditCard,
   IconEyeCheck,
   IconFilter,
+  IconFocus,
   IconHelpCircle,
   IconHtml,
   IconIdBadge,
@@ -27,6 +30,8 @@ import {
   IconSearch,
   IconShieldBolt,
   IconTag,
+  IconTarget,
+  IconTextWrap,
   IconThumbUp,
   IconTools,
   IconUser,
@@ -210,6 +215,22 @@ const CHECKS_UI_DATA: ChecksUIData = {
     icon: IconIdBadge,
     color: "orange",
   },
+  summarization: {
+    icon: IconTextWrap,
+    color: "blue",
+  },
+  "context-recall": {
+    icon: IconBrain,
+    color: "blue",
+  },
+  "context-precision": {
+    icon: IconTarget,
+    color: "blue",
+  },
+  relevancy: {
+    icon: IconCheckbox,
+    color: "green",
+  },
   other: {
     icon: IconFilter,
     color: "gray",

packages/frontend/components/checks/SmartSelectInput.tsx

@@ -100,18 +100,23 @@ export default function SmartCheckSelect({
       .toLowerCase()
       .includes(search.trim().toLowerCase())
   }
-  const renderedOptions = data?.filter(optionsFilter).map((item) => (
-    <Combobox.Option
-      value={getItemValue(item)}
-      key={getItemValue(item)}
-      active={value?.includes(getItemValue(item))}
-    >
-      <Group gap="sm" wrap="nowrap">
-        {value?.includes(getItemValue(item)) ? <CheckIcon size={12} /> : null}
-        {renderListItem ? renderListItem(item) : renderLabel(item)}
-      </Group>
-    </Combobox.Option>
-  ))
+  const renderedOptions = data?.filter(optionsFilter).map((item) => {
+    const active = multiple
+      ? fixedValue.includes(getItemValue(item))
+      : getItemValue(item) === value
+    return (
+      <Combobox.Option
+        value={getItemValue(item)}
+        key={getItemValue(item)}
+        active={active}
+      >
+        <Group gap="sm" wrap="nowrap">
+          {active ? <CheckIcon size={12} /> : null}
+          {renderListItem ? renderListItem(item) : renderLabel(item)}
+        </Group>
+      </Combobox.Option>
+    )
+  })
 
   useEffect(() => {
     if (!value) {
@@ -125,6 +130,7 @@ export default function SmartCheckSelect({
         <PillsInput
           onClick={() => combobox.openDropdown()}
           variant="unstyled"
+          size="xs"
           miw={width}
           w="min-content"
         >