Merge branch 'main' into feat/feedback-filter

.github/workflows/test-build-deploy.yml

@@ -1,36 +1,41 @@
 name: Build Docker Images
 
-on:
-  push:
-    branches:
-      - main
+on: push
 
 jobs:
-  # run-tests:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Install dependencies
-  #       run: npm ci && npx playwright install --with-deps
-  #     - name: Start the development server
-  #       env:
-  #         DATABASE_URL: ${{ secrets.DATABASE_URL }}
-  #         JWT_SECRET: ${{ secrets.JWT_SECRET }}
-  #         APP_URL: http://localhost:8080
-  #         API_URL: http://localhost:3333
-  #         LUNARY_PUBLIC_KEY: 259d2d94-9446-478a-ae04-484de705b522
-  #         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install dependencies
+        run: npm ci && npx playwright install --with-deps
+      - name: Start backend
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          JWT_SECRET: ${{ secrets.JWT_SECRET }}
+          APP_URL: http://localhost:8080
+          API_URL: http://localhost:3333
+          LUNARY_PUBLIC_KEY: 259d2d94-9446-478a-ae04-484de705b522
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        timeout-minutes: 1
+        run: npm run start:backend  & npx wait-on http://localhost:3333/v1/health
 
-  #       run: npm run dev:test & npx wait-on http://localhost:3333/v1/health
+      - name: Start frontend
+        env:
+          APP_URL: http://localhost:8080
+          API_URL: http://localhost:3333
+          NEXT_PUBLIC_API_URL: xyzPLACEHOLDERxyz
+        timeout-minutes: 2
+        run: (npm run build:frontend && npm run start:frontend)  & npx wait-on http://localhost:8080/
 
-  #     - name: Run tests
-  #       run: npx playwright test
-  #       env:
-  #         BASE_URL: http://localhost:8080
-  #         DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      - name: Run tests
+        run: npx playwright test
+        env:
+          BASE_URL: http://localhost:8080
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
 
   build-and-push:
-    # needs: run-tests
+    needs: run-tests
     runs-on: ubuntu-latest
     steps:
       - name: Check out the private Ops repo
@@ -60,13 +65,46 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./ops/Dockerfile
+          file: ./ops/backend.Dockerfile
           push: true
           tags: |
             lunary/backend:latest
             lunary/backend:rev-${{ steps.date.outputs.date }}-${{ steps.commit.outputs.hash }}
           platforms: linux/amd64
 
+      - name: Build and push frontend
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./ops/frontend.Dockerfile
+          push: true
+          tags: |
+            lunary/frontend:latest
+            lunary/frontend:rev-${{ steps.date.outputs.date }}-${{ steps.commit.outputs.hash }}
+          platforms: linux/amd64
+
+      - name: Build and push radar
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./ops/radar.Dockerfile
+          push: true
+          tags: |
+            lunary/radar:latest
+            lunary/radar:rev-${{ steps.date.outputs.date }}-${{ steps.commit.outputs.hash }}
+          platforms: linux/amd64
+
+      - name: Build and push ml
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./ops/ml.Dockerfile
+          push: true
+          tags: |
+            lunary/ml:latest
+            lunary/ml:rev-${{ steps.date.outputs.date }}-${{ steps.commit.outputs.hash }}
+          platforms: linux/amd64
+
   deploy:
     needs: build-and-push
     runs-on: ubuntu-latest

e2e/db-utils.ts

@@ -1,13 +1,13 @@
 import sql from "../packages/backend/src/utils/db"
 
 export async function setOrgPro() {
-  await sql`update org set plan = 'pro' where name = 'TESTORG'`
+  return sql`update org set plan = 'pro' where name = 'TESTORG'`
 }
 
 export async function setOrgFree() {
-  await sql`update org set plan = 'free' where name = 'TESTORG'`
+  return sql`update org set plan = 'free' where name = 'TESTORG'`
 }
 
 export async function deleteOrg() {
-  await sql`delete from org where name = 'TESTORG'`
+  return sql`delete from org where name = 'TESTORG'`
 }

package.json

@@ -7,17 +7,14 @@
     "start:frontend": "npm -w packages/frontend run start",
     "start:backend": "npm -w packages/backend run start",
     "start:radar": "npm -w packages/backend run start:radar",
-    "start:ml": ". ./packages/ml/venv/bin/activate && (ulimit -t 40; python3 packages/ml/main.py)",
+    "start:ml": ". ./packages/ml/venv/bin/activate && python3 packages/ml/main.py",
     "build:frontend": "npm -w packages/frontend run build",
     "migrate:db": "npm -w packages/backend run migrate:db",
     "dev": "concurrently \"npm run dev:frontend\" \"npm run dev:backend\" \"npm run start:ml\"",
     "dev:frontend": "npm -w packages/frontend run dev",
     "dev:backend": "npm -w packages/backend run dev",
     "dev:radar": "npm -w packages/backend run dev:radar",
-    "dev:test": "concurrently \"npm run dev:frontend\" \"npm run dev:backend:test\"",
-    "todo": "TODO: clean scripts below and verify they all work",
     "dev:withradar": "concurrently \"npm run dev:frontend\" \"npm run dev:backend\" \"npm run start:ml\" \"npm run dev:radar\"",
-    "dev:backend:test": "npm -w packages/backend run test",
     "test": "npx playwright test",
     "test:ui": "npx playwright test --ui"
   },

packages/backend/package.json

@@ -2,7 +2,7 @@
   "name": "@lunary/backend",
   "main": "index.ts",
   "scripts": {
-    "start": "tsx src/index.ts",
+    "start": "npm run migrate:db && tsx src/index.ts",
     "start:radar": "tsx src/radar.ts",
     "migrate:db": "tsx src/migrate.ts",
     "build": "tsup src/index.ts --format esm",

packages/backend/src/index.ts

@@ -16,8 +16,6 @@ import { setDefaultBody } from "./utils/misc"
 import ratelimit from "./utils/ratelimit"
 import { initSentry, requestHandler, tracingMiddleWare } from "./utils/sentry"
 
-import { hasReadAccess } from "shared"
-
 checkDbConnection()
 setupCronJobs()
 initSentry()

packages/backend/src/utils/ml.ts

@@ -1,5 +1,5 @@
 export async function callML(method: string, data: any) {
-  const response = await fetch(`http://localhost:4242/${method}`, {
+  const response = await fetch(`http://${process.env.ML_URL}/${method}`, {
     method: "POST",
     // For example at the first ML calls, it needs to DL the models, so it can take a while
     // So add timeout

packages/backend/src/utils/sentry.ts

@@ -46,6 +46,7 @@ export function sendErrorToSentry(err: unknown, ctx: Context) {
       return Sentry.addRequestDataToEvent(event, ctx.request)
     })
     scope.setUser({ id: ctx.state.userId })
+    scope.setTag("IP Address", ctx.get("Cf-Connecting-Ip"))
     Sentry.captureException(err)
   })
 }

packages/frontend/components/layout/Navbar.tsx

@@ -57,16 +57,19 @@ export default function Navbar() {
     setSendingEmail(true)
 
     const ok = await errorHandler(
-      fetch(`${window.API_URL}/v1/users/send-verification`, {
-        method: "POST",
-        body: JSON.stringify({
-          email: user?.email,
-          name: user?.name,
-        }),
-        headers: {
-          "Content-Type": "application/json",
+      fetch(
+        `${process.env.NEXT_PUBLIC_API_URL || window.location.origin}/v1/users/send-verification`,
+        {
+          method: "POST",
+          body: JSON.stringify({
+            email: user?.email,
+            name: user?.name,
+          }),
+          headers: {
+            "Content-Type": "application/json",
+          },
         },
-      }),
+      ),
     )
 
     if (ok) {

packages/frontend/load-env.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# This script is necessary because we need to be able to inject API_URL and APP_URL after build time, and Next does not provide and easy way to do that.
+# This script is necessary because we need to be able to inject API_URL after build time, and Next does not provide and easy way to do that.
 # This should work find both with npm run dev, locally and with docker
 
 # Loads .env
@@ -10,10 +10,9 @@ if [ -f .env ]; then
 fi
 
 
-if [ -z "$API_URL" ] || [ -z "$APP_URL" ]; then
+if [ -z "$API_URL" ]; then
   echo "Error: API_URL not set. Please set the API_URL environment variables."
   exit 1
 fi
 
-echo "window.API_URL = '${API_URL}'; window.APP_URL = '${APP_URL}';" > public/config.js
-
+LC_ALL=C  find .next -type f -exec perl -pi -e "s|xyzPLACEHOLDERxyz|${API_URL}|g" {} +

packages/frontend/package.json

@@ -2,10 +2,10 @@
   "name": "@lunary/frontend",
   "main": "pages/index.ts",
   "scripts": {
-    "dev": "npm run load-env && next dev -p 8080",
-    "build": "next build",
-    "load-env": "bash ./load-env.sh",
-    "start": "npm run load-env && next start -p 8080"
+    "start": "npm run load-env && next start -p 8080",
+    "dev": "next dev -p 8080",
+    "build": "rm -rf .next && next build",
+    "load-env": "bash ./load-env.sh"
   },
   "dependencies": {
     "@calcom/embed-react": "^1.3.0",

packages/frontend/pages/_app.tsx

@@ -118,7 +118,6 @@ export default function App({ Component, pageProps }: AppProps) {
       <Head>
         <link href="https://lunary.ai/logo.png" rel="icon" type="image/png" />
       </Head>
-      <Script src="/config.js" strategy="beforeInteractive" />
       <AuthProvider>
         <SWRConfig
           value={{

packages/frontend/pages/request-password-reset.tsx

@@ -31,7 +31,7 @@ export default function PasswordReset() {
     setLoading(true)
 
     const res = await errorHandler(
-      fetch(`${window.API_URL}/auth/request-password-reset`, {
+      fetch(`${process.env.API_URL}/auth/request-password-reset`, {
         method: "POST",
         headers: {
           "Content-Type": "application/json",

packages/frontend/pages/signup.tsx

@@ -132,8 +132,7 @@ function SignupPage() {
           message: "Check your emails for the confirmation link",
         })
       } else {
-        // redirect to dashboard
-        window.location.href = "/"
+        nextStep()
       }
 
       analytics.track("Signup", {

packages/frontend/pages/team.tsx

@@ -149,29 +149,31 @@ function SAMLConfig() {
               <Table.Td>Assertion Consumer Service (ACS) URL:</Table.Td>
               <Table.Td>
                 <CopyInput
-                  value={`${window.API_URL}/auth/saml/${org?.id}/acs`}
+                  value={`${process.env.API_URL}/auth/saml/${org?.id}/acs`}
                 />
               </Table.Td>
             </Table.Tr>
             <Table.Tr>
               <Table.Td>Single Logout Service (SLO) URL:</Table.Td>
               <Table.Td>
                 <CopyInput
-                  value={`${window.API_URL}/auth/saml/${org?.id}/slo`}
+                  value={`${process.env.API_URL}/auth/saml/${org?.id}/slo`}
                 />
               </Table.Td>
             </Table.Tr>
             <Table.Tr>
               <Table.Td>Sign on URL:</Table.Td>
               <Table.Td>
-                <CopyInput value={`${window.APP_URL}/login`} />
+                <CopyInput
+                  value={`${process.env.NEXT_PUBLIC_API_URL || window.location.origin}/login`}
+                />
               </Table.Td>
             </Table.Tr>
             <Table.Tr>
               <Table.Td>Single Logout URL:</Table.Td>
               <Table.Td>
                 <CopyInput
-                  value={`${window.API_URL}/auth/saml/${org?.id}/slo`}
+                  value={`${process.env.API_URL}/auth/saml/${org?.id}/slo`}
                 />
               </Table.Td>
             </Table.Tr>

packages/frontend/utils/fetcher/client.ts

@@ -2,7 +2,8 @@ import Router from "next/router"
 import { signOut } from "../auth"
 import { showErrorNotification } from "../errors"
 
-const BASE_URL = (window.API_URL as string) || process.env.NEXT_PUBLIC_API_URL
+const BASE_URL =
+  (process.env.API_URL as string) || process.env.NEXT_PUBLIC_API_URL
 
 export function buildUrl(path: string) {
   if (path.includes("/auth")) {

packages/ml/requirements.txt

@@ -1,4 +1,5 @@
 --extra-index-url https://download.pytorch.org/whl/cpu
+waitress==3.0.0
 detoxify==0.5.2
 Flask==3.0.2
 lingua-language-detector==2.0.2

playwright.config.ts

@@ -7,7 +7,7 @@ import { defineConfig, devices } from "@playwright/test"
 import dotenv from "dotenv"
 
 // Read from default ".env" file.
-dotenv.config()
+dotenv.config({ path: "./packages/backend/.env" })
 
 /**
  * See https://playwright.dev/docs/test-configuration.