From 805080d61e0c4c6523fe52217707f789364549fe Mon Sep 17 00:00:00 2001
From: Dinmukhamed Mailibay <47117969+dinmukhamedm@users.noreply.github.com>
Date: Wed, 30 Oct 2024 20:27:16 -0700
Subject: [PATCH] Easier contribution experience for frontend-only changes
 (#114)

* add default vars to environment example in frontend

* add thin docker-compose for hot-reload frontend changes
---
 CONTRIBUTING.md                   | 63 +++++++++++++++++++-------
 docker-compose-local-dev-full.yml | 73 ++++++++++++++++++++++++++++++
 docker-compose-local-dev.yml      | 75 ++++++++++---------------------
 docker-compose.yml                |  4 +-
 frontend/.env.local.example       | 30 ++++++++-----
 5 files changed, 163 insertions(+), 82 deletions(-)
 create mode 100644 docker-compose-local-dev-full.yml

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 915edb80..7e46ae9f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,22 +26,24 @@ Don't get overwhelmed by the number of docker-compose files. Here's a quick over
 - `docker-compose.yml` is the simplest one that spins up frontend, app-server, and postgres. Good for quickstarts.
 - `docker-compose-full.yml` is the one you want to use for running the full stack locally. This is the best 
 for self-hosting.
+- `docker-compose-local-dev-full.yml` full file for local development. To be used when you make changes
+  to the backend. It will only run the dependency services (postgres, qdrant, clickhouse, rabbitmq).
+  You will need to run `cargo r`, `pnpm run dev`, and `python server.py` manually.
 - `docker-compose-local-dev.yml` is the one you want to use for local development. It will only
-  run the dependency services (postgres, qdrant, clickhouse, rabbitmq). You will need to run
-  `cargo r`, `pnpm run dev`, and `python server.py` manually.
+  run postgres and app-server. Good for frontend changes.
 - `docker-compose-local-build.yml` will build the services from the source and run them in production mode. This is good for self-hosting with your own changes,
 or for testing the changes after developing on your own and before opening a PR.
 
-| Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
-|---------|-------------------|------------------------|----------------------------|------------------------------|
-| postgres | ✅ | ✅ | ✅ | ✅ |
-| qdrant | ❌ | ✅ | ✅ | ✅ |
-| clickhouse | ❌ | ✅ | ✅ | ✅ |
-| rabbitmq | ❌ | ✅ | ✅ | ✅ |
-| app-server | ℹ️ | ✅ | 💻 | 🔧 |
-| frontend | ℹ️ | ✅ | 💻 | 🔧 |
-| semantic-search-service | ❌ | ✅ |  💻 | 🔧 |
-| python-executor | ❌ | ✅ | 💻 | 🔧 |
+| Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
+|---------|-------------------|------------------------|------------------------------|----------------------------|------------------------------|
+| postgres | ✅ | ✅ | ✅ | ✅ | ✅ |
+| qdrant | ❌ | ✅ | ✅ | ❌ | ✅ |
+| clickhouse | ❌ | ✅ | ✅ | ❌ | ✅ |
+| rabbitmq | ❌ | ✅ | ✅ | ❌ | ✅ |
+| app-server | ℹ️ | ✅ | 💻 | ℹ️ | 🔧 | 
+| frontend | ℹ️ | ✅ | 💻 | 💻 | 🔧 |
+| semantic-search-service | ❌ | ✅ | 💻 | ❌ | 🔧 |
+| python-executor | ❌ | ✅ | 💻 | ❌ | 🔧 |
 
 - ✅ – service present, image is pulled from a container registry.
 - 🔧 – service present, image is built from the source. This may take a while.
@@ -50,10 +52,39 @@ or for testing the changes after developing on your own and before opening a PR.
 - ❌ – service not present.
 
 
-## Running Laminar locally
+## Running Laminar locally for development
 
-If you want to test your local changes, you can run code separately in
-development mode.
+Use this guide if you are changing frontend code only.
+For making backend changes or changes across the full stack,
+see [Advanced] section below.
+
+### 0. Configure environment variables
+
+```sh
+cd frontend
+cp .env.local.example .env.local
+```
+
+### 1. Spin up app-server and postgres
+
+```sh
+docker compose -f docker-compose-local-dev.yml up
+```
+
+### 2. Run frontend in development mode
+
+```sh
+cd frontend
+pnpm run dev
+```
+
+Next.js is hot-reloadable in development mode, so any changes you make will be reflected
+immediately.
+
+## [Advanced] Running full stack locally for development
+
+This guide is for when you are changing backend code, or when you want to run the full stack
+locally for development. If you only want to change frontend code, see the section above.
 
 ### 0. Configure environment variables
 
@@ -68,7 +99,7 @@ cp frontend/.env.local.example frontend/.env.local
 ### 1. Spin up dependency containers
 
 ```sh
-docker compose -f docker-compose-local-dev.yml up
+docker compose -f docker-compose-local-dev-full.yml up
 ```
 
 This will spin up postgres, qdrant, clickhouse, and RabbitMQ.
diff --git a/docker-compose-local-dev-full.yml b/docker-compose-local-dev-full.yml
new file mode 100644
index 00000000..07371f11
--- /dev/null
+++ b/docker-compose-local-dev-full.yml
@@ -0,0 +1,73 @@
+# This compose definition does not build Laminar images, it is intended for local development.
+# This file is meant to be used with running the Laminar services manually from each service's directory.
+# Refer to CONTRIBUTING.md for more information on how to run the services locally.
+name: lmnr
+
+services:
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - "6333:6333"
+      - "6334:6334"
+    volumes:
+      - type: volume
+        source: qdrant-data
+        target: /data
+    
+  rabbitmq:
+    image: rabbitmq
+    ports:
+      - "5672:5672"
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 7s
+      timeout: 5s
+      retries: 3
+
+  clickhouse:
+    build:
+      context: ./clickhouse
+    container_name: clickhouse
+    hostname: clickhouse
+    ports:
+      - "8123:8123"
+    volumes:
+      - type: volume
+        source: clickhouse-data
+        target: /var/lib/clickhouse/
+      - type: volume
+        source: clickhouse-logs
+        target: /var/log/clickhouse-server/
+    cap_add:
+      - SYS_NICE
+      - NET_ADMIN
+      - IPC_LOCK
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  postgres:
+    image: postgres:16
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+      interval: 2s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  qdrant-data:
+  clickhouse-data:
+  clickhouse-logs:
+  postgres-data:
diff --git a/docker-compose-local-dev.yml b/docker-compose-local-dev.yml
index 07371f11..0462cf72 100644
--- a/docker-compose-local-dev.yml
+++ b/docker-compose-local-dev.yml
@@ -1,55 +1,13 @@
-# This compose definition does not build Laminar images, it is intended for local development.
-# This file is meant to be used with running the Laminar services manually from each service's directory.
-# Refer to CONTRIBUTING.md for more information on how to run the services locally.
+# This compose file is a lightweight version of docker-compose-local-dev-full.yml.
+# It is intended to be used for local development on frontend only.
+# It does not include ClickHouse,
+# Qdrant, Semantic Search, Python executor, and RabbitMQ.
+# It only includes postgres, and app-server.
+# Run frontend manually with `ENVIRONMENT=LITE pnpm run dev`.
+
 name: lmnr
 
 services:
-  qdrant:
-    image: qdrant/qdrant
-    ports:
-      - "6333:6333"
-      - "6334:6334"
-    volumes:
-      - type: volume
-        source: qdrant-data
-        target: /data
-    
-  rabbitmq:
-    image: rabbitmq
-    ports:
-      - "5672:5672"
-    environment:
-      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
-      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
-    healthcheck:
-      test: rabbitmq-diagnostics -q ping
-      interval: 7s
-      timeout: 5s
-      retries: 3
-
-  clickhouse:
-    build:
-      context: ./clickhouse
-    container_name: clickhouse
-    hostname: clickhouse
-    ports:
-      - "8123:8123"
-    volumes:
-      - type: volume
-        source: clickhouse-data
-        target: /var/lib/clickhouse/
-      - type: volume
-        source: clickhouse-logs
-        target: /var/log/clickhouse-server/
-    cap_add:
-      - SYS_NICE
-      - NET_ADMIN
-      - IPC_LOCK
-    ulimits:
-      nofile:
-        soft: 262144
-        hard: 262144
-
   postgres:
     image: postgres:16
     ports:
@@ -66,8 +24,21 @@ services:
       timeout: 5s
       retries: 5
 
+  app-server:
+    image: ghcr.io/lmnr-ai/app-server
+    pull_policy: always
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PORT: 8000
+      GRPC_PORT: 8001
+      DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
+      SHARED_SECRET_TOKEN: ${SHARED_SECRET_TOKEN}
+      ENVIRONMENT: LITE # this disables runtime dependency on clickhouse, rabbitmq, semantic search, and python executor
+
 volumes:
-  qdrant-data:
-  clickhouse-data:
-  clickhouse-logs:
   postgres-data:
diff --git a/docker-compose.yml b/docker-compose.yml
index 573f7249..ae03fd30 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -39,7 +39,7 @@ services:
       NEXTAUTH_URL: http://localhost:3000
       NEXTAUTH_SECRET: some_secret
       NEXT_PUBLIC_URL: http://localhost:3000
-      ENVIRONMENT: LITE # this disables runtime reliance on clickhouse
+      ENVIRONMENT: LITE # this disables runtime dependency on clickhouse
 
   app-server:
     image: ghcr.io/lmnr-ai/app-server
@@ -55,7 +55,7 @@ services:
       GRPC_PORT: 8001
       DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
       SHARED_SECRET_TOKEN: ${SHARED_SECRET_TOKEN}
-      ENVIRONMENT: LITE # this disables runtime reliance on clickhouse, rabbitmq, semantic search, and python executor
+      ENVIRONMENT: LITE # this disables runtime dependency on clickhouse, rabbitmq, semantic search, and python executor
 
 volumes:
   postgres-data:
diff --git a/frontend/.env.local.example b/frontend/.env.local.example
index 78252606..17f79d8f 100644
--- a/frontend/.env.local.example
+++ b/frontend/.env.local.example
@@ -1,21 +1,27 @@
-NEXTAUTH_URL=
-NEXTAUTH_SECRET=
+NEXTAUTH_URL=http://localhost:3000
+NEXTAUTH_SECRET=next_secret_abc
+BACKEND_URL=http://localhost:8000
+NEXT_OTEL_FETCH_DISABLED=1
+SHARED_SECRET_TOKEN=some_secret
+
+# these must match what you have in your docker-compose-local-dev.yml for postgres
+# postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@{host}:5432/${POSTGRES_DB}
+DATABASE_URL="postgres://postgres:postgres_passwordabc@localhost:5432/postgres"
+
+# replace with FULL if you are testing with a full stack
+# ENVIRONMENT=FULL
+ENVIRONMENT=LITE # this disables runtime dependency on clickhouse
 
+# for realtime
+SUPABASE_JWT_SECRET=
+# for auth
 AUTH_GITHUB_ID=
 AUTH_GITHUB_SECRET=
-
 AUTH_GOOGLE_ID=
 AUTH_GOOGLE_SECRET=
 
-BACKEND_URL=
-
-SUPABASE_JWT_SECRET=
-
-NEXT_OTEL_FETCH_DISABLED=1
-SHARED_SECRET_TOKEN=some_secret
-
+# for s3
 AWS_REGION=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
-
-ENVIRONMENT=FULL
+S3_IMGS_BUCKET=