Merge pull request #117 from lmnr-ai/dev

Update landing and welcome email, contrib experience, minor fixes
lmnr-ai · Oct 31, 2024 · 3641ee2 · 3641ee2
2 parents 034c95d + b70881c
commit 3641ee2
Show file tree

Hide file tree

Showing 16 changed files with 434 additions and 220 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -26,22 +26,24 @@ Don't get overwhelmed by the number of docker-compose files. Here's a quick over
 - `docker-compose.yml` is the simplest one that spins up frontend, app-server, and postgres. Good for quickstarts.
 - `docker-compose-full.yml` is the one you want to use for running the full stack locally. This is the best 
 for self-hosting.
+- `docker-compose-local-dev-full.yml` full file for local development. To be used when you make changes
+  to the backend. It will only run the dependency services (postgres, qdrant, clickhouse, rabbitmq).
+  You will need to run `cargo r`, `pnpm run dev`, and `python server.py` manually.
 - `docker-compose-local-dev.yml` is the one you want to use for local development. It will only
-  run the dependency services (postgres, qdrant, clickhouse, rabbitmq). You will need to run
-  `cargo r`, `pnpm run dev`, and `python server.py` manually.
+  run postgres and app-server. Good for frontend changes.
 - `docker-compose-local-build.yml` will build the services from the source and run them in production mode. This is good for self-hosting with your own changes,
 or for testing the changes after developing on your own and before opening a PR.
 
-| Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
-|---------|-------------------|------------------------|----------------------------|------------------------------|
-| postgres | ✅ | ✅ | ✅ | ✅ |
-| qdrant | ❌ | ✅ | ✅ | ✅ |
-| clickhouse | ❌ | ✅ | ✅ | ✅ |
-| rabbitmq | ❌ | ✅ | ✅ | ✅ |
-| app-server | ℹ️ | ✅ | 💻 | 🔧 |
-| frontend | ℹ️ | ✅ | 💻 | 🔧 |
-| semantic-search-service | ❌ | ✅ |  💻 | 🔧 |
-| python-executor | ❌ | ✅ | 💻 | 🔧 |
+| Service | docker-compose.yml | docker-compose-full.yml | docker-compose-local-dev-full.yml | docker-compose-local-dev.yml | docker-compose-local-build.yml |
+|---------|-------------------|------------------------|------------------------------|----------------------------|------------------------------|
+| postgres | ✅ | ✅ | ✅ | ✅ | ✅ |
+| qdrant | ❌ | ✅ | ✅ | ❌ | ✅ |
+| clickhouse | ❌ | ✅ | ✅ | ❌ | ✅ |
+| rabbitmq | ❌ | ✅ | ✅ | ❌ | ✅ |
+| app-server | ℹ️ | ✅ | 💻 | ℹ️ | 🔧 | 
+| frontend | ℹ️ | ✅ | 💻 | 💻 | 🔧 |
+| semantic-search-service | ❌ | ✅ | 💻 | ❌ | 🔧 |
+| python-executor | ❌ | ✅ | 💻 | ❌ | 🔧 |
 
 - ✅ – service present, image is pulled from a container registry.
 - 🔧 – service present, image is built from the source. This may take a while.
@@ -50,10 +52,39 @@ or for testing the changes after developing on your own and before opening a PR.
 - ❌ – service not present.
 
 
-## Running Laminar locally
+## Running Laminar locally for development
 
-If you want to test your local changes, you can run code separately in
-development mode.
+Use this guide if you are changing frontend code only.
+For making backend changes or changes across the full stack,
+see [Advanced] section below.
+
+### 0. Configure environment variables
+
+```sh
+cd frontend
+cp .env.local.example .env.local
+```
+
+### 1. Spin up app-server and postgres
+
+```sh
+docker compose -f docker-compose-local-dev.yml up
+```
+
+### 2. Run frontend in development mode
+
+```sh
+cd frontend
+pnpm run dev
+```
+
+Next.js is hot-reloadable in development mode, so any changes you make will be reflected
+immediately.
+
+## [Advanced] Running full stack locally for development
+
+This guide is for when you are changing backend code, or when you want to run the full stack
+locally for development. If you only want to change frontend code, see the section above.
 
 ### 0. Configure environment variables
 
@@ -68,7 +99,7 @@ cp frontend/.env.local.example frontend/.env.local
 ### 1. Spin up dependency containers
 
 ```sh
-docker compose -f docker-compose-local-dev.yml up
+docker compose -f docker-compose-local-dev-full.yml up
 ```
 
 This will spin up postgres, qdrant, clickhouse, and RabbitMQ.

diff --git a/app-server/src/db/trace.rs b/app-server/src/db/trace.rs
@@ -40,7 +40,7 @@ pub enum TraceType {
 #[derive(Serialize, sqlx::FromRow, Clone, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct Trace {
-    pub id: Uuid,
+    id: Uuid,
     #[serde(default)]
     start_time: Option<DateTime<Utc>>,
     #[serde(default)]
@@ -60,12 +60,12 @@ pub struct Trace {
     output_cost: f64,
     cost: f64,
     success: bool,
-    pub project_id: Uuid,
+    project_id: Uuid,
 }
 
 #[derive(Serialize, sqlx::FromRow)]
 #[serde(rename_all = "camelCase")]
-pub struct TraceWithParentSpanAndEvents {
+pub struct TraceWithTopSpan {
     id: Uuid,
     start_time: DateTime<Utc>,
     end_time: Option<DateTime<Utc>>,
@@ -91,9 +91,6 @@ pub struct TraceWithParentSpanAndEvents {
     top_span_name: Option<String>,
     top_span_type: Option<SpanType>,
     top_span_path: Option<String>,
-
-    // 'events' is a list of partial event objects, using Option because of Coalesce
-    events: Option<Value>,
 }
 
 #[derive(FromRow, Debug)]
@@ -282,26 +279,6 @@ fn add_text_join(
     Ok(())
 }
 
-const TRACE_EVENTS_EXPRESSION: &str = "
-    trace_events AS (
-        SELECT
-            traces.id as trace_id,
-            jsonb_agg(
-                jsonb_build_object(
-                    'id', events.id,
-                    'typeId', events.template_id,
-                    'templateName', event_templates.name,
-                    'spanId', events.span_id
-                )
-            ) as events
-        FROM events
-        JOIN event_templates ON events.template_id = event_templates.id
-        JOIN spans ON spans.span_id = events.span_id
-        JOIN traces ON traces.id = spans.trace_id
-        WHERE traces.start_time IS NOT NULL AND traces.end_time IS NOT NULL
-        GROUP BY traces.id
-    )";
-
 fn add_filters_to_traces_query(query: &mut QueryBuilder<Postgres>, filters: &Option<Vec<Filter>>) {
     if let Some(filters) = filters {
         filters.iter().for_each(|filter| {
@@ -416,11 +393,9 @@ pub async fn get_traces(
     filters: &Option<Vec<Filter>>,
     date_range: &Option<DateRange>,
     text_search_filter: Option<String>,
-) -> Result<Vec<TraceWithParentSpanAndEvents>> {
+) -> Result<Vec<TraceWithTopSpan>> {
     let mut query = QueryBuilder::<Postgres>::new("WITH ");
     add_traces_info_expression(&mut query, date_range, project_id)?;
-    query.push(", ");
-    query.push(TRACE_EVENTS_EXPRESSION);
 
     query.push(
         "
@@ -441,15 +416,13 @@ pub async fn get_traces(
             output_cost,
             cost,
             success,
-            COALESCE(trace_events.events, '[]'::jsonb) AS events,
             top_span_input_preview,
             top_span_output_preview,
             top_span_name,
             top_span_type,
             top_span_path,
             status
-        FROM traces_info
-        LEFT JOIN trace_events ON trace_events.trace_id = traces_info.id ",
+        FROM traces_info ",
     );
     if let Some(search) = text_search_filter {
         add_text_join(&mut query, date_range, &search)?;
@@ -466,7 +439,7 @@ pub async fn get_traces(
         .push_bind(limit as i64);
 
     let traces = query
-        .build_query_as::<'_, TraceWithParentSpanAndEvents>()
+        .build_query_as::<'_, TraceWithTopSpan>()
         .fetch_all(pool)
         .await?;
 
@@ -483,14 +456,11 @@ pub async fn count_traces(
 ) -> Result<i64> {
     let mut query = QueryBuilder::<Postgres>::new("WITH ");
     add_traces_info_expression(&mut query, date_range, project_id)?;
-    query.push(", ");
-    query.push(TRACE_EVENTS_EXPRESSION);
     query.push(
         "
         SELECT
             COUNT(DISTINCT(id)) as total_count
         FROM traces_info
-        LEFT JOIN trace_events ON trace_events.trace_id = traces_info.id
         ",
     );
     if let Some(search) = text_search_filter {

diff --git a/app-server/src/routes/traces.rs b/app-server/src/routes/traces.rs
@@ -8,7 +8,7 @@ use crate::{
         events::EventWithTemplateName,
         modifiers::{DateRange, Filter, RelativeDateInterval},
         spans::Span,
-        trace::{Session, Trace, TraceWithParentSpanAndEvents},
+        trace::{Session, Trace, TraceWithTopSpan},
         DB,
     },
 };
@@ -82,7 +82,7 @@ pub async fn get_traces(
     let (total_count, any_in_project) =
         count_result.map_err(|e| anyhow::anyhow!("Failed to count traces: {:?}", e))?;
 
-    let response = PaginatedResponse::<TraceWithParentSpanAndEvents> {
+    let response = PaginatedResponse::<TraceWithTopSpan> {
         total_count,
         items: traces,
         any_in_project,

diff --git a/docker-compose-local-dev-full.yml b/docker-compose-local-dev-full.yml
@@ -0,0 +1,73 @@
+# This compose definition does not build Laminar images, it is intended for local development.
+# This file is meant to be used with running the Laminar services manually from each service's directory.
+# Refer to CONTRIBUTING.md for more information on how to run the services locally.
+name: lmnr
+
+services:
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - "6333:6333"
+      - "6334:6334"
+    volumes:
+      - type: volume
+        source: qdrant-data
+        target: /data
+
+  rabbitmq:
+    image: rabbitmq
+    ports:
+      - "5672:5672"
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 7s
+      timeout: 5s
+      retries: 3
+
+  clickhouse:
+    build:
+      context: ./clickhouse
+    container_name: clickhouse
+    hostname: clickhouse
+    ports:
+      - "8123:8123"
+    volumes:
+      - type: volume
+        source: clickhouse-data
+        target: /var/lib/clickhouse/
+      - type: volume
+        source: clickhouse-logs
+        target: /var/log/clickhouse-server/
+    cap_add:
+      - SYS_NICE
+      - NET_ADMIN
+      - IPC_LOCK
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+
+  postgres:
+    image: postgres:16
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_DB: ${POSTGRES_DB}
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}", "-d", "${POSTGRES_DB}"]
+      interval: 2s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  qdrant-data:
+  clickhouse-data:
+  clickhouse-logs:
+  postgres-data:
diff --git a/docker-compose-local-dev.yml b/docker-compose-local-dev.yml
@@ -1,55 +1,13 @@
-# This compose definition does not build Laminar images, it is intended for local development.
-# This file is meant to be used with running the Laminar services manually from each service's directory.
-# Refer to CONTRIBUTING.md for more information on how to run the services locally.
+# This compose file is a lightweight version of docker-compose-local-dev-full.yml.
+# It is intended to be used for local development on frontend only.
+# It does not include ClickHouse,
+# Qdrant, Semantic Search, Python executor, and RabbitMQ.
+# It only includes postgres, and app-server.
+# Run frontend manually with `ENVIRONMENT=LITE pnpm run dev`.
+
 name: lmnr
 
 services:
-  qdrant:
-    image: qdrant/qdrant
-    ports:
-      - "6333:6333"
-      - "6334:6334"
-    volumes:
-      - type: volume
-        source: qdrant-data
-        target: /data
-
-  rabbitmq:
-    image: rabbitmq
-    ports:
-      - "5672:5672"
-    environment:
-      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER}
-      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS}
-    healthcheck:
-      test: rabbitmq-diagnostics -q ping
-      interval: 7s
-      timeout: 5s
-      retries: 3
-
-  clickhouse:
-    build:
-      context: ./clickhouse
-    container_name: clickhouse
-    hostname: clickhouse
-    ports:
-      - "8123:8123"
-    volumes:
-      - type: volume
-        source: clickhouse-data
-        target: /var/lib/clickhouse/
-      - type: volume
-        source: clickhouse-logs
-        target: /var/log/clickhouse-server/
-    cap_add:
-      - SYS_NICE
-      - NET_ADMIN
-      - IPC_LOCK
-    ulimits:
-      nofile:
-        soft: 262144
-        hard: 262144
-
   postgres:
     image: postgres:16
     ports:
@@ -66,8 +24,21 @@ services:
       timeout: 5s
       retries: 5
 
+  app-server:
+    image: ghcr.io/lmnr-ai/app-server
+    pull_policy: always
+    ports:
+      - "8000:8000"
+      - "8001:8001"
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PORT: 8000
+      GRPC_PORT: 8001
+      DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
+      SHARED_SECRET_TOKEN: ${SHARED_SECRET_TOKEN}
+      ENVIRONMENT: LITE # this disables runtime dependency on clickhouse, rabbitmq, semantic search, and python executor
+
 volumes:
-  qdrant-data:
-  clickhouse-data:
-  clickhouse-logs:
   postgres-data: