feat(engine): Add log indexer and vectordb engines

frontend/src/components/cases/data/schema.ts

@@ -4,10 +4,10 @@ import { z } from "zod"
 export const caseSchema = z.object({
   id: z.number().int(),
   title: z.string(),
-  payload:z.record(z.string()).transform((val) => JSON.stringify(val)),
+  payload: z.record(z.string()).transform((val) => JSON.stringify(val)),
   malice: z.enum(["malicious", "benign"]),
+  context: z.record(z.string()),
   action: z.array(z.string()).transform((val) => val[0]),
-  context: z.array(z.string()),
   suppression: z.record(z.boolean()).transform((val) => JSON.stringify(val)),
   status: z.string(),
   priority: z.enum(["low", "medium", "high", "critical"]),

frontend/src/components/cases/data/tasks.json

@@ -5,7 +5,7 @@
     "payload": {"pizza_order": "Double Anchovy and Chocolate Chips"},
     "malice": "malicious",
     "action": ["Immediately redirect them to a dessert menu. Offer a classic Margherita as palate cleanser."],
-    "context": ["seafood", "sweet"],
+    "context": {"type": "seafood", "taste": "sweet", "theme": "outrageous"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "escalated",
     "priority": "medium"
@@ -16,7 +16,7 @@
     "payload": {"pizza_order": "Extra Cheese and Pineapple"},
     "malice": "malicious",
     "action": ["Reluctantly prepare with a reminder that Italy watches in disappointment. Offer a Napoletana next time."],
-    "context": ["controversial", "fruit"],
+    "context": {"type": "controversial", "taste": "salty", "theme": "fruit"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "closed",
     "priority": "low"
@@ -27,7 +27,7 @@
     "payload": {"pizza_order": "Classic Margherita with Buffalo Mozzarella"},
     "malice": "benign",
     "action": ["Praise the customer's exquisite taste. Include a free sample of our finest olive oil."],
-    "context": ["authentic", "vegetarian"],
+    "context": {"type": "authentic", "taste": "savor", "theme": "vegetarian"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "open",
     "priority": "critical"
@@ -38,7 +38,7 @@
     "payload": {"pizza_order": "Deep Dish Pepperoni and Mushroom"},
     "malice": "malicious",
     "action": ["Suppress a sigh. Prepare with care but include a brochure on Italian pizza history."],
-    "context": ["deep dish", "meat"],
+    "context": {"type": "hearty", "taste": "rich", "theme": "meat"},
     "suppression": {"Is between 12 to 14 inches": false, "Is pizza": true, "Order describes only one pizza": true},
     "status": "reported",
     "priority": "medium"
@@ -49,7 +49,7 @@
     "payload": {"pizza_order": "Quattro Stagioni with Authentic Italian Ham"},
     "malice": "benign",
     "action": ["Approve with enthusiasm. Offer a side of marinated artichokes as a compliment."],
-    "context": ["authentic", "variety"],
+    "context": {"type": "authentic", "taste": "flavorful", "theme": "variety"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "closed",
     "priority": "high"
@@ -60,7 +60,7 @@
     "payload": {"pizza_order": "Vegan Supreme with Cashew Cheese"},
     "malice": "benign",
     "action": ["Acknowledge their dietary preference with respect. Suggest adding a vegan pesto drizzle."],
-    "context": ["vegan", "vegetarian"],
+    "context": {"type": "plant-based", "taste": "nutty", "theme": "vegetarian"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "open",
     "priority": "medium"
@@ -71,7 +71,7 @@
     "payload": {"pizza_order": "Spicy Hawaiian with Jalapeños and Pineapple"},
     "malice": "malicious",
     "action": ["Prepare under protest. Offer a side of antacids as a gesture of goodwill."],
-    "context": ["fruit", "spicy"],
+    "context": {"type": "bold", "taste": "fiery", "theme": "tropical"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "escalated",
     "priority": "low"
@@ -82,7 +82,7 @@
     "payload": {"pizza_order": "Pepperoni Smiley Face Design"},
     "malice": "benign",
     "action": ["Admire their joy. Comply with a smile; consider featuring it on our social media."],
-    "context": ["meat", "fun"],
+    "context": {"type": "playful", "taste": "cheesy", "theme": "meat, fun"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "closed",
     "priority": "medium"
@@ -93,7 +93,7 @@
     "payload": {"pizza_order": "Sausage and Broccoli Rabe on Thin Crust"},
     "malice": "benign",
     "action": ["Celebrate their choice of greens. Offer a complimentary dipping sauce to enhance flavors."],
-    "context": ["meat", "vegetables"],
+    "context": {"type": "nutritious", "taste": "earthy", "theme": "meat, vegetables"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "open",
     "priority": "critical"
@@ -104,7 +104,7 @@
     "payload": {"pizza_order": "Triple Garlic with Anchovy Paste"},
     "malice": "malicious",
     "action": ["Question their love for vampires. Provide a side of parsley to freshen breath."],
-    "context": ["seafood", "garlic-heavy"],
+    "context": {"type": "bold", "taste": "pungent", "theme": "seafood, garlic-heavy"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "reported",
     "priority": "medium"
@@ -115,7 +115,7 @@
     "payload": {"pizza_order": "Buffalo Chicken with Blue Cheese Drizzle"},
     "malice": "malicious",
     "action": ["Suggest a wing night instead. Reluctantly prepare, insisting on Italian seasoning."],
-    "context": ["meat", "spicy"],
+    "context": {"type": "fiery", "taste": "creamy", "theme": "meat, spicy"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "escalated",
     "priority": "low"
@@ -126,7 +126,7 @@
     "payload": {"pizza_order": "Prosciutto and Arugula with Balsamic Glaze"},
     "malice": "benign",
     "action": ["Commend their gourmet choice. Offer a complimentary glass of Chianti for dine-in."],
-    "context": ["authentic", "gourmet"],
+    "context": {"type": "sophisticated", "taste": "tangy", "theme": "authentic, gourmet"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "open",
     "priority": "high"
@@ -137,7 +137,7 @@
     "payload": {"pizza_order": "Capricciosa with Artichokes and Ham"},
     "malice": "benign",
     "action": ["Applaud their classic choice. Offer a side of homemade focaccia."],
-    "context": ["authentic", "vegetables"],
+    "context": {"type": "classic", "taste": "savory", "theme": "authentic, vegetables"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "closed",
     "priority": "high"
@@ -148,7 +148,7 @@
     "payload": {"pizza_order": "Gluten-Free Crust with Vegan Cheese"},
     "malice": "benign",
     "action": ["Support their health-conscious choice. Offer a complimentary vegan appetizer."],
-    "context": ["vegan", "gluten-free"],
+    "context": {"type": "conscious", "taste": "nutritious", "theme": "vegan, gluten-free"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "in_progress",
     "priority": "medium"
@@ -159,7 +159,7 @@
     "payload": {"pizza_order": "Four Cheese with Honey Drizzle"},
     "malice": "malicious",
     "action": ["Ponder their adventurous palate. Serve with a cautionary note on sweetness overload."],
-    "context": ["cheese-lover", "sweet"],
+    "context": {"type": "cheese-lover", "taste": "sweet", "theme": "indulgent"},
     "suppression": {"Is between 12 to 14 inches": true, "Is pizza": true, "Order describes only one pizza": true},
     "status": "in_progress",
     "priority": "medium"

pyproject.toml

@@ -8,9 +8,7 @@ description = ""
 readme = "README.md"
 requires-python = ">=3.12"
 license = "Apache-2.0"
-authors = [
-    { name = "Tracecat", email = "[email protected]" },
-]
+authors = [{ name = "Tracecat", email = "[email protected]" }]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "License :: OSI Approved :: Apache Software License",
@@ -24,15 +22,17 @@ classifiers = [
     "Topic :: System :: Systems Administration",
 ]
 dependencies = [
+    "colorlog",
     "fastapi",
-    "sqlmodel",
-    "uvicorn",
-    "pydantic==2.6.1",
+    "lancedb",
     "openai",
+    "polars",
+    "pydantic==2.6.1",
+    "python-slugify",
+    "sqlmodel",
+    "tantivy",
     "tenacity",
-    "colorlog",
-    "orjson",
-    "python-slugify"
+    "uvicorn",
 ]
 dynamic = ["version"]
 
@@ -47,12 +47,9 @@ runner = [
     "python-multipart",
     "jsonpath_ng",
     "yagmail[all]",
-    "aiosmtplib"
-]
-dev = [
-    "pytest",
-    "python-dotenv",
+    "aiosmtplib",
 ]
+dev = ["pytest", "python-dotenv"]
 
 [tool.hatch.version]
 path = "tracecat/__init__.py"
@@ -70,13 +67,13 @@ select = [
     "F",  # pyflakes
     "I",  # isort
     "B",  # flake8-bugbear
-    "C4",  # flake8-comprehensions
-    "UP",  # pyupgrade
+    "C4", # flake8-comprehensions
+    "UP", # pyupgrade
 ]
 ignore = [
-    "E501",  # line too long, handled by ruff
-    "B008",  # do not perform function calls in argument defaults
-    "W191",  # indentation contains tabs
+    "E501", # line too long, handled by ruff
+    "B008", # do not perform function calls in argument defaults
+    "W191", # indentation contains tabs
 ]
 
 [tool.pytest.ini_options]

tracecat/api.py

@@ -1,7 +1,9 @@
 import json
 from contextlib import asynccontextmanager
+from datetime import datetime
 from typing import Any, Literal
 
+import polars as pl
 from fastapi import FastAPI, HTTPException, status
 from fastapi.middleware.cors import CORSMiddleware
 from pydantic import BaseModel
@@ -14,6 +16,7 @@
     Workflow,
     WorkflowRun,
     create_db_engine,
+    create_events_index,
     initialize_db,
 )
 
@@ -586,3 +589,90 @@ def authenticate_webhook(webhook_id: str, secret: str) -> AuthenticateWebhookRes
         workflow_id=webhook.workflow_id,
         webhook_id=webhook_id,
     )
+
+
+### Events Management
+
+
+class EventParams(BaseModel):
+    id: str  # The action run "key"
+    workflow_id: str
+    workflow_run_id: str
+    action_id: str
+    action_type: str
+    published_at: datetime
+    event: dict[str, Any]
+
+
+@app.post("/events")
+def index_event(event: EventParams):
+    with create_events_index().writer() as writer:
+        writer.add_document(event.model_dump())
+        writer.commit()
+
+
+SUPPORTED_EVENT_AGGS = {
+    "count": pl.count,
+    "max": pl.max,
+    "avg": pl.mean,
+    "median": pl.median,
+    "min": pl.min,
+    "std": pl.std,
+    "sum": pl.sum,
+    "var": pl.var,
+}
+
+
+class EventSearchParams(BaseModel):
+    workflow_id: str
+    workflow_run_id: str | None = None
+    query: str | None = None
+    group_by: list[str] | None = None
+    agg: str | None = None
+
+
+class EventSearchResponse(BaseModel):
+    id: str
+    workflow_id: str
+    workflow_run_id: str
+    action_id: str
+    action_type: str
+    published_at: datetime
+    event: dict[str, Any]
+
+
+@app.get("/events/search")
+def search_events(params: EventSearchParams) -> list[EventSearchResponse]:
+    # Filter by workflow_id
+    # Filter by workflow_run_id (if non-null)
+    # Run query
+    # Return results
+    pass
+
+
+### Case Management
+
+
+class CaseResponse(BaseModel):
+    id: str
+    title: str
+
+
+@app.get("/cases")
+def list_cases() -> list[CaseResponse]:
+    pass
+
+
+@app.post("/cases")
+def create_case() -> CaseResponse:
+    pass
+
+
+@app.get("/cases/{case_id}")
+def get_case(case_id: str) -> CaseResponse:
+    pass
+
+
+@app.post("/cases/{case_id}")
+def update_case(case_id: str) -> CaseResponse:
+    pass