diff --git a/memory-store/migrations/000001_initial.down.sql b/memory-store/migrations/000001_initial.down.sql
index ddc44dbc8..6f5aa4b5c 100644
--- a/memory-store/migrations/000001_initial.down.sql
+++ b/memory-store/migrations/000001_initial.down.sql
@@ -1,17 +1,27 @@
+BEGIN;
+
-- Drop the update_updated_at_column function
-DROP FUNCTION IF EXISTS update_updated_at_column();
+DROP FUNCTION IF EXISTS update_updated_at_column ();
-- Drop misc extensions
DROP EXTENSION IF EXISTS "uuid-ossp" CASCADE;
+
DROP EXTENSION IF EXISTS citext CASCADE;
+
DROP EXTENSION IF EXISTS btree_gist CASCADE;
+
DROP EXTENSION IF EXISTS btree_gin CASCADE;
-- Drop timescale's pgai extensions
DROP EXTENSION IF EXISTS ai CASCADE;
+
DROP EXTENSION IF EXISTS vectorscale CASCADE;
+
DROP EXTENSION IF EXISTS vector CASCADE;
-- Drop timescaledb extensions
DROP EXTENSION IF EXISTS timescaledb_toolkit CASCADE;
+
DROP EXTENSION IF EXISTS timescaledb CASCADE;
+
+COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000001_initial.up.sql b/memory-store/migrations/000001_initial.up.sql
index da04e3c4b..6eba5ab6c 100644
--- a/memory-store/migrations/000001_initial.up.sql
+++ b/memory-store/migrations/000001_initial.up.sql
@@ -2,28 +2,34 @@ BEGIN;
-- init timescaledb
CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
+
CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit CASCADE;
-- add timescale's pgai extension
CREATE EXTENSION IF NOT EXISTS vector CASCADE;
+
CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE;
+
CREATE EXTENSION IF NOT EXISTS ai CASCADE;
-- add misc extensions (for indexing etc)
CREATE EXTENSION IF NOT EXISTS btree_gin CASCADE;
+
CREATE EXTENSION IF NOT EXISTS btree_gist CASCADE;
+
CREATE EXTENSION IF NOT EXISTS citext CASCADE;
+
CREATE EXTENSION IF NOT EXISTS "uuid-ossp" CASCADE;
-- Create function to update the updated_at timestamp
-CREATE OR REPLACE FUNCTION update_updated_at_column()
-RETURNS TRIGGER AS $$
+CREATE
+OR REPLACE FUNCTION update_updated_at_column () RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-COMMENT ON FUNCTION update_updated_at_column() IS 'Trigger function to automatically update updated_at timestamp';
+COMMENT ON FUNCTION update_updated_at_column () IS 'Trigger function to automatically update updated_at timestamp';
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000002_developers.up.sql b/memory-store/migrations/000002_developers.up.sql
index 0802dcf6f..9ca9dca69 100644
--- a/memory-store/migrations/000002_developers.up.sql
+++ b/memory-store/migrations/000002_developers.up.sql
@@ -3,8 +3,10 @@ BEGIN;
-- Create developers table
CREATE TABLE IF NOT EXISTS developers (
developer_id UUID NOT NULL,
- email TEXT NOT NULL CONSTRAINT ct_developers_email_format CHECK (email ~* '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'),
- active BOOLEAN NOT NULL DEFAULT true,
+ email TEXT NOT NULL CONSTRAINT ct_developers_email_format CHECK (
+ email ~* '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'
+ ),
+ active BOOLEAN NOT NULL DEFAULT TRUE,
tags TEXT[] DEFAULT ARRAY[]::TEXT[],
settings JSONB NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -23,7 +25,9 @@ CREATE INDEX IF NOT EXISTS idx_developers_email ON developers (email);
CREATE INDEX IF NOT EXISTS idx_developers_tags ON developers USING GIN (tags);
-- Create partial index for active developers
-CREATE INDEX IF NOT EXISTS idx_developers_active ON developers (developer_id) WHERE active = true;
+CREATE INDEX IF NOT EXISTS idx_developers_active ON developers (developer_id)
+WHERE
+ active = TRUE;
-- Create trigger to automatically update updated_at
DO $$
@@ -39,4 +43,5 @@ $$;
-- Add comment to table
COMMENT ON TABLE developers IS 'Stores developer information including their settings and tags';
+
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000003_users.down.sql b/memory-store/migrations/000003_users.down.sql
index 3b1b98648..41a27bfc4 100644
--- a/memory-store/migrations/000003_users.down.sql
+++ b/memory-store/migrations/000003_users.down.sql
@@ -5,12 +5,14 @@ DROP TRIGGER IF EXISTS update_users_updated_at ON users;
-- Drop indexes
DROP INDEX IF EXISTS users_metadata_gin_idx;
+
DROP INDEX IF EXISTS users_developer_id_idx;
+
DROP INDEX IF EXISTS users_id_sorted_idx;
-- Drop foreign key constraint
ALTER TABLE IF EXISTS users
- DROP CONSTRAINT IF EXISTS users_developer_id_fkey;
+DROP CONSTRAINT IF EXISTS users_developer_id_fkey;
-- Finally drop the table
DROP TABLE IF EXISTS users;
diff --git a/memory-store/migrations/000003_users.up.sql b/memory-store/migrations/000003_users.up.sql
index c32ff48fe..028e40ef5 100644
--- a/memory-store/migrations/000003_users.up.sql
+++ b/memory-store/migrations/000003_users.up.sql
@@ -46,4 +46,5 @@ END $$;
-- Add comment to table (comments are idempotent by default)
COMMENT ON TABLE users IS 'Stores user information linked to developers';
+
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000004_agents.down.sql b/memory-store/migrations/000004_agents.down.sql
index 0504684fb..be81aaa30 100644
--- a/memory-store/migrations/000004_agents.down.sql
+++ b/memory-store/migrations/000004_agents.down.sql
@@ -5,7 +5,9 @@ DROP TRIGGER IF EXISTS trg_agents_updated_at ON agents;
-- Drop indexes
DROP INDEX IF EXISTS idx_agents_metadata;
+
DROP INDEX IF EXISTS idx_agents_developer;
+
DROP INDEX IF EXISTS idx_agents_id_sorted;
-- Drop table (this will automatically drop associated constraints)
diff --git a/memory-store/migrations/000004_agents.up.sql b/memory-store/migrations/000004_agents.up.sql
index 82eb9c84f..32e066f71 100644
--- a/memory-store/migrations/000004_agents.up.sql
+++ b/memory-store/migrations/000004_agents.up.sql
@@ -2,18 +2,31 @@ BEGIN;
-- Drop existing objects if they exist
DROP TRIGGER IF EXISTS trg_agents_updated_at ON agents;
+
DROP INDEX IF EXISTS idx_agents_metadata;
+
DROP INDEX IF EXISTS idx_agents_developer;
+
DROP INDEX IF EXISTS idx_agents_id_sorted;
+
DROP TABLE IF EXISTS agents;
-- Create agents table
CREATE TABLE IF NOT EXISTS agents (
developer_id UUID NOT NULL,
agent_id UUID NOT NULL,
- canonical_name citext NOT NULL CONSTRAINT ct_agents_canonical_name_length CHECK (length(canonical_name) >= 1 AND length(canonical_name) <= 255),
- name TEXT NOT NULL CONSTRAINT ct_agents_name_length CHECK (length(name) >= 1 AND length(name) <= 255),
- about TEXT CONSTRAINT ct_agents_about_length CHECK (about IS NULL OR length(about) <= 1000),
+ canonical_name citext NOT NULL CONSTRAINT ct_agents_canonical_name_length CHECK (
+ length(canonical_name) >= 1
+ AND length(canonical_name) <= 255
+ ),
+ name TEXT NOT NULL CONSTRAINT ct_agents_name_length CHECK (
+ length(name) >= 1
+ AND length(name) <= 255
+ ),
+ about TEXT CONSTRAINT ct_agents_about_length CHECK (
+ about IS NULL
+ OR length(about) <= 1000
+ ),
instructions TEXT[] DEFAULT ARRAY[]::TEXT[],
model TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -29,11 +42,9 @@ CREATE TABLE IF NOT EXISTS agents (
CREATE INDEX IF NOT EXISTS idx_agents_id_sorted ON agents (agent_id DESC);
-- Create foreign key constraint and index on developer_id
-ALTER TABLE agents
- DROP CONSTRAINT IF EXISTS fk_agents_developer,
- ADD CONSTRAINT fk_agents_developer
- FOREIGN KEY (developer_id)
- REFERENCES developers(developer_id);
+ALTER TABLE agents
+DROP CONSTRAINT IF EXISTS fk_agents_developer,
+ADD CONSTRAINT fk_agents_developer FOREIGN KEY (developer_id) REFERENCES developers (developer_id);
CREATE INDEX IF NOT EXISTS idx_agents_developer ON agents (developer_id);
@@ -41,11 +52,12 @@ CREATE INDEX IF NOT EXISTS idx_agents_developer ON agents (developer_id);
CREATE INDEX IF NOT EXISTS idx_agents_metadata ON agents USING GIN (metadata);
-- Create trigger to automatically update updated_at
-CREATE OR REPLACE TRIGGER trg_agents_updated_at
- BEFORE UPDATE ON agents
- FOR EACH ROW
- EXECUTE FUNCTION update_updated_at_column();
+CREATE
+OR REPLACE TRIGGER trg_agents_updated_at BEFORE
+UPDATE ON agents FOR EACH ROW
+EXECUTE FUNCTION update_updated_at_column ();
-- Add comment to table
COMMENT ON TABLE agents IS 'Stores AI agent configurations and metadata for developers';
+
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000005_files.down.sql b/memory-store/migrations/000005_files.down.sql
index 870eac359..80bf6fecd 100644
--- a/memory-store/migrations/000005_files.down.sql
+++ b/memory-store/migrations/000005_files.down.sql
@@ -8,6 +8,7 @@ DROP TABLE IF EXISTS user_files;
-- Drop files table and its dependencies
DROP TRIGGER IF EXISTS trg_files_updated_at ON files;
+
DROP TABLE IF EXISTS files;
COMMIT;
diff --git a/memory-store/migrations/000005_files.up.sql b/memory-store/migrations/000005_files.up.sql
index bf368db9a..ef4c22b3d 100644
--- a/memory-store/migrations/000005_files.up.sql
+++ b/memory-store/migrations/000005_files.up.sql
@@ -4,9 +4,18 @@ BEGIN;
CREATE TABLE IF NOT EXISTS files (
developer_id UUID NOT NULL,
file_id UUID NOT NULL,
- name TEXT NOT NULL CONSTRAINT ct_files_name_length CHECK (length(name) >= 1 AND length(name) <= 255),
- description TEXT DEFAULT NULL CONSTRAINT ct_files_description_length CHECK (description IS NULL OR length(description) <= 1000),
- mime_type TEXT DEFAULT NULL CONSTRAINT ct_files_mime_type_length CHECK (mime_type IS NULL OR length(mime_type) <= 127),
+ name TEXT NOT NULL CONSTRAINT ct_files_name_length CHECK (
+ length(name) >= 1
+ AND length(name) <= 255
+ ),
+ description TEXT DEFAULT NULL CONSTRAINT ct_files_description_length CHECK (
+ description IS NULL
+ OR length(description) <= 1000
+ ),
+ mime_type TEXT DEFAULT NULL CONSTRAINT ct_files_mime_type_length CHECK (
+ mime_type IS NULL
+ OR length(mime_type) <= 127
+ ),
size BIGINT NOT NULL CONSTRAINT ct_files_size_positive CHECK (size > 0),
hash BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
@@ -53,8 +62,8 @@ CREATE TABLE IF NOT EXISTS user_files (
user_id UUID NOT NULL,
file_id UUID NOT NULL,
CONSTRAINT pk_user_files PRIMARY KEY (developer_id, user_id, file_id),
- CONSTRAINT fk_user_files_user FOREIGN KEY (developer_id, user_id) REFERENCES users(developer_id, user_id),
- CONSTRAINT fk_user_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files(developer_id, file_id)
+ CONSTRAINT fk_user_files_user FOREIGN KEY (developer_id, user_id) REFERENCES users (developer_id, user_id),
+ CONSTRAINT fk_user_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files (developer_id, file_id)
);
-- Create index if it doesn't exist
@@ -66,8 +75,8 @@ CREATE TABLE IF NOT EXISTS agent_files (
agent_id UUID NOT NULL,
file_id UUID NOT NULL,
CONSTRAINT pk_agent_files PRIMARY KEY (developer_id, agent_id, file_id),
- CONSTRAINT fk_agent_files_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents(developer_id, agent_id),
- CONSTRAINT fk_agent_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files(developer_id, file_id)
+ CONSTRAINT fk_agent_files_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id),
+ CONSTRAINT fk_agent_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files (developer_id, file_id)
);
-- Create index if it doesn't exist
diff --git a/memory-store/migrations/000006_docs.down.sql b/memory-store/migrations/000006_docs.down.sql
index 50139bb87..468b1b483 100644
--- a/memory-store/migrations/000006_docs.down.sql
+++ b/memory-store/migrations/000006_docs.down.sql
@@ -2,28 +2,40 @@ BEGIN;
-- Drop indexes
DROP INDEX IF EXISTS idx_docs_content_trgm;
+
DROP INDEX IF EXISTS idx_docs_title_trgm;
+
DROP INDEX IF EXISTS idx_docs_search_tsv;
+
DROP INDEX IF EXISTS idx_docs_metadata;
+
DROP INDEX IF EXISTS idx_agent_docs_agent;
+
DROP INDEX IF EXISTS idx_user_docs_user;
+
DROP INDEX IF EXISTS idx_docs_developer;
+
DROP INDEX IF EXISTS idx_docs_id_sorted;
-- Drop triggers
DROP TRIGGER IF EXISTS trg_docs_search_tsv ON docs;
+
DROP TRIGGER IF EXISTS trg_docs_updated_at ON docs;
-- Drop the constraint that depends on is_valid_language function
-ALTER TABLE IF EXISTS docs DROP CONSTRAINT IF EXISTS ct_docs_valid_language;
+ALTER TABLE IF EXISTS docs
+DROP CONSTRAINT IF EXISTS ct_docs_valid_language;
-- Drop functions
-DROP FUNCTION IF EXISTS docs_update_search_tsv();
-DROP FUNCTION IF EXISTS is_valid_language(text);
+DROP FUNCTION IF EXISTS docs_update_search_tsv ();
+
+DROP FUNCTION IF EXISTS is_valid_language (text);
-- Drop tables (in correct order due to foreign key constraints)
DROP TABLE IF EXISTS agent_docs;
+
DROP TABLE IF EXISTS user_docs;
+
DROP TABLE IF EXISTS docs;
COMMIT;
diff --git a/memory-store/migrations/000006_docs.up.sql b/memory-store/migrations/000006_docs.up.sql
index c4a241e65..5b532bbef 100644
--- a/memory-store/migrations/000006_docs.up.sql
+++ b/memory-store/migrations/000006_docs.up.sql
@@ -1,8 +1,8 @@
BEGIN;
-- Create function to validate language (make it OR REPLACE)
-CREATE OR REPLACE FUNCTION is_valid_language(lang text)
-RETURNS boolean AS $$
+CREATE
+OR REPLACE FUNCTION is_valid_language (lang text) RETURNS boolean AS $$
BEGIN
RETURN EXISTS (
SELECT 1 FROM pg_ts_config WHERE cfgname::text = lang
@@ -29,8 +29,7 @@ CREATE TABLE IF NOT EXISTS docs (
CONSTRAINT ct_docs_embedding_dimensions_positive CHECK (embedding_dimensions > 0),
CONSTRAINT ct_docs_valid_modality CHECK (modality IN ('text', 'image', 'mixed')),
CONSTRAINT ct_docs_index_positive CHECK (index >= 0),
- CONSTRAINT ct_docs_valid_language
- CHECK (is_valid_language(language))
+ CONSTRAINT ct_docs_valid_language CHECK (is_valid_language (language))
);
-- Create sorted index on doc_id if not exists
@@ -70,8 +69,8 @@ CREATE TABLE IF NOT EXISTS user_docs (
user_id UUID NOT NULL,
doc_id UUID NOT NULL,
CONSTRAINT pk_user_docs PRIMARY KEY (developer_id, user_id, doc_id),
- CONSTRAINT fk_user_docs_user FOREIGN KEY (developer_id, user_id) REFERENCES users(developer_id, user_id),
- CONSTRAINT fk_user_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs(developer_id, doc_id)
+ CONSTRAINT fk_user_docs_user FOREIGN KEY (developer_id, user_id) REFERENCES users (developer_id, user_id),
+ CONSTRAINT fk_user_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs (developer_id, doc_id)
);
-- Create the agent_docs table
@@ -80,20 +79,26 @@ CREATE TABLE IF NOT EXISTS agent_docs (
agent_id UUID NOT NULL,
doc_id UUID NOT NULL,
CONSTRAINT pk_agent_docs PRIMARY KEY (developer_id, agent_id, doc_id),
- CONSTRAINT fk_agent_docs_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents(developer_id, agent_id),
- CONSTRAINT fk_agent_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs(developer_id, doc_id)
+ CONSTRAINT fk_agent_docs_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id),
+ CONSTRAINT fk_agent_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs (developer_id, doc_id)
);
-- Create indexes if not exists
CREATE INDEX IF NOT EXISTS idx_user_docs_user ON user_docs (developer_id, user_id);
+
CREATE INDEX IF NOT EXISTS idx_agent_docs_agent ON agent_docs (developer_id, agent_id);
+
CREATE INDEX IF NOT EXISTS idx_docs_metadata ON docs USING GIN (metadata);
-- Enable necessary PostgreSQL extensions
CREATE EXTENSION IF NOT EXISTS unaccent;
+
CREATE EXTENSION IF NOT EXISTS pg_trgm;
+
CREATE EXTENSION IF NOT EXISTS dict_int CASCADE;
+
CREATE EXTENSION IF NOT EXISTS dict_xsyn CASCADE;
+
CREATE EXTENSION IF NOT EXISTS fuzzystrmatch CASCADE;
-- Configure text search for all supported languages
@@ -132,8 +137,8 @@ BEGIN
END $$;
-- Create function to update tsvector
-CREATE OR REPLACE FUNCTION docs_update_search_tsv()
-RETURNS trigger AS $$
+CREATE
+OR REPLACE FUNCTION docs_update_search_tsv () RETURNS trigger AS $$
BEGIN
NEW.search_tsv :=
setweight(to_tsvector(NEW.language::regconfig, unaccent(coalesce(NEW.title, ''))), 'A') ||
@@ -158,13 +163,28 @@ END $$;
-- Create indexes if not exists
CREATE INDEX IF NOT EXISTS idx_docs_search_tsv ON docs USING GIN (search_tsv);
+
CREATE INDEX IF NOT EXISTS idx_docs_title_trgm ON docs USING GIN (title gin_trgm_ops);
+
CREATE INDEX IF NOT EXISTS idx_docs_content_trgm ON docs USING GIN (content gin_trgm_ops);
-- Update existing rows (if any)
-UPDATE docs SET search_tsv =
- setweight(to_tsvector(language::regconfig, unaccent(coalesce(title, ''))), 'A') ||
- setweight(to_tsvector(language::regconfig, unaccent(coalesce(content, ''))), 'B')
-WHERE search_tsv IS NULL;
+UPDATE docs
+SET
+ search_tsv = setweight(
+ to_tsvector(
+ language::regconfig,
+ unaccent (coalesce(title, ''))
+ ),
+ 'A'
+ ) || setweight(
+ to_tsvector(
+ language::regconfig,
+ unaccent (coalesce(content, ''))
+ ),
+ 'B'
+ )
+WHERE
+ search_tsv IS NULL;
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000007_ann.up.sql b/memory-store/migrations/000007_ann.up.sql
index 0b08e9b07..3cc606fde 100644
--- a/memory-store/migrations/000007_ann.up.sql
+++ b/memory-store/migrations/000007_ann.up.sql
@@ -1,37 +1,41 @@
-- Create vector similarity search index using diskann and timescale vectorizer
-SELECT ai.create_vectorizer(
- source => 'docs',
- destination => 'docs_embeddings',
- embedding => ai.embedding_voyageai('voyage-3', 1024), -- need to parameterize this
- -- actual chunking is managed by the docs table
- -- this is to prevent running out of context window
- chunking => ai.chunking_recursive_character_text_splitter(
- chunk_column => 'content',
- chunk_size => 30000, -- 30k characters ~= 7.5k tokens
- chunk_overlap => 600, -- 600 characters ~= 150 tokens
- separators => array[ -- tries separators in order
- -- markdown headers
- E'\n#',
- E'\n##',
- E'\n###',
- E'\n---',
- E'\n***',
- -- html tags
- E'', -- Split on major document sections
- E'', -- Split on div boundaries
- E'',
- E'
', -- Split on paragraphs
- E'
', -- Split on line breaks
- -- other separators
- E'\n\n', -- paragraphs
- '. ', '? ', '! ', '; ', -- sentences (note space after punctuation)
- E'\n', -- line breaks
- ' ' -- words (last resort)
- ]
- ),
- scheduling => ai.scheduling_timescaledb(),
- indexing => ai.indexing_diskann(),
- formatting => ai.formatting_python_template(E'Title: $title\n\n$chunk'),
- processing => ai.processing_default(),
- enqueue_existing => true
-);
\ No newline at end of file
+SELECT
+ ai.create_vectorizer (
+ source => 'docs',
+ destination => 'docs_embeddings',
+ embedding => ai.embedding_voyageai ('voyage-3', 1024), -- need to parameterize this
+ -- actual chunking is managed by the docs table
+ -- this is to prevent running out of context window
+ chunking => ai.chunking_recursive_character_text_splitter (
+ chunk_column => 'content',
+ chunk_size => 30000, -- 30k characters ~= 7.5k tokens
+ chunk_overlap => 600, -- 600 characters ~= 150 tokens
+ separators => ARRAY[ -- tries separators in order
+ -- markdown headers
+ E'\n#',
+ E'\n##',
+ E'\n###',
+ E'\n---',
+ E'\n***',
+ -- html tags
+ E'', -- Split on major document sections
+ E'', -- Split on div boundaries
+ E'',
+ E'', -- Split on paragraphs
+ E'
', -- Split on line breaks
+ -- other separators
+ E'\n\n', -- paragraphs
+ '. ',
+ '? ',
+ '! ',
+ '; ', -- sentences (note space after punctuation)
+ E'\n', -- line breaks
+ ' ' -- words (last resort)
+ ]
+ ),
+ scheduling => ai.scheduling_timescaledb (),
+ indexing => ai.indexing_diskann (),
+ formatting => ai.formatting_python_template (E'Title: $title\n\n$chunk'),
+ processing => ai.processing_default (),
+ enqueue_existing => TRUE
+ );
\ No newline at end of file
diff --git a/memory-store/migrations/000008_tools.up.sql b/memory-store/migrations/000008_tools.up.sql
index bcf59def8..159ef3688 100644
--- a/memory-store/migrations/000008_tools.up.sql
+++ b/memory-store/migrations/000008_tools.up.sql
@@ -7,13 +7,21 @@ CREATE TABLE IF NOT EXISTS tools (
tool_id UUID NOT NULL,
task_id UUID DEFAULT NULL,
task_version INT DEFAULT NULL,
- type TEXT NOT NULL CONSTRAINT ct_tools_type_length CHECK (length(type) >= 1 AND length(type) <= 255),
- name TEXT NOT NULL CONSTRAINT ct_tools_name_length CHECK (length(name) >= 1 AND length(name) <= 255),
- description TEXT CONSTRAINT ct_tools_description_length CHECK (description IS NULL OR length(description) <= 1000),
+ type TEXT NOT NULL CONSTRAINT ct_tools_type_length CHECK (
+ length(type) >= 1
+ AND length(type) <= 255
+ ),
+ name TEXT NOT NULL CONSTRAINT ct_tools_name_length CHECK (
+ length(name) >= 1
+ AND length(name) <= 255
+ ),
+ description TEXT CONSTRAINT ct_tools_description_length CHECK (
+ description IS NULL
+ OR length(description) <= 1000
+ ),
spec JSONB NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
CONSTRAINT pk_tools PRIMARY KEY (developer_id, agent_id, tool_id, type, name)
);
@@ -21,7 +29,9 @@ CREATE TABLE IF NOT EXISTS tools (
CREATE INDEX IF NOT EXISTS idx_tools_id_sorted ON tools (tool_id DESC);
-- Create sorted index on task_id if it doesn't exist
-CREATE INDEX IF NOT EXISTS idx_tools_task_id_sorted ON tools (task_id DESC) WHERE task_id IS NOT NULL;
+CREATE INDEX IF NOT EXISTS idx_tools_task_id_sorted ON tools (task_id DESC)
+WHERE
+ task_id IS NOT NULL;
-- Create foreign key constraint and index if they don't exist
DO $$ BEGIN
@@ -39,11 +49,12 @@ CREATE INDEX IF NOT EXISTS idx_tools_developer_agent ON tools (developer_id, age
-- Drop trigger if exists and recreate
DROP TRIGGER IF EXISTS trg_tools_updated_at ON tools;
-CREATE TRIGGER trg_tools_updated_at
- BEFORE UPDATE ON tools
- FOR EACH ROW
- EXECUTE FUNCTION update_updated_at_column();
+
+CREATE TRIGGER trg_tools_updated_at BEFORE
+UPDATE ON tools FOR EACH ROW
+EXECUTE FUNCTION update_updated_at_column ();
-- Add comment to table
COMMENT ON TABLE tools IS 'Stores tool configurations and specifications for AI agents';
+
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000009_sessions.down.sql b/memory-store/migrations/000009_sessions.down.sql
index d1c0b2911..33d535e53 100644
--- a/memory-store/migrations/000009_sessions.down.sql
+++ b/memory-store/migrations/000009_sessions.down.sql
@@ -2,16 +2,18 @@ BEGIN;
-- Drop triggers first
DROP TRIGGER IF EXISTS trg_validate_participant_before_update ON session_lookup;
+
DROP TRIGGER IF EXISTS trg_validate_participant_before_insert ON session_lookup;
-- Drop the validation function
-DROP FUNCTION IF EXISTS validate_participant();
+DROP FUNCTION IF EXISTS validate_participant ();
-- Drop session_lookup table and its indexes
DROP TABLE IF EXISTS session_lookup;
-- Drop sessions table and its indexes
DROP TRIGGER IF EXISTS trg_sessions_updated_at ON sessions;
+
DROP TABLE IF EXISTS sessions CASCADE;
-- Drop the enum type
diff --git a/memory-store/migrations/000009_sessions.up.sql b/memory-store/migrations/000009_sessions.up.sql
index 30f135ed7..71e83b7ec 100644
--- a/memory-store/migrations/000009_sessions.up.sql
+++ b/memory-store/migrations/000009_sessions.up.sql
@@ -7,19 +7,21 @@ CREATE TABLE IF NOT EXISTS sessions (
situation TEXT,
system_template TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- -- TODO: Derived from entries
+ -- NOTE: Derived from entries
-- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
metadata JSONB NOT NULL DEFAULT '{}'::JSONB,
- render_templates BOOLEAN NOT NULL DEFAULT true,
+ render_templates BOOLEAN NOT NULL DEFAULT TRUE,
token_budget INTEGER,
context_overflow TEXT,
forward_tool_calls BOOLEAN,
recall_options JSONB NOT NULL DEFAULT '{}'::JSONB,
- CONSTRAINT pk_sessions PRIMARY KEY (developer_id, session_id)
+ CONSTRAINT pk_sessions PRIMARY KEY (developer_id, session_id),
+ CONSTRAINT uq_sessions_session_id UNIQUE (session_id)
);
-- Create indexes if they don't exist
CREATE INDEX IF NOT EXISTS idx_sessions_id_sorted ON sessions (session_id DESC);
+
CREATE INDEX IF NOT EXISTS idx_sessions_metadata ON sessions USING GIN (metadata);
-- Create foreign key if it doesn't exist
@@ -62,16 +64,23 @@ CREATE TABLE IF NOT EXISTS session_lookup (
session_id UUID NOT NULL,
participant_type participant_type NOT NULL,
participant_id UUID NOT NULL,
- PRIMARY KEY (developer_id, session_id, participant_type, participant_id),
- FOREIGN KEY (developer_id, session_id) REFERENCES sessions(developer_id, session_id)
+ PRIMARY KEY (
+ developer_id,
+ session_id,
+ participant_type,
+ participant_id
+ ),
+ FOREIGN KEY (developer_id, session_id) REFERENCES sessions (developer_id, session_id)
);
-- Create indexes if they don't exist
CREATE INDEX IF NOT EXISTS idx_session_lookup_by_session ON session_lookup (developer_id, session_id);
+
CREATE INDEX IF NOT EXISTS idx_session_lookup_by_participant ON session_lookup (developer_id, participant_id);
-- Create or replace the validation function
-CREATE OR REPLACE FUNCTION validate_participant() RETURNS trigger AS $$
+CREATE
+OR REPLACE FUNCTION validate_participant () RETURNS trigger AS $$
BEGIN
IF NEW.participant_type = 'user' THEN
PERFORM 1 FROM users WHERE developer_id = NEW.developer_id AND user_id = NEW.participant_id;
@@ -101,7 +110,6 @@ BEGIN
FOR EACH ROW
EXECUTE FUNCTION validate_participant();
END IF;
-
IF NOT EXISTS (
SELECT 1 FROM pg_trigger WHERE tgname = 'trg_validate_participant_before_update'
) THEN
diff --git a/memory-store/migrations/000010_tasks.down.sql b/memory-store/migrations/000010_tasks.down.sql
index b7f758779..84608ea71 100644
--- a/memory-store/migrations/000010_tasks.down.sql
+++ b/memory-store/migrations/000010_tasks.down.sql
@@ -4,11 +4,16 @@ BEGIN;
DO $$
BEGIN
IF EXISTS (
- SELECT 1
- FROM information_schema.table_constraints
- WHERE constraint_name = 'fk_tools_task_id'
+ SELECT
+ 1
+ FROM
+ information_schema.table_constraints
+ WHERE
+ constraint_name = 'fk_tools_task_id'
) THEN
- ALTER TABLE tools DROP CONSTRAINT fk_tools_task_id;
+ ALTER TABLE tools
+ DROP CONSTRAINT fk_tools_task_id;
+
END IF;
END $$;
diff --git a/memory-store/migrations/000010_tasks.up.sql b/memory-store/migrations/000010_tasks.up.sql
index c2bfeb454..2ba6b7910 100644
--- a/memory-store/migrations/000010_tasks.up.sql
+++ b/memory-store/migrations/000010_tasks.up.sql
@@ -3,13 +3,22 @@ BEGIN;
-- Create tasks table if it doesn't exist
CREATE TABLE IF NOT EXISTS tasks (
developer_id UUID NOT NULL,
- canonical_name CITEXT NOT NULL CONSTRAINT ct_tasks_canonical_name_length CHECK (length(canonical_name) >= 1 AND length(canonical_name) <= 255),
+ canonical_name CITEXT NOT NULL CONSTRAINT ct_tasks_canonical_name_length CHECK (
+ length(canonical_name) >= 1
+ AND length(canonical_name) <= 255
+ ),
agent_id UUID NOT NULL,
task_id UUID NOT NULL,
- version INTEGER NOT NULL DEFAULT 1,
+ VERSION INTEGER NOT NULL DEFAULT 1,
updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- name TEXT NOT NULL CONSTRAINT ct_tasks_name_length CHECK (length(name) >= 1 AND length(name) <= 255),
- description TEXT DEFAULT NULL CONSTRAINT ct_tasks_description_length CHECK (description IS NULL OR length(description) <= 1000),
+ name TEXT NOT NULL CONSTRAINT ct_tasks_name_length CHECK (
+ length(name) >= 1
+ AND length(name) <= 255
+ ),
+ description TEXT DEFAULT NULL CONSTRAINT ct_tasks_description_length CHECK (
+ description IS NULL
+ OR length(description) <= 1000
+ ),
input_schema JSON NOT NULL,
inherit_tools BOOLEAN DEFAULT FALSE,
workflows JSON[] DEFAULT ARRAY[]::JSON[],
@@ -17,10 +26,8 @@ CREATE TABLE IF NOT EXISTS tasks (
metadata JSONB DEFAULT '{}'::JSONB,
CONSTRAINT pk_tasks PRIMARY KEY (developer_id, task_id),
CONSTRAINT uq_tasks_canonical_name_unique UNIQUE (developer_id, canonical_name),
- CONSTRAINT uq_tasks_version_unique UNIQUE (task_id, version),
- CONSTRAINT fk_tasks_agent
- FOREIGN KEY (developer_id, agent_id)
- REFERENCES agents(developer_id, agent_id),
+ CONSTRAINT uq_tasks_version_unique UNIQUE (task_id, VERSION),
+ CONSTRAINT fk_tasks_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id),
CONSTRAINT ct_tasks_canonical_name_valid_identifier CHECK (canonical_name ~ '^[a-zA-Z][a-zA-Z0-9_]*$')
);
diff --git a/memory-store/migrations/000011_executions.up.sql b/memory-store/migrations/000011_executions.up.sql
index 74ab5bf97..cf0666136 100644
--- a/memory-store/migrations/000011_executions.up.sql
+++ b/memory-store/migrations/000011_executions.up.sql
@@ -7,21 +7,16 @@ CREATE TABLE IF NOT EXISTS executions (
task_version INTEGER NOT NULL,
execution_id UUID NOT NULL,
input JSONB NOT NULL,
-
-- NOTE: These will be generated using continuous aggregates from transitions
-- status TEXT DEFAULT 'pending',
-- output JSONB DEFAULT NULL,
-- error TEXT DEFAULT NULL,
-- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
metadata JSONB NOT NULL DEFAULT '{}'::JSONB,
created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
-
CONSTRAINT pk_executions PRIMARY KEY (execution_id),
- CONSTRAINT fk_executions_developer
- FOREIGN KEY (developer_id) REFERENCES developers(developer_id),
- CONSTRAINT fk_executions_task
- FOREIGN KEY (developer_id, task_id) REFERENCES tasks(developer_id, task_id)
+ CONSTRAINT fk_executions_developer FOREIGN KEY (developer_id) REFERENCES developers (developer_id),
+ CONSTRAINT fk_executions_task FOREIGN KEY (developer_id, task_id) REFERENCES tasks (developer_id, task_id)
);
-- Create sorted index on execution_id (optimized for UUID v7)
@@ -38,4 +33,5 @@ CREATE INDEX IF NOT EXISTS idx_executions_metadata ON executions USING GIN (meta
-- Add comment to table (comments are idempotent by default)
COMMENT ON TABLE executions IS 'Stores executions associated with AI agents for developers';
+
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000012_transitions.down.sql b/memory-store/migrations/000012_transitions.down.sql
index 590ebc901..faac2e308 100644
--- a/memory-store/migrations/000012_transitions.down.sql
+++ b/memory-store/migrations/000012_transitions.down.sql
@@ -1,15 +1,20 @@
BEGIN;
-- Drop foreign key constraint if exists
-ALTER TABLE IF EXISTS transitions
- DROP CONSTRAINT IF EXISTS fk_transitions_execution;
+ALTER TABLE IF EXISTS transitions
+DROP CONSTRAINT IF EXISTS fk_transitions_execution;
-- Drop indexes if they exist
DROP INDEX IF EXISTS idx_transitions_metadata;
+
DROP INDEX IF EXISTS idx_transitions_execution_id_sorted;
+
DROP INDEX IF EXISTS idx_transitions_transition_id_sorted;
+
DROP INDEX IF EXISTS idx_transitions_label;
+
DROP INDEX IF EXISTS idx_transitions_next;
+
DROP INDEX IF EXISTS idx_transitions_current;
-- Drop the transitions table (this will also remove it from hypertables)
@@ -17,10 +22,12 @@ DROP TABLE IF EXISTS transitions;
-- Drop custom types if they exist
DROP TYPE IF EXISTS transition_cursor;
+
DROP TYPE IF EXISTS transition_type;
-- Drop the trigger and function for transition validation
DROP TRIGGER IF EXISTS validate_transition ON transitions;
-DROP FUNCTION IF EXISTS check_valid_transition();
+
+DROP FUNCTION IF EXISTS check_valid_transition ();
COMMIT;
diff --git a/memory-store/migrations/000012_transitions.up.sql b/memory-store/migrations/000012_transitions.up.sql
index 515af713c..6fd7dbcd1 100644
--- a/memory-store/migrations/000012_transitions.up.sql
+++ b/memory-store/migrations/000012_transitions.up.sql
@@ -46,8 +46,19 @@ CREATE TABLE IF NOT EXISTS transitions (
);
-- Convert to hypertable if not already
-SELECT create_hypertable('transitions', by_range('created_at', INTERVAL '1 day'), if_not_exists => TRUE);
-SELECT add_dimension('transitions', by_hash('execution_id', 2), if_not_exists => TRUE);
+SELECT
+ create_hypertable (
+ 'transitions',
+ by_range ('created_at', INTERVAL '1 day'),
+ if_not_exists => TRUE
+ );
+
+SELECT
+ add_dimension (
+ 'transitions',
+ by_hash ('execution_id', 2),
+ if_not_exists => TRUE
+ );
-- Create indexes if they don't exist
DO $$
@@ -94,7 +105,8 @@ END $$;
COMMENT ON TABLE transitions IS 'Stores transitions associated with AI agents for developers';
-- Create a trigger function that checks for valid transitions
-CREATE OR REPLACE FUNCTION check_valid_transition() RETURNS trigger AS $$
+CREATE
+OR REPLACE FUNCTION check_valid_transition () RETURNS trigger AS $$
DECLARE
previous_type transition_type;
valid_next_types transition_type[];
@@ -146,9 +158,7 @@ END;
$$ LANGUAGE plpgsql;
-- Create a trigger on the transitions table
-CREATE TRIGGER validate_transition
-BEFORE INSERT ON transitions
-FOR EACH ROW
-EXECUTE FUNCTION check_valid_transition();
+CREATE TRIGGER validate_transition BEFORE INSERT ON transitions FOR EACH ROW
+EXECUTE FUNCTION check_valid_transition ();
COMMIT;
\ No newline at end of file
diff --git a/memory-store/migrations/000013_executions_continuous_view.down.sql b/memory-store/migrations/000013_executions_continuous_view.down.sql
index d833ca4d4..fcab7b023 100644
--- a/memory-store/migrations/000013_executions_continuous_view.down.sql
+++ b/memory-store/migrations/000013_executions_continuous_view.down.sql
@@ -1,13 +1,15 @@
BEGIN;
-- Drop the continuous aggregate policy
-SELECT remove_continuous_aggregate_policy('latest_transitions');
+SELECT
+ remove_continuous_aggregate_policy ('latest_transitions');
-- Drop the views
DROP VIEW IF EXISTS latest_executions;
+
DROP MATERIALIZED VIEW IF EXISTS latest_transitions;
-- Drop the helper function
-DROP FUNCTION IF EXISTS to_text(transition_type);
+DROP FUNCTION IF EXISTS to_text (transition_type);
COMMIT;
diff --git a/memory-store/migrations/000013_executions_continuous_view.up.sql b/memory-store/migrations/000013_executions_continuous_view.up.sql
index b33530824..43285efbc 100644
--- a/memory-store/migrations/000013_executions_continuous_view.up.sql
+++ b/memory-store/migrations/000013_executions_continuous_view.up.sql
@@ -1,39 +1,39 @@
BEGIN;
-- create a function to convert transition_type to text (needed coz ::text is stable not immutable)
-create or replace function to_text(transition_type)
-RETURNS text AS
-$$
+CREATE
+OR REPLACE function to_text (transition_type) RETURNS text AS $$
select $1
$$ STRICT IMMUTABLE LANGUAGE sql;
-- create a continuous view that aggregates the transitions table
-create materialized view if not exists latest_transitions
-with
+CREATE MATERIALIZED VIEW IF NOT EXISTS latest_transitions
+WITH
(
timescaledb.continuous,
- timescaledb.materialized_only = false
- ) as
-select
- time_bucket ('1 day', created_at) as bucket,
+ timescaledb.materialized_only = FALSE
+ ) AS
+SELECT
+ time_bucket ('1 day', created_at) AS bucket,
execution_id,
- count(*) as total_transitions,
- state_agg (created_at, to_text (type)) as state,
- max(created_at) as created_at,
- last (type, created_at) as type,
- last (step_definition, created_at) as step_definition,
- last (step_label, created_at) as step_label,
- last (current_step, created_at) as current_step,
- last (next_step, created_at) as next_step,
- last (output, created_at) as output,
- last (task_token, created_at) as task_token,
- last (metadata, created_at) as metadata
-from
+ count(*) AS total_transitions,
+ state_agg (created_at, to_text (type)) AS state,
+ max(created_at) AS created_at,
+ last (type, created_at) AS type,
+ last (step_definition, created_at) AS step_definition,
+ last (step_label, created_at) AS step_label,
+ last (current_step, created_at) AS current_step,
+ last (next_step, created_at) AS next_step,
+ last (output, created_at) AS output,
+ last (task_token, created_at) AS task_token,
+ last (metadata, created_at) AS metadata
+FROM
transitions
-group by
+GROUP BY
bucket,
execution_id
-with no data;
+WITH
+ no data;
SELECT
add_continuous_aggregate_policy (
@@ -44,7 +44,7 @@ SELECT
);
-- Create a view that combines executions with their latest transitions
-create or replace view latest_executions as
+CREATE OR REPLACE VIEW latest_executions AS
SELECT
e.developer_id,
e.task_id,
@@ -53,7 +53,7 @@ SELECT
e.input,
e.metadata,
e.created_at,
- lt.created_at as updated_at,
+ lt.created_at AS updated_at,
-- Map transition types to status using CASE statement
CASE lt.type::text
WHEN 'init' THEN 'starting'
@@ -66,20 +66,20 @@ SELECT
WHEN 'error' THEN 'failed'
WHEN 'cancelled' THEN 'cancelled'
ELSE 'queued'
- END as status,
+ END AS status,
lt.output,
-- Extract error from output if type is 'error'
CASE
WHEN lt.type::text = 'error' THEN lt.output ->> 'error'
ELSE NULL
- END as error,
+ END AS error,
lt.total_transitions,
lt.current_step,
lt.next_step,
lt.step_definition,
lt.step_label,
lt.task_token,
- lt.metadata as transition_metadata
+ lt.metadata AS transition_metadata
FROM
executions e,
latest_transitions lt
diff --git a/memory-store/migrations/000014_temporal_lookup.up.sql b/memory-store/migrations/000014_temporal_lookup.up.sql
index 1650ab3ac..724ee1340 100644
--- a/memory-store/migrations/000014_temporal_lookup.up.sql
+++ b/memory-store/migrations/000014_temporal_lookup.up.sql
@@ -1,17 +1,16 @@
BEGIN;
-- Create temporal_executions_lookup table
-CREATE TABLE
- IF NOT EXISTS temporal_executions_lookup (
- execution_id UUID NOT NULL,
- id TEXT NOT NULL,
- run_id TEXT,
- first_execution_run_id TEXT,
- result_run_id TEXT,
- created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
- CONSTRAINT pk_temporal_executions_lookup PRIMARY KEY (execution_id, id),
- CONSTRAINT fk_temporal_executions_lookup_execution FOREIGN KEY (execution_id) REFERENCES executions (execution_id)
- );
+CREATE TABLE IF NOT EXISTS temporal_executions_lookup (
+ execution_id UUID NOT NULL,
+ id TEXT NOT NULL,
+ run_id TEXT,
+ first_execution_run_id TEXT,
+ result_run_id TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ CONSTRAINT pk_temporal_executions_lookup PRIMARY KEY (execution_id, id),
+ CONSTRAINT fk_temporal_executions_lookup_execution FOREIGN KEY (execution_id) REFERENCES executions (execution_id)
+);
-- Create sorted index on execution_id (optimized for UUID v7)
CREATE INDEX IF NOT EXISTS idx_temporal_executions_lookup_execution_id_sorted ON temporal_executions_lookup (execution_id DESC);