diff --git a/memory-store/migrations/000001_initial.down.sql b/memory-store/migrations/000001_initial.down.sql index ddc44dbc8..6f5aa4b5c 100644 --- a/memory-store/migrations/000001_initial.down.sql +++ b/memory-store/migrations/000001_initial.down.sql @@ -1,17 +1,27 @@ +BEGIN; + -- Drop the update_updated_at_column function -DROP FUNCTION IF EXISTS update_updated_at_column(); +DROP FUNCTION IF EXISTS update_updated_at_column (); -- Drop misc extensions DROP EXTENSION IF EXISTS "uuid-ossp" CASCADE; + DROP EXTENSION IF EXISTS citext CASCADE; + DROP EXTENSION IF EXISTS btree_gist CASCADE; + DROP EXTENSION IF EXISTS btree_gin CASCADE; -- Drop timescale's pgai extensions DROP EXTENSION IF EXISTS ai CASCADE; + DROP EXTENSION IF EXISTS vectorscale CASCADE; + DROP EXTENSION IF EXISTS vector CASCADE; -- Drop timescaledb extensions DROP EXTENSION IF EXISTS timescaledb_toolkit CASCADE; + DROP EXTENSION IF EXISTS timescaledb CASCADE; + +COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000001_initial.up.sql b/memory-store/migrations/000001_initial.up.sql index da04e3c4b..6eba5ab6c 100644 --- a/memory-store/migrations/000001_initial.up.sql +++ b/memory-store/migrations/000001_initial.up.sql @@ -2,28 +2,34 @@ BEGIN; -- init timescaledb CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; + CREATE EXTENSION IF NOT EXISTS timescaledb_toolkit CASCADE; -- add timescale's pgai extension CREATE EXTENSION IF NOT EXISTS vector CASCADE; + CREATE EXTENSION IF NOT EXISTS vectorscale CASCADE; + CREATE EXTENSION IF NOT EXISTS ai CASCADE; -- add misc extensions (for indexing etc) CREATE EXTENSION IF NOT EXISTS btree_gin CASCADE; + CREATE EXTENSION IF NOT EXISTS btree_gist CASCADE; + CREATE EXTENSION IF NOT EXISTS citext CASCADE; + CREATE EXTENSION IF NOT EXISTS "uuid-ossp" CASCADE; -- Create function to update the updated_at timestamp -CREATE OR REPLACE FUNCTION update_updated_at_column() -RETURNS TRIGGER AS $$ +CREATE +OR REPLACE FUNCTION update_updated_at_column () RETURNS TRIGGER AS $$ BEGIN NEW.updated_at = CURRENT_TIMESTAMP; RETURN NEW; END; $$ language 'plpgsql'; -COMMENT ON FUNCTION update_updated_at_column() IS 'Trigger function to automatically update updated_at timestamp'; +COMMENT ON FUNCTION update_updated_at_column () IS 'Trigger function to automatically update updated_at timestamp'; COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000002_developers.up.sql b/memory-store/migrations/000002_developers.up.sql index 0802dcf6f..9ca9dca69 100644 --- a/memory-store/migrations/000002_developers.up.sql +++ b/memory-store/migrations/000002_developers.up.sql @@ -3,8 +3,10 @@ BEGIN; -- Create developers table CREATE TABLE IF NOT EXISTS developers ( developer_id UUID NOT NULL, - email TEXT NOT NULL CONSTRAINT ct_developers_email_format CHECK (email ~* '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'), - active BOOLEAN NOT NULL DEFAULT true, + email TEXT NOT NULL CONSTRAINT ct_developers_email_format CHECK ( + email ~* '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$' + ), + active BOOLEAN NOT NULL DEFAULT TRUE, tags TEXT[] DEFAULT ARRAY[]::TEXT[], settings JSONB NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -23,7 +25,9 @@ CREATE INDEX IF NOT EXISTS idx_developers_email ON developers (email); CREATE INDEX IF NOT EXISTS idx_developers_tags ON developers USING GIN (tags); -- Create partial index for active developers -CREATE INDEX IF NOT EXISTS idx_developers_active ON developers (developer_id) WHERE active = true; +CREATE INDEX IF NOT EXISTS idx_developers_active ON developers (developer_id) +WHERE + active = TRUE; -- Create trigger to automatically update updated_at DO $$ @@ -39,4 +43,5 @@ $$; -- Add comment to table COMMENT ON TABLE developers IS 'Stores developer information including their settings and tags'; + COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000003_users.down.sql b/memory-store/migrations/000003_users.down.sql index 3b1b98648..41a27bfc4 100644 --- a/memory-store/migrations/000003_users.down.sql +++ b/memory-store/migrations/000003_users.down.sql @@ -5,12 +5,14 @@ DROP TRIGGER IF EXISTS update_users_updated_at ON users; -- Drop indexes DROP INDEX IF EXISTS users_metadata_gin_idx; + DROP INDEX IF EXISTS users_developer_id_idx; + DROP INDEX IF EXISTS users_id_sorted_idx; -- Drop foreign key constraint ALTER TABLE IF EXISTS users - DROP CONSTRAINT IF EXISTS users_developer_id_fkey; +DROP CONSTRAINT IF EXISTS users_developer_id_fkey; -- Finally drop the table DROP TABLE IF EXISTS users; diff --git a/memory-store/migrations/000003_users.up.sql b/memory-store/migrations/000003_users.up.sql index c32ff48fe..028e40ef5 100644 --- a/memory-store/migrations/000003_users.up.sql +++ b/memory-store/migrations/000003_users.up.sql @@ -46,4 +46,5 @@ END $$; -- Add comment to table (comments are idempotent by default) COMMENT ON TABLE users IS 'Stores user information linked to developers'; + COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000004_agents.down.sql b/memory-store/migrations/000004_agents.down.sql index 0504684fb..be81aaa30 100644 --- a/memory-store/migrations/000004_agents.down.sql +++ b/memory-store/migrations/000004_agents.down.sql @@ -5,7 +5,9 @@ DROP TRIGGER IF EXISTS trg_agents_updated_at ON agents; -- Drop indexes DROP INDEX IF EXISTS idx_agents_metadata; + DROP INDEX IF EXISTS idx_agents_developer; + DROP INDEX IF EXISTS idx_agents_id_sorted; -- Drop table (this will automatically drop associated constraints) diff --git a/memory-store/migrations/000004_agents.up.sql b/memory-store/migrations/000004_agents.up.sql index 82eb9c84f..32e066f71 100644 --- a/memory-store/migrations/000004_agents.up.sql +++ b/memory-store/migrations/000004_agents.up.sql @@ -2,18 +2,31 @@ BEGIN; -- Drop existing objects if they exist DROP TRIGGER IF EXISTS trg_agents_updated_at ON agents; + DROP INDEX IF EXISTS idx_agents_metadata; + DROP INDEX IF EXISTS idx_agents_developer; + DROP INDEX IF EXISTS idx_agents_id_sorted; + DROP TABLE IF EXISTS agents; -- Create agents table CREATE TABLE IF NOT EXISTS agents ( developer_id UUID NOT NULL, agent_id UUID NOT NULL, - canonical_name citext NOT NULL CONSTRAINT ct_agents_canonical_name_length CHECK (length(canonical_name) >= 1 AND length(canonical_name) <= 255), - name TEXT NOT NULL CONSTRAINT ct_agents_name_length CHECK (length(name) >= 1 AND length(name) <= 255), - about TEXT CONSTRAINT ct_agents_about_length CHECK (about IS NULL OR length(about) <= 1000), + canonical_name citext NOT NULL CONSTRAINT ct_agents_canonical_name_length CHECK ( + length(canonical_name) >= 1 + AND length(canonical_name) <= 255 + ), + name TEXT NOT NULL CONSTRAINT ct_agents_name_length CHECK ( + length(name) >= 1 + AND length(name) <= 255 + ), + about TEXT CONSTRAINT ct_agents_about_length CHECK ( + about IS NULL + OR length(about) <= 1000 + ), instructions TEXT[] DEFAULT ARRAY[]::TEXT[], model TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -29,11 +42,9 @@ CREATE TABLE IF NOT EXISTS agents ( CREATE INDEX IF NOT EXISTS idx_agents_id_sorted ON agents (agent_id DESC); -- Create foreign key constraint and index on developer_id -ALTER TABLE agents - DROP CONSTRAINT IF EXISTS fk_agents_developer, - ADD CONSTRAINT fk_agents_developer - FOREIGN KEY (developer_id) - REFERENCES developers(developer_id); +ALTER TABLE agents +DROP CONSTRAINT IF EXISTS fk_agents_developer, +ADD CONSTRAINT fk_agents_developer FOREIGN KEY (developer_id) REFERENCES developers (developer_id); CREATE INDEX IF NOT EXISTS idx_agents_developer ON agents (developer_id); @@ -41,11 +52,12 @@ CREATE INDEX IF NOT EXISTS idx_agents_developer ON agents (developer_id); CREATE INDEX IF NOT EXISTS idx_agents_metadata ON agents USING GIN (metadata); -- Create trigger to automatically update updated_at -CREATE OR REPLACE TRIGGER trg_agents_updated_at - BEFORE UPDATE ON agents - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); +CREATE +OR REPLACE TRIGGER trg_agents_updated_at BEFORE +UPDATE ON agents FOR EACH ROW +EXECUTE FUNCTION update_updated_at_column (); -- Add comment to table COMMENT ON TABLE agents IS 'Stores AI agent configurations and metadata for developers'; + COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000005_files.down.sql b/memory-store/migrations/000005_files.down.sql index 870eac359..80bf6fecd 100644 --- a/memory-store/migrations/000005_files.down.sql +++ b/memory-store/migrations/000005_files.down.sql @@ -8,6 +8,7 @@ DROP TABLE IF EXISTS user_files; -- Drop files table and its dependencies DROP TRIGGER IF EXISTS trg_files_updated_at ON files; + DROP TABLE IF EXISTS files; COMMIT; diff --git a/memory-store/migrations/000005_files.up.sql b/memory-store/migrations/000005_files.up.sql index bf368db9a..ef4c22b3d 100644 --- a/memory-store/migrations/000005_files.up.sql +++ b/memory-store/migrations/000005_files.up.sql @@ -4,9 +4,18 @@ BEGIN; CREATE TABLE IF NOT EXISTS files ( developer_id UUID NOT NULL, file_id UUID NOT NULL, - name TEXT NOT NULL CONSTRAINT ct_files_name_length CHECK (length(name) >= 1 AND length(name) <= 255), - description TEXT DEFAULT NULL CONSTRAINT ct_files_description_length CHECK (description IS NULL OR length(description) <= 1000), - mime_type TEXT DEFAULT NULL CONSTRAINT ct_files_mime_type_length CHECK (mime_type IS NULL OR length(mime_type) <= 127), + name TEXT NOT NULL CONSTRAINT ct_files_name_length CHECK ( + length(name) >= 1 + AND length(name) <= 255 + ), + description TEXT DEFAULT NULL CONSTRAINT ct_files_description_length CHECK ( + description IS NULL + OR length(description) <= 1000 + ), + mime_type TEXT DEFAULT NULL CONSTRAINT ct_files_mime_type_length CHECK ( + mime_type IS NULL + OR length(mime_type) <= 127 + ), size BIGINT NOT NULL CONSTRAINT ct_files_size_positive CHECK (size > 0), hash BYTEA NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -53,8 +62,8 @@ CREATE TABLE IF NOT EXISTS user_files ( user_id UUID NOT NULL, file_id UUID NOT NULL, CONSTRAINT pk_user_files PRIMARY KEY (developer_id, user_id, file_id), - CONSTRAINT fk_user_files_user FOREIGN KEY (developer_id, user_id) REFERENCES users(developer_id, user_id), - CONSTRAINT fk_user_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files(developer_id, file_id) + CONSTRAINT fk_user_files_user FOREIGN KEY (developer_id, user_id) REFERENCES users (developer_id, user_id), + CONSTRAINT fk_user_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files (developer_id, file_id) ); -- Create index if it doesn't exist @@ -66,8 +75,8 @@ CREATE TABLE IF NOT EXISTS agent_files ( agent_id UUID NOT NULL, file_id UUID NOT NULL, CONSTRAINT pk_agent_files PRIMARY KEY (developer_id, agent_id, file_id), - CONSTRAINT fk_agent_files_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents(developer_id, agent_id), - CONSTRAINT fk_agent_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files(developer_id, file_id) + CONSTRAINT fk_agent_files_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id), + CONSTRAINT fk_agent_files_file FOREIGN KEY (developer_id, file_id) REFERENCES files (developer_id, file_id) ); -- Create index if it doesn't exist diff --git a/memory-store/migrations/000006_docs.down.sql b/memory-store/migrations/000006_docs.down.sql index 50139bb87..468b1b483 100644 --- a/memory-store/migrations/000006_docs.down.sql +++ b/memory-store/migrations/000006_docs.down.sql @@ -2,28 +2,40 @@ BEGIN; -- Drop indexes DROP INDEX IF EXISTS idx_docs_content_trgm; + DROP INDEX IF EXISTS idx_docs_title_trgm; + DROP INDEX IF EXISTS idx_docs_search_tsv; + DROP INDEX IF EXISTS idx_docs_metadata; + DROP INDEX IF EXISTS idx_agent_docs_agent; + DROP INDEX IF EXISTS idx_user_docs_user; + DROP INDEX IF EXISTS idx_docs_developer; + DROP INDEX IF EXISTS idx_docs_id_sorted; -- Drop triggers DROP TRIGGER IF EXISTS trg_docs_search_tsv ON docs; + DROP TRIGGER IF EXISTS trg_docs_updated_at ON docs; -- Drop the constraint that depends on is_valid_language function -ALTER TABLE IF EXISTS docs DROP CONSTRAINT IF EXISTS ct_docs_valid_language; +ALTER TABLE IF EXISTS docs +DROP CONSTRAINT IF EXISTS ct_docs_valid_language; -- Drop functions -DROP FUNCTION IF EXISTS docs_update_search_tsv(); -DROP FUNCTION IF EXISTS is_valid_language(text); +DROP FUNCTION IF EXISTS docs_update_search_tsv (); + +DROP FUNCTION IF EXISTS is_valid_language (text); -- Drop tables (in correct order due to foreign key constraints) DROP TABLE IF EXISTS agent_docs; + DROP TABLE IF EXISTS user_docs; + DROP TABLE IF EXISTS docs; COMMIT; diff --git a/memory-store/migrations/000006_docs.up.sql b/memory-store/migrations/000006_docs.up.sql index c4a241e65..5b532bbef 100644 --- a/memory-store/migrations/000006_docs.up.sql +++ b/memory-store/migrations/000006_docs.up.sql @@ -1,8 +1,8 @@ BEGIN; -- Create function to validate language (make it OR REPLACE) -CREATE OR REPLACE FUNCTION is_valid_language(lang text) -RETURNS boolean AS $$ +CREATE +OR REPLACE FUNCTION is_valid_language (lang text) RETURNS boolean AS $$ BEGIN RETURN EXISTS ( SELECT 1 FROM pg_ts_config WHERE cfgname::text = lang @@ -29,8 +29,7 @@ CREATE TABLE IF NOT EXISTS docs ( CONSTRAINT ct_docs_embedding_dimensions_positive CHECK (embedding_dimensions > 0), CONSTRAINT ct_docs_valid_modality CHECK (modality IN ('text', 'image', 'mixed')), CONSTRAINT ct_docs_index_positive CHECK (index >= 0), - CONSTRAINT ct_docs_valid_language - CHECK (is_valid_language(language)) + CONSTRAINT ct_docs_valid_language CHECK (is_valid_language (language)) ); -- Create sorted index on doc_id if not exists @@ -70,8 +69,8 @@ CREATE TABLE IF NOT EXISTS user_docs ( user_id UUID NOT NULL, doc_id UUID NOT NULL, CONSTRAINT pk_user_docs PRIMARY KEY (developer_id, user_id, doc_id), - CONSTRAINT fk_user_docs_user FOREIGN KEY (developer_id, user_id) REFERENCES users(developer_id, user_id), - CONSTRAINT fk_user_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs(developer_id, doc_id) + CONSTRAINT fk_user_docs_user FOREIGN KEY (developer_id, user_id) REFERENCES users (developer_id, user_id), + CONSTRAINT fk_user_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs (developer_id, doc_id) ); -- Create the agent_docs table @@ -80,20 +79,26 @@ CREATE TABLE IF NOT EXISTS agent_docs ( agent_id UUID NOT NULL, doc_id UUID NOT NULL, CONSTRAINT pk_agent_docs PRIMARY KEY (developer_id, agent_id, doc_id), - CONSTRAINT fk_agent_docs_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents(developer_id, agent_id), - CONSTRAINT fk_agent_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs(developer_id, doc_id) + CONSTRAINT fk_agent_docs_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id), + CONSTRAINT fk_agent_docs_doc FOREIGN KEY (developer_id, doc_id) REFERENCES docs (developer_id, doc_id) ); -- Create indexes if not exists CREATE INDEX IF NOT EXISTS idx_user_docs_user ON user_docs (developer_id, user_id); + CREATE INDEX IF NOT EXISTS idx_agent_docs_agent ON agent_docs (developer_id, agent_id); + CREATE INDEX IF NOT EXISTS idx_docs_metadata ON docs USING GIN (metadata); -- Enable necessary PostgreSQL extensions CREATE EXTENSION IF NOT EXISTS unaccent; + CREATE EXTENSION IF NOT EXISTS pg_trgm; + CREATE EXTENSION IF NOT EXISTS dict_int CASCADE; + CREATE EXTENSION IF NOT EXISTS dict_xsyn CASCADE; + CREATE EXTENSION IF NOT EXISTS fuzzystrmatch CASCADE; -- Configure text search for all supported languages @@ -132,8 +137,8 @@ BEGIN END $$; -- Create function to update tsvector -CREATE OR REPLACE FUNCTION docs_update_search_tsv() -RETURNS trigger AS $$ +CREATE +OR REPLACE FUNCTION docs_update_search_tsv () RETURNS trigger AS $$ BEGIN NEW.search_tsv := setweight(to_tsvector(NEW.language::regconfig, unaccent(coalesce(NEW.title, ''))), 'A') || @@ -158,13 +163,28 @@ END $$; -- Create indexes if not exists CREATE INDEX IF NOT EXISTS idx_docs_search_tsv ON docs USING GIN (search_tsv); + CREATE INDEX IF NOT EXISTS idx_docs_title_trgm ON docs USING GIN (title gin_trgm_ops); + CREATE INDEX IF NOT EXISTS idx_docs_content_trgm ON docs USING GIN (content gin_trgm_ops); -- Update existing rows (if any) -UPDATE docs SET search_tsv = - setweight(to_tsvector(language::regconfig, unaccent(coalesce(title, ''))), 'A') || - setweight(to_tsvector(language::regconfig, unaccent(coalesce(content, ''))), 'B') -WHERE search_tsv IS NULL; +UPDATE docs +SET + search_tsv = setweight( + to_tsvector( + language::regconfig, + unaccent (coalesce(title, '')) + ), + 'A' + ) || setweight( + to_tsvector( + language::regconfig, + unaccent (coalesce(content, '')) + ), + 'B' + ) +WHERE + search_tsv IS NULL; COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000007_ann.up.sql b/memory-store/migrations/000007_ann.up.sql index 0b08e9b07..3cc606fde 100644 --- a/memory-store/migrations/000007_ann.up.sql +++ b/memory-store/migrations/000007_ann.up.sql @@ -1,37 +1,41 @@ -- Create vector similarity search index using diskann and timescale vectorizer -SELECT ai.create_vectorizer( - source => 'docs', - destination => 'docs_embeddings', - embedding => ai.embedding_voyageai('voyage-3', 1024), -- need to parameterize this - -- actual chunking is managed by the docs table - -- this is to prevent running out of context window - chunking => ai.chunking_recursive_character_text_splitter( - chunk_column => 'content', - chunk_size => 30000, -- 30k characters ~= 7.5k tokens - chunk_overlap => 600, -- 600 characters ~= 150 tokens - separators => array[ -- tries separators in order - -- markdown headers - E'\n#', - E'\n##', - E'\n###', - E'\n---', - E'\n***', - -- html tags - E'', -- Split on major document sections - E'', -- Split on div boundaries - E'', - E'

', -- Split on paragraphs - E'
', -- Split on line breaks - -- other separators - E'\n\n', -- paragraphs - '. ', '? ', '! ', '; ', -- sentences (note space after punctuation) - E'\n', -- line breaks - ' ' -- words (last resort) - ] - ), - scheduling => ai.scheduling_timescaledb(), - indexing => ai.indexing_diskann(), - formatting => ai.formatting_python_template(E'Title: $title\n\n$chunk'), - processing => ai.processing_default(), - enqueue_existing => true -); \ No newline at end of file +SELECT + ai.create_vectorizer ( + source => 'docs', + destination => 'docs_embeddings', + embedding => ai.embedding_voyageai ('voyage-3', 1024), -- need to parameterize this + -- actual chunking is managed by the docs table + -- this is to prevent running out of context window + chunking => ai.chunking_recursive_character_text_splitter ( + chunk_column => 'content', + chunk_size => 30000, -- 30k characters ~= 7.5k tokens + chunk_overlap => 600, -- 600 characters ~= 150 tokens + separators => ARRAY[ -- tries separators in order + -- markdown headers + E'\n#', + E'\n##', + E'\n###', + E'\n---', + E'\n***', + -- html tags + E'', -- Split on major document sections + E'', -- Split on div boundaries + E'', + E'

', -- Split on paragraphs + E'
', -- Split on line breaks + -- other separators + E'\n\n', -- paragraphs + '. ', + '? ', + '! ', + '; ', -- sentences (note space after punctuation) + E'\n', -- line breaks + ' ' -- words (last resort) + ] + ), + scheduling => ai.scheduling_timescaledb (), + indexing => ai.indexing_diskann (), + formatting => ai.formatting_python_template (E'Title: $title\n\n$chunk'), + processing => ai.processing_default (), + enqueue_existing => TRUE + ); \ No newline at end of file diff --git a/memory-store/migrations/000008_tools.up.sql b/memory-store/migrations/000008_tools.up.sql index bcf59def8..159ef3688 100644 --- a/memory-store/migrations/000008_tools.up.sql +++ b/memory-store/migrations/000008_tools.up.sql @@ -7,13 +7,21 @@ CREATE TABLE IF NOT EXISTS tools ( tool_id UUID NOT NULL, task_id UUID DEFAULT NULL, task_version INT DEFAULT NULL, - type TEXT NOT NULL CONSTRAINT ct_tools_type_length CHECK (length(type) >= 1 AND length(type) <= 255), - name TEXT NOT NULL CONSTRAINT ct_tools_name_length CHECK (length(name) >= 1 AND length(name) <= 255), - description TEXT CONSTRAINT ct_tools_description_length CHECK (description IS NULL OR length(description) <= 1000), + type TEXT NOT NULL CONSTRAINT ct_tools_type_length CHECK ( + length(type) >= 1 + AND length(type) <= 255 + ), + name TEXT NOT NULL CONSTRAINT ct_tools_name_length CHECK ( + length(name) >= 1 + AND length(name) <= 255 + ), + description TEXT CONSTRAINT ct_tools_description_length CHECK ( + description IS NULL + OR length(description) <= 1000 + ), spec JSONB NOT NULL, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT pk_tools PRIMARY KEY (developer_id, agent_id, tool_id, type, name) ); @@ -21,7 +29,9 @@ CREATE TABLE IF NOT EXISTS tools ( CREATE INDEX IF NOT EXISTS idx_tools_id_sorted ON tools (tool_id DESC); -- Create sorted index on task_id if it doesn't exist -CREATE INDEX IF NOT EXISTS idx_tools_task_id_sorted ON tools (task_id DESC) WHERE task_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_tools_task_id_sorted ON tools (task_id DESC) +WHERE + task_id IS NOT NULL; -- Create foreign key constraint and index if they don't exist DO $$ BEGIN @@ -39,11 +49,12 @@ CREATE INDEX IF NOT EXISTS idx_tools_developer_agent ON tools (developer_id, age -- Drop trigger if exists and recreate DROP TRIGGER IF EXISTS trg_tools_updated_at ON tools; -CREATE TRIGGER trg_tools_updated_at - BEFORE UPDATE ON tools - FOR EACH ROW - EXECUTE FUNCTION update_updated_at_column(); + +CREATE TRIGGER trg_tools_updated_at BEFORE +UPDATE ON tools FOR EACH ROW +EXECUTE FUNCTION update_updated_at_column (); -- Add comment to table COMMENT ON TABLE tools IS 'Stores tool configurations and specifications for AI agents'; + COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000009_sessions.down.sql b/memory-store/migrations/000009_sessions.down.sql index d1c0b2911..33d535e53 100644 --- a/memory-store/migrations/000009_sessions.down.sql +++ b/memory-store/migrations/000009_sessions.down.sql @@ -2,16 +2,18 @@ BEGIN; -- Drop triggers first DROP TRIGGER IF EXISTS trg_validate_participant_before_update ON session_lookup; + DROP TRIGGER IF EXISTS trg_validate_participant_before_insert ON session_lookup; -- Drop the validation function -DROP FUNCTION IF EXISTS validate_participant(); +DROP FUNCTION IF EXISTS validate_participant (); -- Drop session_lookup table and its indexes DROP TABLE IF EXISTS session_lookup; -- Drop sessions table and its indexes DROP TRIGGER IF EXISTS trg_sessions_updated_at ON sessions; + DROP TABLE IF EXISTS sessions CASCADE; -- Drop the enum type diff --git a/memory-store/migrations/000009_sessions.up.sql b/memory-store/migrations/000009_sessions.up.sql index 30f135ed7..71e83b7ec 100644 --- a/memory-store/migrations/000009_sessions.up.sql +++ b/memory-store/migrations/000009_sessions.up.sql @@ -7,19 +7,21 @@ CREATE TABLE IF NOT EXISTS sessions ( situation TEXT, system_template TEXT NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - -- TODO: Derived from entries + -- NOTE: Derived from entries -- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, metadata JSONB NOT NULL DEFAULT '{}'::JSONB, - render_templates BOOLEAN NOT NULL DEFAULT true, + render_templates BOOLEAN NOT NULL DEFAULT TRUE, token_budget INTEGER, context_overflow TEXT, forward_tool_calls BOOLEAN, recall_options JSONB NOT NULL DEFAULT '{}'::JSONB, - CONSTRAINT pk_sessions PRIMARY KEY (developer_id, session_id) + CONSTRAINT pk_sessions PRIMARY KEY (developer_id, session_id), + CONSTRAINT uq_sessions_session_id UNIQUE (session_id) ); -- Create indexes if they don't exist CREATE INDEX IF NOT EXISTS idx_sessions_id_sorted ON sessions (session_id DESC); + CREATE INDEX IF NOT EXISTS idx_sessions_metadata ON sessions USING GIN (metadata); -- Create foreign key if it doesn't exist @@ -62,16 +64,23 @@ CREATE TABLE IF NOT EXISTS session_lookup ( session_id UUID NOT NULL, participant_type participant_type NOT NULL, participant_id UUID NOT NULL, - PRIMARY KEY (developer_id, session_id, participant_type, participant_id), - FOREIGN KEY (developer_id, session_id) REFERENCES sessions(developer_id, session_id) + PRIMARY KEY ( + developer_id, + session_id, + participant_type, + participant_id + ), + FOREIGN KEY (developer_id, session_id) REFERENCES sessions (developer_id, session_id) ); -- Create indexes if they don't exist CREATE INDEX IF NOT EXISTS idx_session_lookup_by_session ON session_lookup (developer_id, session_id); + CREATE INDEX IF NOT EXISTS idx_session_lookup_by_participant ON session_lookup (developer_id, participant_id); -- Create or replace the validation function -CREATE OR REPLACE FUNCTION validate_participant() RETURNS trigger AS $$ +CREATE +OR REPLACE FUNCTION validate_participant () RETURNS trigger AS $$ BEGIN IF NEW.participant_type = 'user' THEN PERFORM 1 FROM users WHERE developer_id = NEW.developer_id AND user_id = NEW.participant_id; @@ -101,7 +110,6 @@ BEGIN FOR EACH ROW EXECUTE FUNCTION validate_participant(); END IF; - IF NOT EXISTS ( SELECT 1 FROM pg_trigger WHERE tgname = 'trg_validate_participant_before_update' ) THEN diff --git a/memory-store/migrations/000010_tasks.down.sql b/memory-store/migrations/000010_tasks.down.sql index b7f758779..84608ea71 100644 --- a/memory-store/migrations/000010_tasks.down.sql +++ b/memory-store/migrations/000010_tasks.down.sql @@ -4,11 +4,16 @@ BEGIN; DO $$ BEGIN IF EXISTS ( - SELECT 1 - FROM information_schema.table_constraints - WHERE constraint_name = 'fk_tools_task_id' + SELECT + 1 + FROM + information_schema.table_constraints + WHERE + constraint_name = 'fk_tools_task_id' ) THEN - ALTER TABLE tools DROP CONSTRAINT fk_tools_task_id; + ALTER TABLE tools + DROP CONSTRAINT fk_tools_task_id; + END IF; END $$; diff --git a/memory-store/migrations/000010_tasks.up.sql b/memory-store/migrations/000010_tasks.up.sql index c2bfeb454..2ba6b7910 100644 --- a/memory-store/migrations/000010_tasks.up.sql +++ b/memory-store/migrations/000010_tasks.up.sql @@ -3,13 +3,22 @@ BEGIN; -- Create tasks table if it doesn't exist CREATE TABLE IF NOT EXISTS tasks ( developer_id UUID NOT NULL, - canonical_name CITEXT NOT NULL CONSTRAINT ct_tasks_canonical_name_length CHECK (length(canonical_name) >= 1 AND length(canonical_name) <= 255), + canonical_name CITEXT NOT NULL CONSTRAINT ct_tasks_canonical_name_length CHECK ( + length(canonical_name) >= 1 + AND length(canonical_name) <= 255 + ), agent_id UUID NOT NULL, task_id UUID NOT NULL, - version INTEGER NOT NULL DEFAULT 1, + VERSION INTEGER NOT NULL DEFAULT 1, updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - name TEXT NOT NULL CONSTRAINT ct_tasks_name_length CHECK (length(name) >= 1 AND length(name) <= 255), - description TEXT DEFAULT NULL CONSTRAINT ct_tasks_description_length CHECK (description IS NULL OR length(description) <= 1000), + name TEXT NOT NULL CONSTRAINT ct_tasks_name_length CHECK ( + length(name) >= 1 + AND length(name) <= 255 + ), + description TEXT DEFAULT NULL CONSTRAINT ct_tasks_description_length CHECK ( + description IS NULL + OR length(description) <= 1000 + ), input_schema JSON NOT NULL, inherit_tools BOOLEAN DEFAULT FALSE, workflows JSON[] DEFAULT ARRAY[]::JSON[], @@ -17,10 +26,8 @@ CREATE TABLE IF NOT EXISTS tasks ( metadata JSONB DEFAULT '{}'::JSONB, CONSTRAINT pk_tasks PRIMARY KEY (developer_id, task_id), CONSTRAINT uq_tasks_canonical_name_unique UNIQUE (developer_id, canonical_name), - CONSTRAINT uq_tasks_version_unique UNIQUE (task_id, version), - CONSTRAINT fk_tasks_agent - FOREIGN KEY (developer_id, agent_id) - REFERENCES agents(developer_id, agent_id), + CONSTRAINT uq_tasks_version_unique UNIQUE (task_id, VERSION), + CONSTRAINT fk_tasks_agent FOREIGN KEY (developer_id, agent_id) REFERENCES agents (developer_id, agent_id), CONSTRAINT ct_tasks_canonical_name_valid_identifier CHECK (canonical_name ~ '^[a-zA-Z][a-zA-Z0-9_]*$') ); diff --git a/memory-store/migrations/000011_executions.up.sql b/memory-store/migrations/000011_executions.up.sql index 74ab5bf97..cf0666136 100644 --- a/memory-store/migrations/000011_executions.up.sql +++ b/memory-store/migrations/000011_executions.up.sql @@ -7,21 +7,16 @@ CREATE TABLE IF NOT EXISTS executions ( task_version INTEGER NOT NULL, execution_id UUID NOT NULL, input JSONB NOT NULL, - -- NOTE: These will be generated using continuous aggregates from transitions -- status TEXT DEFAULT 'pending', -- output JSONB DEFAULT NULL, -- error TEXT DEFAULT NULL, -- updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - metadata JSONB NOT NULL DEFAULT '{}'::JSONB, created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT pk_executions PRIMARY KEY (execution_id), - CONSTRAINT fk_executions_developer - FOREIGN KEY (developer_id) REFERENCES developers(developer_id), - CONSTRAINT fk_executions_task - FOREIGN KEY (developer_id, task_id) REFERENCES tasks(developer_id, task_id) + CONSTRAINT fk_executions_developer FOREIGN KEY (developer_id) REFERENCES developers (developer_id), + CONSTRAINT fk_executions_task FOREIGN KEY (developer_id, task_id) REFERENCES tasks (developer_id, task_id) ); -- Create sorted index on execution_id (optimized for UUID v7) @@ -38,4 +33,5 @@ CREATE INDEX IF NOT EXISTS idx_executions_metadata ON executions USING GIN (meta -- Add comment to table (comments are idempotent by default) COMMENT ON TABLE executions IS 'Stores executions associated with AI agents for developers'; + COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000012_transitions.down.sql b/memory-store/migrations/000012_transitions.down.sql index 590ebc901..faac2e308 100644 --- a/memory-store/migrations/000012_transitions.down.sql +++ b/memory-store/migrations/000012_transitions.down.sql @@ -1,15 +1,20 @@ BEGIN; -- Drop foreign key constraint if exists -ALTER TABLE IF EXISTS transitions - DROP CONSTRAINT IF EXISTS fk_transitions_execution; +ALTER TABLE IF EXISTS transitions +DROP CONSTRAINT IF EXISTS fk_transitions_execution; -- Drop indexes if they exist DROP INDEX IF EXISTS idx_transitions_metadata; + DROP INDEX IF EXISTS idx_transitions_execution_id_sorted; + DROP INDEX IF EXISTS idx_transitions_transition_id_sorted; + DROP INDEX IF EXISTS idx_transitions_label; + DROP INDEX IF EXISTS idx_transitions_next; + DROP INDEX IF EXISTS idx_transitions_current; -- Drop the transitions table (this will also remove it from hypertables) @@ -17,10 +22,12 @@ DROP TABLE IF EXISTS transitions; -- Drop custom types if they exist DROP TYPE IF EXISTS transition_cursor; + DROP TYPE IF EXISTS transition_type; -- Drop the trigger and function for transition validation DROP TRIGGER IF EXISTS validate_transition ON transitions; -DROP FUNCTION IF EXISTS check_valid_transition(); + +DROP FUNCTION IF EXISTS check_valid_transition (); COMMIT; diff --git a/memory-store/migrations/000012_transitions.up.sql b/memory-store/migrations/000012_transitions.up.sql index 515af713c..6fd7dbcd1 100644 --- a/memory-store/migrations/000012_transitions.up.sql +++ b/memory-store/migrations/000012_transitions.up.sql @@ -46,8 +46,19 @@ CREATE TABLE IF NOT EXISTS transitions ( ); -- Convert to hypertable if not already -SELECT create_hypertable('transitions', by_range('created_at', INTERVAL '1 day'), if_not_exists => TRUE); -SELECT add_dimension('transitions', by_hash('execution_id', 2), if_not_exists => TRUE); +SELECT + create_hypertable ( + 'transitions', + by_range ('created_at', INTERVAL '1 day'), + if_not_exists => TRUE + ); + +SELECT + add_dimension ( + 'transitions', + by_hash ('execution_id', 2), + if_not_exists => TRUE + ); -- Create indexes if they don't exist DO $$ @@ -94,7 +105,8 @@ END $$; COMMENT ON TABLE transitions IS 'Stores transitions associated with AI agents for developers'; -- Create a trigger function that checks for valid transitions -CREATE OR REPLACE FUNCTION check_valid_transition() RETURNS trigger AS $$ +CREATE +OR REPLACE FUNCTION check_valid_transition () RETURNS trigger AS $$ DECLARE previous_type transition_type; valid_next_types transition_type[]; @@ -146,9 +158,7 @@ END; $$ LANGUAGE plpgsql; -- Create a trigger on the transitions table -CREATE TRIGGER validate_transition -BEFORE INSERT ON transitions -FOR EACH ROW -EXECUTE FUNCTION check_valid_transition(); +CREATE TRIGGER validate_transition BEFORE INSERT ON transitions FOR EACH ROW +EXECUTE FUNCTION check_valid_transition (); COMMIT; \ No newline at end of file diff --git a/memory-store/migrations/000013_executions_continuous_view.down.sql b/memory-store/migrations/000013_executions_continuous_view.down.sql index d833ca4d4..fcab7b023 100644 --- a/memory-store/migrations/000013_executions_continuous_view.down.sql +++ b/memory-store/migrations/000013_executions_continuous_view.down.sql @@ -1,13 +1,15 @@ BEGIN; -- Drop the continuous aggregate policy -SELECT remove_continuous_aggregate_policy('latest_transitions'); +SELECT + remove_continuous_aggregate_policy ('latest_transitions'); -- Drop the views DROP VIEW IF EXISTS latest_executions; + DROP MATERIALIZED VIEW IF EXISTS latest_transitions; -- Drop the helper function -DROP FUNCTION IF EXISTS to_text(transition_type); +DROP FUNCTION IF EXISTS to_text (transition_type); COMMIT; diff --git a/memory-store/migrations/000013_executions_continuous_view.up.sql b/memory-store/migrations/000013_executions_continuous_view.up.sql index b33530824..43285efbc 100644 --- a/memory-store/migrations/000013_executions_continuous_view.up.sql +++ b/memory-store/migrations/000013_executions_continuous_view.up.sql @@ -1,39 +1,39 @@ BEGIN; -- create a function to convert transition_type to text (needed coz ::text is stable not immutable) -create or replace function to_text(transition_type) -RETURNS text AS -$$ +CREATE +OR REPLACE function to_text (transition_type) RETURNS text AS $$ select $1 $$ STRICT IMMUTABLE LANGUAGE sql; -- create a continuous view that aggregates the transitions table -create materialized view if not exists latest_transitions -with +CREATE MATERIALIZED VIEW IF NOT EXISTS latest_transitions +WITH ( timescaledb.continuous, - timescaledb.materialized_only = false - ) as -select - time_bucket ('1 day', created_at) as bucket, + timescaledb.materialized_only = FALSE + ) AS +SELECT + time_bucket ('1 day', created_at) AS bucket, execution_id, - count(*) as total_transitions, - state_agg (created_at, to_text (type)) as state, - max(created_at) as created_at, - last (type, created_at) as type, - last (step_definition, created_at) as step_definition, - last (step_label, created_at) as step_label, - last (current_step, created_at) as current_step, - last (next_step, created_at) as next_step, - last (output, created_at) as output, - last (task_token, created_at) as task_token, - last (metadata, created_at) as metadata -from + count(*) AS total_transitions, + state_agg (created_at, to_text (type)) AS state, + max(created_at) AS created_at, + last (type, created_at) AS type, + last (step_definition, created_at) AS step_definition, + last (step_label, created_at) AS step_label, + last (current_step, created_at) AS current_step, + last (next_step, created_at) AS next_step, + last (output, created_at) AS output, + last (task_token, created_at) AS task_token, + last (metadata, created_at) AS metadata +FROM transitions -group by +GROUP BY bucket, execution_id -with no data; +WITH + no data; SELECT add_continuous_aggregate_policy ( @@ -44,7 +44,7 @@ SELECT ); -- Create a view that combines executions with their latest transitions -create or replace view latest_executions as +CREATE OR REPLACE VIEW latest_executions AS SELECT e.developer_id, e.task_id, @@ -53,7 +53,7 @@ SELECT e.input, e.metadata, e.created_at, - lt.created_at as updated_at, + lt.created_at AS updated_at, -- Map transition types to status using CASE statement CASE lt.type::text WHEN 'init' THEN 'starting' @@ -66,20 +66,20 @@ SELECT WHEN 'error' THEN 'failed' WHEN 'cancelled' THEN 'cancelled' ELSE 'queued' - END as status, + END AS status, lt.output, -- Extract error from output if type is 'error' CASE WHEN lt.type::text = 'error' THEN lt.output ->> 'error' ELSE NULL - END as error, + END AS error, lt.total_transitions, lt.current_step, lt.next_step, lt.step_definition, lt.step_label, lt.task_token, - lt.metadata as transition_metadata + lt.metadata AS transition_metadata FROM executions e, latest_transitions lt diff --git a/memory-store/migrations/000014_temporal_lookup.up.sql b/memory-store/migrations/000014_temporal_lookup.up.sql index 1650ab3ac..724ee1340 100644 --- a/memory-store/migrations/000014_temporal_lookup.up.sql +++ b/memory-store/migrations/000014_temporal_lookup.up.sql @@ -1,17 +1,16 @@ BEGIN; -- Create temporal_executions_lookup table -CREATE TABLE - IF NOT EXISTS temporal_executions_lookup ( - execution_id UUID NOT NULL, - id TEXT NOT NULL, - run_id TEXT, - first_execution_run_id TEXT, - result_run_id TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, - CONSTRAINT pk_temporal_executions_lookup PRIMARY KEY (execution_id, id), - CONSTRAINT fk_temporal_executions_lookup_execution FOREIGN KEY (execution_id) REFERENCES executions (execution_id) - ); +CREATE TABLE IF NOT EXISTS temporal_executions_lookup ( + execution_id UUID NOT NULL, + id TEXT NOT NULL, + run_id TEXT, + first_execution_run_id TEXT, + result_run_id TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT pk_temporal_executions_lookup PRIMARY KEY (execution_id, id), + CONSTRAINT fk_temporal_executions_lookup_execution FOREIGN KEY (execution_id) REFERENCES executions (execution_id) +); -- Create sorted index on execution_id (optimized for UUID v7) CREATE INDEX IF NOT EXISTS idx_temporal_executions_lookup_execution_id_sorted ON temporal_executions_lookup (execution_id DESC);