Paig evaluation paig UI support

.github/workflows/paig-server-ci.yml

@@ -50,14 +50,22 @@ jobs:
         python3 -m build -w
         pip install dist/*.whl
         cd ..
-
+        
     - name: Build and Install paig-authorizer-core wheel
       run: |
         . venv/bin/activate
         cd paig-authorizer-core
         python3 -m build -w
         pip install dist/*.whl
         cd ..
+
+    - name: Build and Install paig-evaluation wheel
+      run: |
+        . venv/bin/activate
+        cd paig-evaluation
+        python3 -m build -w
+        pip install dist/*.whl
+        cd ..
 
     - name: Install PAIG-Server dependencies
       run: |

paig-server/backend/paig/__main__.py

@@ -66,7 +66,7 @@ def main(
         action: str,
     ) -> None:
 
-    if action.lower() == "stop":
+    if action and action.lower() == "stop":
         stop_server()
         return
     elif action.lower() == "status":

paig-server/backend/paig/alembic_db/env.py

@@ -31,6 +31,7 @@
 from api.user.database.db_models import user_model, groups_model
 from api.audit.RDS_service.db_models import access_audit_model
 from api.encryption.database.db_models import encryption_master_key_model, encryption_key_model
+from api.evaluation.database.db_models import eval_model, eval_targets, eval_config
 from core.db_session.session import Base
 target_metadata = Base.metadata
 

paig-server/backend/paig/alembic_db/versions/701ddf55a1b4_added_evaluation_tables.py

@@ -0,0 +1,139 @@
+"""Added evaluation tables
+
+Revision ID: 701ddf55a1b4
+Revises: a95b604c47fb
+Create Date: 2025-02-20 12:12:01.526199
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import core.db_models.utils
+
+
+# revision identifiers, used by Alembic.
+revision: str = '701ddf55a1b4'
+down_revision: Union[str, None] = 'a95b604c47fb'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('eval_config',
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('purpose', sa.Text(), nullable=True),
+    sa.Column('application_ids', sa.String(length=255), nullable=True),
+    sa.Column('application_names', sa.Text(), nullable=True),
+    sa.Column('categories', sa.Text(), nullable=True),
+    sa.Column('custom_prompts', sa.Text(), nullable=True),
+    sa.Column('version', sa.Integer(), nullable=False),
+    sa.Column('owner', sa.String(length=255), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_config_id'), 'eval_config', ['id'], unique=False)
+    op.create_table('eval_run',
+    sa.Column('name', sa.String(length=255), nullable=False),
+    sa.Column('owner', sa.String(length=255), nullable=False),
+    sa.Column('purpose', sa.Text(), nullable=True),
+    sa.Column('eval_id', sa.String(length=255), nullable=False),
+    sa.Column('config_id', sa.String(length=255), nullable=False),
+    sa.Column('config_name', sa.String(length=255), nullable=True),
+    sa.Column('application_names', sa.Text(), nullable=True),
+    sa.Column('cumulative_result', sa.Text(), nullable=True),
+    sa.Column('passed', sa.String(length=255), nullable=True),
+    sa.Column('failed', sa.String(length=255), nullable=True),
+    sa.Column('base_run_id', sa.String(length=255), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_run_id'), 'eval_run', ['id'], unique=False)
+    op.create_table('eval_config_history',
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('purpose', sa.Text(), nullable=True),
+    sa.Column('application_ids', sa.String(length=255), nullable=True),
+    sa.Column('application_names', sa.Text(), nullable=True),
+    sa.Column('generated_config', sa.Text(), nullable=True),
+    sa.Column('categories', sa.Text(), nullable=True),
+    sa.Column('custom_prompts', sa.Text(), nullable=True),
+    sa.Column('version', sa.Integer(), nullable=False),
+    sa.Column('owner', sa.String(length=255), nullable=True),
+    sa.Column('eval_config_id', sa.Integer(), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.ForeignKeyConstraint(['eval_config_id'], ['eval_config.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_config_history_id'), 'eval_config_history', ['id'], unique=False)
+    op.create_table('eval_result_prompt',
+    sa.Column('eval_run_id', sa.String(length=255), nullable=False),
+    sa.Column('eval_id', sa.String(length=255), nullable=False),
+    sa.Column('prompt_uuid', sa.String(length=255), nullable=False),
+    sa.Column('prompt', sa.Text(), nullable=False),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.ForeignKeyConstraint(['eval_run_id'], ['eval_run.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_result_prompt_id'), 'eval_result_prompt', ['id'], unique=False)
+    op.create_table('eval_target',
+    sa.Column('application_id', sa.Integer(), nullable=True),
+    sa.Column('config', sa.JSON(), nullable=False),
+    sa.Column('name', sa.String(length=255), nullable=True),
+    sa.Column('url', sa.Text(), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.ForeignKeyConstraint(['application_id'], ['ai_application.id'], ),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_target_id'), 'eval_target', ['id'], unique=False)
+    op.create_table('eval_result_response',
+    sa.Column('eval_run_id', sa.String(length=255), nullable=False),
+    sa.Column('eval_result_prompt_uuid', sa.String(length=255), nullable=False),
+    sa.Column('eval_id', sa.String(length=255), nullable=False),
+    sa.Column('response', sa.Text(), nullable=True),
+    sa.Column('application_name', sa.String(length=255), nullable=False),
+    sa.Column('failure_reason', sa.Text(), nullable=True),
+    sa.Column('category_score', sa.Text(), nullable=True),
+    sa.Column('category', sa.String(length=255), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('create_time', sa.DateTime(), nullable=True),
+    sa.Column('update_time', sa.DateTime(), nullable=True),
+    sa.Column('status', sa.String(length=255), nullable=True),
+    sa.ForeignKeyConstraint(['eval_result_prompt_uuid'], ['eval_result_prompt.prompt_uuid'], ),
+    sa.ForeignKeyConstraint(['eval_run_id'], ['eval_run.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_index(op.f('ix_eval_result_response_id'), 'eval_result_response', ['id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_eval_result_response_id'), table_name='eval_result_response')
+    op.drop_table('eval_result_response')
+    op.drop_index(op.f('ix_eval_target_id'), table_name='eval_target')
+    op.drop_table('eval_target')
+    op.drop_index(op.f('ix_eval_result_prompt_id'), table_name='eval_result_prompt')
+    op.drop_table('eval_result_prompt')
+    op.drop_index(op.f('ix_eval_config_history_id'), table_name='eval_config_history')
+    op.drop_table('eval_config_history')
+    op.drop_index(op.f('ix_eval_run_id'), table_name='eval_run')
+    op.drop_table('eval_run')
+    op.drop_index(op.f('ix_eval_config_id'), table_name='eval_config')
+    op.drop_table('eval_config')
+    # ### end Alembic commands ###

paig-server/backend/paig/api/evaluation/api_schemas/eval_config_schema.py

@@ -0,0 +1,95 @@
+from fastapi import Query
+from pydantic import BaseModel, Field
+from typing import List, Optional
+
+from core.api_schemas.base_view import BaseView
+from core.factory.database_initiator import BaseAPIFilter
+
+class ConfigCommonModel(BaseModel):
+    purpose: str = Field(..., description="The purpose of the config")
+    name: str = Field(..., max_length=1024)
+    categories: List[str] = Field(default_factory=[], description="The categories of evaluation")
+    custom_prompts: List[str] = Field(default_factory=[], description="Custom prompts for evaluation")
+
+
+class ConfigCreateRequest(ConfigCommonModel):
+    application_ids: str
+
+class ConfigUpdateRequest(ConfigCommonModel):
+    application_ids: str
+
+
+class EvalConfigFilter(BaseAPIFilter):
+    """
+    Filter class for AI application queries.
+
+    Attributes:
+        id (int, optional): Filter by ID.
+        purpose (str, optional): Filter by purpose.
+        name (str, optional): Filter by name.
+    """
+
+    id: Optional[int] = Field(default=None, description="Filter by id")
+    purpose: Optional[str] = Field(default=None, description="Filter by purpose")
+    name: Optional[str] = Field(default=None, description="Filter by name")
+
+class EvalConfigView(BaseView):
+    purpose: str = Field(..., description="The purpose of the config")
+    name: str = Field(..., max_length=1024, description="The name of the config")
+    categories: str = Field(..., description="The categories of evaluation")
+    custom_prompts: str = Field(..., description="Custom prompts for evaluation")
+    status: str = Field(..., max_length=1024, description="The status of the config")
+    version: int = Field(..., gt=0, description="The version of the config")
+    application_names: str = Field(..., description="The application names")
+    eval_run_count: int = Field(..., ge=0, description="The number of evaluation runs")
+    owner: Optional[str] = Field(None, description="The User Name", alias="owner")
+    model_config = BaseView.model_config
+
+
+class QueryParamsBase(BaseAPIFilter):
+    purpose: Optional[str] = Field(None, description="purpose", alias="purpose")
+    name: Optional[str] = Field(None, description="The Config name", alias="name")
+    owner: Optional[str] = Field(None, description="The User ID", alias="owner")
+    application_names: Optional[str] = Field(None, description="The Application name", alias="application_names")
+
+
+
+class IncludeQueryParams(QueryParamsBase):
+    pass
+
+def include_query_params(
+        include_query_application_names: Optional[str] = Query(None, alias="includeQuery.application_names"),
+        include_query_purpose: Optional[str] = Query(None, alias="includeQuery.purpose"),
+        include_query_name: Optional[str] = Query(None, alias="includeQuery.name"),
+        include_query_owner: Optional[str] = Query(None, alias="includeQuery.owner"),
+) -> IncludeQueryParams:
+    return IncludeQueryParams(
+        application_names=include_query_application_names,
+        purpose=include_query_purpose,
+        name=include_query_name,
+        owner=include_query_owner
+    )
+
+
+def exclude_query_params(
+        exclude_query_application_names: Optional[str] = Query(None, alias="excludeQuery.application_names"),
+        exclude_query_purpose: Optional[str] = Query(None, alias="excludeQuery.purpose"),
+        exclude_query_name: Optional[str] = Query(None, alias="excludeQuery.name"),
+        exclude_query_owner: Optional[str] = Query(None, alias="excludeQuery.owner"),
+) -> QueryParamsBase:
+    return QueryParamsBase(
+        application_names=exclude_query_application_names,
+        purpose=exclude_query_purpose,
+        name=exclude_query_name,
+        owner=exclude_query_owner
+    )
+
+
+def extract_include_query_params(params):
+    params_dict = params.model_dump(exclude=BaseAPIFilter.model_fields.keys(), by_alias=False, exclude_none=True)
+
+    # Extract only the required fields
+    filtered_params = {params.model_fields[field].alias: value for field, value in params_dict.items() if
+                       value is not None}
+
+    return filtered_params