From 8033549d92d7355e7d12912d69f8e9fff70afee7 Mon Sep 17 00:00:00 2001 From: Diwank Singh Tomer Date: Fri, 4 Oct 2024 21:36:59 -0400 Subject: [PATCH] fix(typespec,agents-api): Update metadata_filter to have object type (#586) Signed-off-by: Diwank Singh Tomer ---- > [!IMPORTANT] > Refactors metadata filtering by introducing `FilterModel` and `create_filter_extractor`, updating API endpoints and typespec for structured metadata filtering. > > - **Behavior**: > - Introduces `FilterModel` and `create_filter_extractor` in `query_filter.py` for structured metadata filtering. > - Updates `list_agents`, `list_user_docs`, `list_agent_docs`, and `list_sessions` to use `FilterModel` for `metadata_filter`. > - Removes JSON string parsing for `metadata_filter` in `list_agents.py`, `list_docs.py`, and `list_sessions.py`. > - **Types**: > - Adds `concreteType` alias in `scalars.tsp`. > - Updates `MetadataFilter` alias in `types.tsp` to use `concreteType`. > - Changes `metadata_filter` in `PaginationOptions` model in `types.tsp` to use `MetadataFilter`. > > This description was created by [Ellipsis](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral) for 63eda8b9da45b5200862c799e7d9da97299f0029. It will automatically update as commits are pushed. --------- Signed-off-by: Diwank Singh Tomer Co-authored-by: creatorrr --- .../agents_api/dependencies/query_filter.py | 54 +++++++++++++++++++ .../agents_api/models/docs/list_docs.py | 4 +- .../agents_api/routers/agents/list_agents.py | 22 ++++---- .../agents_api/routers/docs/list_docs.py | 33 ++++-------- .../routers/sessions/list_sessions.py | 19 +++---- typespec/common/interfaces.tsp | 2 +- typespec/common/scalars.tsp | 2 + typespec/common/types.tsp | 5 +- .../@typespec/openapi3/openapi-0.4.0.yaml | 35 ++++++------ .../@typespec/openapi3/openapi-1.0.0.yaml | 35 ++++++------ 10 files changed, 127 insertions(+), 84 deletions(-) create mode 100644 agents-api/agents_api/dependencies/query_filter.py diff --git a/agents-api/agents_api/dependencies/query_filter.py b/agents-api/agents_api/dependencies/query_filter.py new file mode 100644 index 000000000..c100f1489 --- /dev/null +++ b/agents-api/agents_api/dependencies/query_filter.py @@ -0,0 +1,54 @@ +from typing import Any, Callable + +from fastapi import Request + + +def convert_value(value: str) -> Any: + """ + Attempts to convert a string value to an int or float. Returns the original string if conversion fails. + """ + for convert in (int, float): + try: + return convert(value) + except ValueError: + continue + return value + + +def create_filter_extractor( + prefix: str = "filter", +) -> Callable[[Request], dict[str, Any]]: + """ + Creates a dependency function to extract filter parameters with a given prefix. + + Args: + prefix (str): The prefix to identify filter parameters. + + Returns: + Callable[[Request], dict[str, Any]]: The dependency function. + """ + + # Add a dot to the prefix to allow for nested filters + prefix += "." + + def extract_filters(request: Request) -> dict[str, Any]: + """ + Extracts query parameters that start with the specified prefix and returns them as a dictionary. + + Args: + request (Request): The incoming HTTP request. + + Returns: + dict[str, Any]: A dictionary containing the filter parameters. + """ + + filters: dict[str, Any] = {} + + for key, value in request.query_params.items(): + if key.startswith(prefix): + filter_key = key[len(prefix) :] + filters[filter_key] = convert_value(value) + + return filters + + return extract_filters diff --git a/agents-api/agents_api/models/docs/list_docs.py b/agents-api/agents_api/models/docs/list_docs.py index 3c095c2db..4dad7ec06 100644 --- a/agents-api/agents_api/models/docs/list_docs.py +++ b/agents-api/agents_api/models/docs/list_docs.py @@ -90,7 +90,8 @@ def list_docs( created_at, metadata, }}, - snippets[id, snippet_data] + snippets[id, snippet_data], + {metadata_filter_str} :limit $limit :offset $offset @@ -112,6 +113,5 @@ def list_docs( "owner_type": owner_type, "limit": limit, "offset": offset, - "metadata_filter": metadata_filter_str, }, ) diff --git a/agents-api/agents_api/routers/agents/list_agents.py b/agents-api/agents_api/routers/agents/list_agents.py index 21cd736b5..ef9bb09db 100644 --- a/agents-api/agents_api/routers/agents/list_agents.py +++ b/agents-api/agents_api/routers/agents/list_agents.py @@ -1,12 +1,11 @@ -import json -from json import JSONDecodeError from typing import Annotated, Literal from uuid import UUID -from fastapi import Depends, HTTPException, status +from fastapi import Depends from ...autogen.openapi_model import Agent, ListResponse from ...dependencies.developer_id import get_developer_id +from ...dependencies.query_filter import create_filter_extractor from ...models.agent.list_agents import list_agents as list_agents_query from .router import router @@ -14,27 +13,24 @@ @router.get("/agents", tags=["agents"]) async def list_agents( x_developer_id: Annotated[UUID, Depends(get_developer_id)], + # Expects the dot notation of object in query params + # Example: + # > ?metadata_filter.name=John&metadata_filter.age=30 + metadata_filter: Annotated[ + dict, Depends(create_filter_extractor("metadata_filter")) + ], limit: int = 100, offset: int = 0, sort_by: Literal["created_at", "updated_at"] = "created_at", direction: Literal["asc", "desc"] = "desc", - metadata_filter: str = "{}", ) -> ListResponse[Agent]: - try: - metadata_filter = json.loads(metadata_filter) - except JSONDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="metadata_filter is not a valid JSON", - ) - agents = list_agents_query( developer_id=x_developer_id, limit=limit, offset=offset, sort_by=sort_by, direction=direction, - metadata_filter=metadata_filter, + metadata_filter=metadata_filter or {}, ) return ListResponse[Agent](items=agents) diff --git a/agents-api/agents_api/routers/docs/list_docs.py b/agents-api/agents_api/routers/docs/list_docs.py index 2ba99a932..a4701646d 100644 --- a/agents-api/agents_api/routers/docs/list_docs.py +++ b/agents-api/agents_api/routers/docs/list_docs.py @@ -1,12 +1,11 @@ -import json -from json import JSONDecodeError from typing import Annotated, Literal from uuid import UUID -from fastapi import Depends, HTTPException, status +from fastapi import Depends from ...autogen.openapi_model import Doc, ListResponse from ...dependencies.developer_id import get_developer_id +from ...dependencies.query_filter import create_filter_extractor from ...models.docs.list_docs import list_docs as list_docs_query from .router import router @@ -14,21 +13,15 @@ @router.get("/users/{user_id}/docs", tags=["docs"]) async def list_user_docs( x_developer_id: Annotated[UUID, Depends(get_developer_id)], + metadata_filter: Annotated[ + dict, Depends(create_filter_extractor("metadata_filter")) + ], user_id: UUID, limit: int = 100, offset: int = 0, sort_by: Literal["created_at", "updated_at"] = "created_at", direction: Literal["asc", "desc"] = "desc", - metadata_filter: str = "{}", ) -> ListResponse[Doc]: - try: - metadata_filter = json.loads(metadata_filter) - except JSONDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="metadata_filter is not a valid JSON", - ) - docs = list_docs_query( developer_id=x_developer_id, owner_type="user", @@ -37,7 +30,7 @@ async def list_user_docs( offset=offset, sort_by=sort_by, direction=direction, - metadata_filter=metadata_filter, + metadata_filter=metadata_filter or {}, ) return ListResponse[Doc](items=docs) @@ -46,21 +39,15 @@ async def list_user_docs( @router.get("/agents/{agent_id}/docs", tags=["docs"]) async def list_agent_docs( x_developer_id: Annotated[UUID, Depends(get_developer_id)], + metadata_filter: Annotated[ + dict, Depends(create_filter_extractor("metadata_filter")) + ], agent_id: UUID, limit: int = 100, offset: int = 0, sort_by: Literal["created_at", "updated_at"] = "created_at", direction: Literal["asc", "desc"] = "desc", - metadata_filter: str = "{}", ) -> ListResponse[Doc]: - try: - metadata_filter = json.loads(metadata_filter) - except JSONDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="metadata_filter is not a valid JSON", - ) - docs = list_docs_query( developer_id=x_developer_id, owner_type="agent", @@ -69,7 +56,7 @@ async def list_agent_docs( offset=offset, sort_by=sort_by, direction=direction, - metadata_filter=metadata_filter, + metadata_filter=metadata_filter or {}, ) return ListResponse[Doc](items=docs) diff --git a/agents-api/agents_api/routers/sessions/list_sessions.py b/agents-api/agents_api/routers/sessions/list_sessions.py index 21d7b643b..6a4555e6e 100644 --- a/agents-api/agents_api/routers/sessions/list_sessions.py +++ b/agents-api/agents_api/routers/sessions/list_sessions.py @@ -1,12 +1,11 @@ -import json -from json import JSONDecodeError from typing import Annotated, Literal from uuid import UUID -from fastapi import Depends, HTTPException, status +from fastapi import Depends from ...autogen.openapi_model import ListResponse, Session from ...dependencies.developer_id import get_developer_id +from ...dependencies.query_filter import create_filter_extractor from ...models.session.list_sessions import list_sessions as list_sessions_query from .router import router @@ -14,27 +13,21 @@ @router.get("/sessions", tags=["sessions"]) async def list_sessions( x_developer_id: Annotated[UUID, Depends(get_developer_id)], + metadata_filter: Annotated[ + dict, Depends(create_filter_extractor("metadata_filter")) + ] = {}, limit: int = 100, offset: int = 0, sort_by: Literal["created_at", "updated_at"] = "created_at", direction: Literal["asc", "desc"] = "desc", - metadata_filter: str = "{}", ) -> ListResponse[Session]: - try: - metadata_filter = json.loads(metadata_filter) - except JSONDecodeError: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="metadata_filter is not a valid JSON", - ) - sessions = list_sessions_query( developer_id=x_developer_id, limit=limit, offset=offset, sort_by=sort_by, direction=direction, - metadata_filter=metadata_filter, + metadata_filter=metadata_filter or {}, ) return ListResponse[Session](items=sessions) diff --git a/typespec/common/interfaces.tsp b/typespec/common/interfaces.tsp index d44d8aab0..d9e4a9e2e 100644 --- a/typespec/common/interfaces.tsp +++ b/typespec/common/interfaces.tsp @@ -136,7 +136,7 @@ interface ChildLimitOffsetPagination< ...PaginationOptions, ): { - results: T[]; + items: T[]; }; } diff --git a/typespec/common/scalars.tsp b/typespec/common/scalars.tsp index 8dc07cbbc..dc206fa02 100644 --- a/typespec/common/scalars.tsp +++ b/typespec/common/scalars.tsp @@ -7,6 +7,8 @@ namespace Common; @format("uuid") scalar uuid extends string; +alias concreteType = numeric | string | boolean | null; + /** * For Unicode character safety * See: https://unicode.org/reports/tr31/ diff --git a/typespec/common/types.tsp b/typespec/common/types.tsp index 7d954a80e..dd331e98c 100644 --- a/typespec/common/types.tsp +++ b/typespec/common/types.tsp @@ -11,6 +11,7 @@ namespace Common; // alias Metadata = Record; +alias MetadataFilter = Record; model ResourceCreatedResponse { @doc("ID of created resource") @@ -48,6 +49,6 @@ model PaginationOptions { /** Sort direction */ @query direction: sortDirection = "asc", - /** JSON string of object that should be used to filter objects by metadata */ - @query metadata_filter: string = "{}", + /** Object to filter results by metadata */ + @query metadata_filter: MetadataFilter, } \ No newline at end of file diff --git a/typespec/tsp-output/@typespec/openapi3/openapi-0.4.0.yaml b/typespec/tsp-output/@typespec/openapi3/openapi-0.4.0.yaml index e578d26d9..7a7afc3e9 100644 --- a/typespec/tsp-output/@typespec/openapi3/openapi-0.4.0.yaml +++ b/typespec/tsp-output/@typespec/openapi3/openapi-0.4.0.yaml @@ -181,12 +181,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Docs.Doc' required: - - results + - items post: operationId: AgentDocsRoute_create description: Create a Doc for this Agent @@ -290,12 +290,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Tasks.Task' required: - - results + - items post: operationId: TasksRoute_create description: Create a new task @@ -434,12 +434,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Tools.Tool' required: - - results + - items post: operationId: AgentToolsRoute_create description: Create a new tool for this agent @@ -713,7 +713,7 @@ paths: schema: type: object properties: - results: + items: type: array items: type: object @@ -725,7 +725,7 @@ paths: required: - transitions required: - - results + - items /executions/{id}/transitions.stream: get: operationId: ExecutionTransitionsStreamRoute_stream @@ -1026,12 +1026,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Executions.Execution' required: - - results + - items /users: get: operationId: UsersRoute_list @@ -1196,12 +1196,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Docs.Doc' required: - - results + - items post: operationId: UserDocsRoute_create description: Create a Doc for this User @@ -1317,10 +1317,15 @@ components: name: metadata_filter in: query required: true - description: JSON string of object that should be used to filter objects by metadata + description: Object to filter results by metadata schema: - type: string - default: '{}' + type: object + additionalProperties: + anyOf: + - type: number + - type: string + - type: boolean + nullable: true explode: false Common.PaginationOptions.offset: name: offset diff --git a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml index e5a5f99e6..d82540194 100644 --- a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml +++ b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml @@ -181,12 +181,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Docs.Doc' required: - - results + - items post: operationId: AgentDocsRoute_create description: Create a Doc for this Agent @@ -290,12 +290,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Tasks.Task' required: - - results + - items post: operationId: TasksRoute_create description: Create a new task @@ -434,12 +434,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Tools.Tool' required: - - results + - items post: operationId: AgentToolsRoute_create description: Create a new tool for this agent @@ -713,7 +713,7 @@ paths: schema: type: object properties: - results: + items: type: array items: type: object @@ -725,7 +725,7 @@ paths: required: - transitions required: - - results + - items /executions/{id}/transitions.stream: get: operationId: ExecutionTransitionsStreamRoute_stream @@ -1026,12 +1026,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Executions.Execution' required: - - results + - items /users: get: operationId: UsersRoute_list @@ -1196,12 +1196,12 @@ paths: schema: type: object properties: - results: + items: type: array items: $ref: '#/components/schemas/Docs.Doc' required: - - results + - items post: operationId: UserDocsRoute_create description: Create a Doc for this User @@ -1317,10 +1317,15 @@ components: name: metadata_filter in: query required: true - description: JSON string of object that should be used to filter objects by metadata + description: Object to filter results by metadata schema: - type: string - default: '{}' + type: object + additionalProperties: + anyOf: + - type: number + - type: string + - type: boolean + nullable: true explode: false Common.PaginationOptions.offset: name: offset