ci(prompts): initial set of integration tests for server and client (#…

…6266)

.github/workflows/python-CI.yml

@@ -308,7 +308,7 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
-            tests/
+            packages/phoenix-client/
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -343,7 +343,11 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
-            tests/
+            tests/unit/
+            tests/conftest.py
+            tests/mypy.ini
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -383,6 +387,8 @@ jobs:
             src/phoenix/
             tests/unit/
             tests/conftest.py
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -424,7 +430,10 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
+            packages/phoenix-client/
             tests/integration/
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -485,7 +494,10 @@ jobs:
           sparse-checkout: |
             requirements/
             src/phoenix/
+            packages/phoenix-client/
             tests/integration/
+            tests/__generated__/
+            tests/__init__.py
       - name: Set up Python ${{ matrix.py }}
         uses: actions/setup-python@v5
         with:
@@ -512,7 +524,7 @@ jobs:
       fail-fast: false
       matrix:
         py: [ 3.9 ]
-        pkg: [ openai, anthropic ]
+        pkg: [ openai, anthropic, mistralai, groq ]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.gitignore

@@ -27,6 +27,7 @@ coverage.xml
 pyrightconfig.json
 !/packages/phoenix-client/src/phoenix/client/helpers/sdk/pyrightconfig.json
 !/packages/phoenix-client/tests/canary/sdk/pyrightconfig.json
+!/tests/integration/pyrightconfig.json
 
 # Demo data
 tutorials/internal/demo_llama_index/*.json

packages/phoenix-client/scripts/codegen/transform.py

@@ -7,40 +7,32 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> ast.ImportFrom:
         if node.module == "dataclasses":
             # Replace `from dataclasses import dataclass` with
             # `from typing import TypedDict`
-            return ast.copy_location(
-                ast.ImportFrom(
-                    module="typing",
-                    names=[ast.alias(name="TypedDict", asname=None)],
-                    level=0,
-                ),
-                node,
+            return ast.ImportFrom(
+                module="typing",
+                names=[ast.alias(name="TypedDict", asname=None)],
+                level=0,
             )
         return node
 
     def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef:
         # Redefine all classes as TypedDict
-        return ast.copy_location(
-            ast.ClassDef(
-                name=node.name,
-                bases=[ast.Name(id="TypedDict", ctx=ast.Load())],
-                keywords=node.keywords,
-                body=[self.visit(child) for child in node.body],
-                decorator_list=[],
-            ),
-            node,
+        return ast.ClassDef(
+            name=node.name,
+            bases=[ast.Name(id="TypedDict", ctx=ast.Load())],
+            keywords=node.keywords,
+            body=[self.visit(child) for child in node.body],
+            decorator_list=[],
         )
 
     def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AnnAssign:
-        if isinstance(node.target, ast.Name) and node.target.id == "schema_":
+        if isinstance(node.target, ast.Name) and node.target.id in ("schema_", "json_"):
             # Convert `schema_: xyz` to `schema: xyz`
-            node = ast.copy_location(
-                ast.AnnAssign(
-                    target=ast.Name(id="schema", ctx=ast.Store()),
-                    annotation=node.annotation,
-                    value=node.value,
-                    simple=node.simple,
-                ),
-                node,
+            target = ast.Name(id=node.target.id.rstrip("_"), ctx=ast.Store())
+            node = ast.AnnAssign(
+                target=target,
+                annotation=node.annotation,
+                value=node.value,
+                simple=node.simple,
             )
         if isinstance(node.value, ast.Constant):
             if (
@@ -50,46 +42,37 @@ def visit_AnnAssign(self, node: ast.AnnAssign) -> ast.AnnAssign:
                 and node.annotation.id == "str"
             ):
                 # Convert `type: str = "xyz"` to `type: Literal["xyz"]`
-                return ast.copy_location(
-                    ast.AnnAssign(
-                        target=node.target,
-                        annotation=ast.Subscript(
-                            value=ast.Name(id="Literal", ctx=ast.Load()),
-                            slice=node.value,
-                            ctx=ast.Load(),
-                        ),
-                        simple=node.simple,
+                return ast.AnnAssign(
+                    target=node.target,
+                    annotation=ast.Subscript(
+                        value=ast.Name(id="Literal", ctx=ast.Load()),
+                        slice=node.value,
+                        ctx=ast.Load(),
                     ),
-                    node,
+                    simple=node.simple,
                 )
             if (
                 isinstance(node.annotation, ast.Subscript)
                 and isinstance(node.annotation.value, ast.Name)
                 and node.annotation.value.id == "Optional"
             ):
                 # Convert `abc: Optional[xyz]` to `abc: xyz`
-                node = ast.copy_location(
-                    ast.AnnAssign(
-                        target=node.target,
-                        annotation=node.annotation.slice,
-                        value=node.value,
-                        simple=node.simple,
-                    ),
-                    node,
+                node = ast.AnnAssign(
+                    target=node.target,
+                    annotation=node.annotation.slice,
+                    value=node.value,
+                    simple=node.simple,
                 )
             # Remove default value, e.g.
             # convert `abc: xyz = 123` to `abc: NotRequired[xyz]`
-            return ast.copy_location(
-                ast.AnnAssign(
-                    target=node.target,
-                    annotation=ast.Subscript(
-                        value=ast.Name(id="NotRequired", ctx=ast.Load()),
-                        slice=node.annotation,
-                        ctx=ast.Load(),
-                    ),
-                    simple=node.simple,
+            return ast.AnnAssign(
+                target=node.target,
+                annotation=ast.Subscript(
+                    value=ast.Name(id="NotRequired", ctx=ast.Load()),
+                    slice=node.annotation,
+                    ctx=ast.Load(),
                 ),
-                node,
+                simple=node.simple,
             )
         return node
 

packages/phoenix-client/src/phoenix/client/helpers/sdk/anthropic/messages.py

@@ -23,6 +23,7 @@
 from phoenix.client.__generated__.v1 import (
     ImageContentPart,
     ImageContentValue,
+    JSONSchemaDraft7ObjectSchema,
     PromptFunctionToolV1,
     PromptMessage,
     PromptToolsV1,
@@ -183,7 +184,7 @@ def _to_tools(
     obj: PromptToolsV1,
 ) -> Iterator[ToolParam]:
     for tool in obj["tools"]:
-        input_schema: dict[str, Any] = dict(tool["schema"]) if "schema" in tool else {}
+        input_schema: dict[str, Any] = dict(tool["schema"]["json"]) if "schema" in tool else {}
         param: ToolParam = {
             "name": tool["name"],
             "input_schema": input_schema,
@@ -205,7 +206,10 @@ def _from_tools(
         if "description" in tool:
             function["description"] = tool["description"]
         if "input_schema" in tool:
-            function["schema"] = tool["input_schema"]
+            function["schema"] = JSONSchemaDraft7ObjectSchema(
+                type="json-schema-draft-7-object-schema",
+                json=tool["input_schema"],
+            )
         functions.append(function)
     return PromptToolsV1(
         type="tools-v1",

packages/phoenix-client/src/phoenix/client/helpers/sdk/groq/chat.py

@@ -22,6 +22,7 @@
 from phoenix.client.__generated__.v1 import (
     ImageContentPart,
     ImageContentValue,
+    JSONSchemaDraft7ObjectSchema,
     PromptFunctionToolV1,
     PromptMessage,
     PromptToolsV1,
@@ -179,7 +180,7 @@ def _to_tools(
         if "description" in tool:
             function["description"] = tool["description"]
         if "schema" in tool and tool["schema"]:
-            function["parameters"] = dict(tool["schema"])
+            function["parameters"] = dict(tool["schema"]["json"])
         yield {"type": "function", "function": function}
 
 
@@ -196,7 +197,10 @@ def _from_tools(
         if "description" in definition:
             function["description"] = definition["description"]
         if "parameters" in definition:
-            function["schema"] = definition["parameters"]
+            function["schema"] = JSONSchemaDraft7ObjectSchema(
+                type="json-schema-draft-7-object-schema",
+                json=definition["parameters"],
+            )
         functions.append(function)
     return PromptToolsV1(type="tools-v1", tools=functions)
 

packages/phoenix-client/src/phoenix/client/helpers/sdk/mistralai/chat_complete.py

@@ -22,6 +22,7 @@
 from phoenix.client.__generated__.v1 import (
     ImageContentPart,
     ImageContentValue,
+    JSONSchemaDraft7ObjectSchema,
     PromptFunctionToolV1,
     PromptMessage,
     PromptToolsV1,
@@ -163,8 +164,8 @@ def _to_tools(
         }
         if "description" in tool:
             function["description"] = tool["description"]
-        if "schema" in tool and tool["schema"]:
-            function["parameters"] = dict(tool["schema"])
+        if "schema" in tool:
+            function["parameters"] = dict(tool["schema"]["json"])
         yield {"type": "function", "function": function}
 
 
@@ -180,7 +181,10 @@ def _from_tools(
         function = PromptFunctionToolV1(
             type="function-tool-v1",
             name=name,
-            schema=definition["parameters"],
+            schema=JSONSchemaDraft7ObjectSchema(
+                type="json-schema-draft-7-object-schema",
+                json=definition["parameters"],
+            ),
         )
         if "description" in definition:
             function["description"] = definition["description"]

packages/phoenix-client/src/phoenix/client/helpers/sdk/openai/chat.py

@@ -22,6 +22,7 @@
 from phoenix.client.__generated__.v1 import (
     ImageContentPart,
     ImageContentValue,
+    JSONSchemaDraft7ObjectSchema,
     PromptFunctionToolV1,
     PromptMessage,
     PromptToolsV1,
@@ -186,8 +187,8 @@ def _to_tools(
         function: FunctionDefinition = {"name": tool["name"]}
         if "description" in tool:
             function["description"] = tool["description"]
-        if "schema" in tool and tool["schema"]:
-            function["parameters"] = dict(tool["schema"])
+        if "schema" in tool:
+            function["parameters"] = dict(tool["schema"]["json"])
         yield {"type": "function", "function": function}
 
 
@@ -204,7 +205,10 @@ def _from_tools(
         if "description" in definition:
             function["description"] = definition["description"]
         if "parameters" in definition:
-            function["schema"] = definition["parameters"]
+            function["schema"] = JSONSchemaDraft7ObjectSchema(
+                type="json-schema-draft-7-object-schema",
+                json=definition["parameters"],
+            )
         functions.append(function)
     return PromptToolsV1(type="tools-v1", tools=functions)
 

packages/phoenix-client/src/phoenix/client/resources/prompts/__init__.py

@@ -20,7 +20,7 @@ def get(
         url = _url(prompt_version_id, prompt_identifier, tag)
         response = self._client.get(url)
         response.raise_for_status()
-        return cast(GetPromptResponseBody, response.content)["data"]
+        return cast(GetPromptResponseBody, response.json())["data"]
 
 
 class AsyncPrompts:
@@ -37,7 +37,7 @@ async def get(
         url = _url(prompt_version_id, prompt_identifier, tag)
         response = await self._client.get(url)
         response.raise_for_status()
-        return cast(GetPromptResponseBody, response.content)["data"]
+        return cast(GetPromptResponseBody, response.json())["data"]
 
 
 def _url(

pyproject.toml

@@ -60,6 +60,7 @@ dependencies = [
   "authlib",
   "websockets",
   "jsonschema>=4.0.0,<=4.23.0", # the upper bound is to keep us off the bleeding edge in case there's a regression since this controls what gets written to the database
+  "arize-phoenix-client",
 ]
 dynamic = ["version"]
 

requirements/canary/sdk/common.txt

@@ -1,4 +1,4 @@
-deepdiff==8.1.1
+deepdiff==8.2.0
 faker==33.3.1
 pyright[nodejs]==1.1.393
 pytest-randomly==3.16.0