F/ffmpeg integration: added new ffmpeg + cloudinary integrations (#853)

<!-- ELLIPSIS_HIDDEN -->


> [!IMPORTANT]
> Add FFmpeg and Cloudinary integrations, update models and
configurations, and enhance media processing functionalities.
> 
>   - **Integrations**:
> - Add FFmpeg and Cloudinary integrations in `integrations-service`.
>     - Update `providers.py` to include FFmpeg and Cloudinary.
>     - Add `cloudinary.py` and `ffmpeg.py` for handling operations.
>   - **Models**:
>     - Add new models for Cloudinary and FFmpeg in `Tools.py`.
> - Update `Chat.py`, `Entries.py`, and `Tasks.py` with new content
models.
>   - **Configurations**:
>     - Update `Dockerfile` to install FFmpeg.
>     - Add `GEMINI_API_KEY` to `.env.example` and `docker-compose.yml`.
>     - Update `gunicorn_conf.py` for logging and worker management.
>   - **Functions**:
>     - Modify `execute_integration.py` for robust error handling.
>     - Update `execute_system.py` to support session creation.
> - Enhance `base_evaluate.py` with increased `MAX_STRING_LENGTH` and
deep blob storage.
>   - **Misc**:
>     - Update `litellm-config.yaml` with new Gemini models.
>     - Add new TypeSpec files for Cloudinary and FFmpeg.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral)<sup>
for 0fe1cec. It will automatically
update as commits are pushed.</sup>


<!-- ELLIPSIS_HIDDEN -->

---------

Signed-off-by: Diwank Singh Tomer <[email protected]>
Co-authored-by: Ahmad-mtos <[email protected]>
Co-authored-by: Ahmad-mtos <[email protected]>
Co-authored-by: Vedantsahai18 <[email protected]>
Co-authored-by: HamadaSalhab <[email protected]>
Co-authored-by: HamadaSalhab <[email protected]>
Co-authored-by: Diwank Singh Tomer <[email protected]>

.env.example

@@ -34,6 +34,7 @@ LITELLM_REDIS_PASSWORD=<your_litellm_redis_password>
 # HUGGING_FACE_HUB_TOKEN=<your_hugging_face_hub_token>
 # ANTHROPIC_API_KEY=<your_anthropic_api_key>
 # GROQ_API_KEY=<your_groq_api_key>
+# GEMINI_API_KEY=<your_gemini_api_key>
 # CLOUDFLARE_API_KEY=<your_cloudflare_api_key>
 # CLOUDFLARE_ACCOUNT_ID=<your_cloudflare_account_id>
 # NVIDIA_NIM_API_KEY=<your_nvidia_nim_api_key>

agents-api/agents_api/activities/execute_integration.py

@@ -49,7 +49,10 @@ async def execute_integration(
             arguments=arguments,
         )
 
-        if "error" in integration_service_response:
+        if (
+            "error" in integration_service_response
+            and integration_service_response["error"]
+        ):
             raise Exception(integration_service_response["error"])
 
         return integration_service_response

agents-api/agents_api/activities/execute_system.py

@@ -14,6 +14,7 @@
     TextOnlyDocSearchRequest,
     VectorDocSearchRequest,
 )
+from ..autogen.Sessions import CreateSessionRequest
 from ..autogen.Tools import SystemDef
 from ..common.protocol.tasks import StepContext
 from ..common.storage_handler import auto_blob_store
@@ -96,6 +97,12 @@ async def execute_system(
             await bg_runner()
             return res
 
+        if system.operation == "create" and system.resource == "session":
+            developer_id = arguments.pop("developer_id")
+            session_id = arguments.pop("session_id", None)
+            data = CreateSessionRequest(**arguments)
+            return handler(developer_id=developer_id, session_id=session_id, data=data)
+
         # Handle regular operations
         if asyncio.iscoroutinefunction(handler):
             return await handler(**arguments)

agents-api/agents_api/activities/task_steps/base_evaluate.py

@@ -6,8 +6,8 @@
 from box import Box
 from openai import BaseModel
 
-# Increase the max string length to 300000
-simpleeval.MAX_STRING_LENGTH = 300000
+# Increase the max string length to 2048000
+simpleeval.MAX_STRING_LENGTH = 2048000
 
 from simpleeval import NameNotDefined, SimpleEval  # noqa: E402
 from temporalio import activity  # noqa: E402
@@ -63,7 +63,7 @@ def _recursive_evaluate(expr, evaluator: SimpleEval):
         raise ValueError(f"Invalid expression: {expr}")
 
 
-@auto_blob_store
+@auto_blob_store(deep=True)
 @beartype
 async def base_evaluate(
     exprs: Any,

agents-api/agents_api/autogen/Chat.py

@@ -165,7 +165,107 @@ class Content(BaseModel):
     """
 
 
+class ContentItem(Content):
+    pass
+
+
+class ContentItemModel(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal["image"] = "image"
+    source: Source
+
+
+class ContentItemModel1(Content):
+    pass
+
+
+class ContentItemModel2(ContentItemModel):
+    pass
+
+
+class ContentItemModel3(Content):
+    pass
+
+
+class ContentItemModel4(ContentItemModel):
+    pass
+
+
+class ContentItemModel5(Content):
+    pass
+
+
+class ContentItemModel6(ContentItemModel):
+    pass
+
+
 class ContentModel(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItem] | list[ContentItemModel]
+
+
+class ContentModel1(Content):
+    pass
+
+
+class ContentModel2(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItemModel1] | list[ContentItemModel2]
+
+
+class ContentModel3(Content):
+    pass
+
+
+class ContentModel4(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItemModel3] | list[ContentItemModel4]
+
+
+class ContentModel5(Content):
+    pass
+
+
+class ContentModel6(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItemModel5] | list[ContentItemModel6]
+
+
+class ContentModel7(BaseModel):
     model_config = ConfigDict(
         populate_by_name=True,
     )
@@ -193,7 +293,8 @@ class Delta(BaseModel):
     """
     tool_call_id: str | None = None
     content: Annotated[
-        str | list[str] | list[Content | ContentModel] | None, Field(...)
+        str | list[str] | list[ContentModel1 | ContentModel7 | ContentModel2] | None,
+        Field(...),
     ] = None
     """
     The content parts of the message
@@ -258,7 +359,8 @@ class Message(BaseModel):
     """
     tool_call_id: str | None = None
     content: Annotated[
-        str | list[str] | list[Content | ContentModel] | None, Field(...)
+        str | list[str] | list[Content | ContentModel7 | ContentModel] | None,
+        Field(...),
     ] = None
     """
     The content parts of the message
@@ -305,7 +407,8 @@ class MessageModel(BaseModel):
     """
     tool_call_id: str | None = None
     content: Annotated[
-        str | list[str] | list[Content | ContentModel] | None, Field(...)
+        str | list[str] | list[ContentModel3 | ContentModel7 | ContentModel4] | None,
+        Field(...),
     ] = None
     """
     The content parts of the message
@@ -405,6 +508,15 @@ class SingleChatOutput(BaseChatOutput):
     message: MessageModel
 
 
+class Source(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal["base64"] = "base64"
+    media_type: str
+    data: str
+
+
 class TokenLogProb(BaseTokenLogProb):
     model_config = ConfigDict(
         populate_by_name=True,

agents-api/agents_api/autogen/Entries.py

@@ -28,7 +28,7 @@ class BaseEntry(BaseModel):
     """
     name: str | None = None
     content: (
-        list[Content | ContentModel]
+        list[Content | ContentModel3 | ContentModel]
         | Tool
         | ChosenFunctionCall
         | ChosenComputer20241022
@@ -37,7 +37,7 @@ class BaseEntry(BaseModel):
         | str
         | ToolResponse
         | list[
-            list[Content | ContentModel]
+            list[ContentModel1 | ContentModel3 | ContentModel2]
             | Tool
             | ChosenFunctionCall
             | ChosenComputer20241022
@@ -95,7 +95,57 @@ class Content(BaseModel):
     """
 
 
+class ContentItem(Content):
+    pass
+
+
+class ContentItemModel(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal["image"] = "image"
+    source: Source
+
+
+class ContentItemModel1(Content):
+    pass
+
+
+class ContentItemModel2(ContentItemModel):
+    pass
+
+
 class ContentModel(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItem] | list[ContentItemModel]
+
+
+class ContentModel1(Content):
+    pass
+
+
+class ContentModel2(BaseModel):
+    """
+    Anthropic image content part
+    """
+
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    tool_use_id: str
+    type: Literal["tool_result"] = "tool_result"
+    content: list[ContentItemModel1] | list[ContentItemModel2]
+
+
+class ContentModel3(BaseModel):
     model_config = ConfigDict(
         populate_by_name=True,
     )
@@ -168,3 +218,12 @@ class Relation(BaseModel):
     head: UUID
     relation: str
     tail: UUID
+
+
+class Source(BaseModel):
+    model_config = ConfigDict(
+        populate_by_name=True,
+    )
+    type: Literal["base64"] = "base64"
+    media_type: str
+    data: str