diff --git a/.env.example b/.env.example index 96fa0abc1..ab4882297 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,7 @@ LITELLM_REDIS_PASSWORD= # HUGGING_FACE_HUB_TOKEN= # ANTHROPIC_API_KEY= # GROQ_API_KEY= +# GEMINI_API_KEY= # CLOUDFLARE_API_KEY= # CLOUDFLARE_ACCOUNT_ID= # NVIDIA_NIM_API_KEY= diff --git a/agents-api/agents_api/activities/execute_integration.py b/agents-api/agents_api/activities/execute_integration.py index 2badce0f9..d8b36fffb 100644 --- a/agents-api/agents_api/activities/execute_integration.py +++ b/agents-api/agents_api/activities/execute_integration.py @@ -49,7 +49,10 @@ async def execute_integration( arguments=arguments, ) - if "error" in integration_service_response: + if ( + "error" in integration_service_response + and integration_service_response["error"] + ): raise Exception(integration_service_response["error"]) return integration_service_response diff --git a/agents-api/agents_api/activities/execute_system.py b/agents-api/agents_api/activities/execute_system.py index 355d98614..43a2db8e5 100644 --- a/agents-api/agents_api/activities/execute_system.py +++ b/agents-api/agents_api/activities/execute_system.py @@ -14,6 +14,7 @@ TextOnlyDocSearchRequest, VectorDocSearchRequest, ) +from ..autogen.Sessions import CreateSessionRequest from ..autogen.Tools import SystemDef from ..common.protocol.tasks import StepContext from ..common.storage_handler import auto_blob_store @@ -96,6 +97,12 @@ async def execute_system( await bg_runner() return res + if system.operation == "create" and system.resource == "session": + developer_id = arguments.pop("developer_id") + session_id = arguments.pop("session_id", None) + data = CreateSessionRequest(**arguments) + return handler(developer_id=developer_id, session_id=session_id, data=data) + # Handle regular operations if asyncio.iscoroutinefunction(handler): return await handler(**arguments) diff --git a/agents-api/agents_api/activities/task_steps/base_evaluate.py b/agents-api/agents_api/activities/task_steps/base_evaluate.py index 06dacb361..d87b961d3 100644 --- a/agents-api/agents_api/activities/task_steps/base_evaluate.py +++ b/agents-api/agents_api/activities/task_steps/base_evaluate.py @@ -6,8 +6,8 @@ from box import Box from openai import BaseModel -# Increase the max string length to 300000 -simpleeval.MAX_STRING_LENGTH = 300000 +# Increase the max string length to 2048000 +simpleeval.MAX_STRING_LENGTH = 2048000 from simpleeval import NameNotDefined, SimpleEval # noqa: E402 from temporalio import activity # noqa: E402 @@ -63,7 +63,7 @@ def _recursive_evaluate(expr, evaluator: SimpleEval): raise ValueError(f"Invalid expression: {expr}") -@auto_blob_store +@auto_blob_store(deep=True) @beartype async def base_evaluate( exprs: Any, diff --git a/agents-api/agents_api/autogen/Chat.py b/agents-api/agents_api/autogen/Chat.py index d013ae8b7..042f9164d 100644 --- a/agents-api/agents_api/autogen/Chat.py +++ b/agents-api/agents_api/autogen/Chat.py @@ -165,7 +165,107 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + +class ContentItemModel3(Content): + pass + + +class ContentItemModel4(ContentItemModel): + pass + + +class ContentItemModel5(Content): + pass + + +class ContentItemModel6(ContentItemModel): + pass + + class ContentModel(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel1(Content): + pass + + +class ContentModel2(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + +class ContentModel3(Content): + pass + + +class ContentModel4(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel3] | list[ContentItemModel4] + + +class ContentModel5(Content): + pass + + +class ContentModel6(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel5] | list[ContentItemModel6] + + +class ContentModel7(BaseModel): model_config = ConfigDict( populate_by_name=True, ) @@ -193,7 +293,8 @@ class Delta(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[ContentModel1 | ContentModel7 | ContentModel2] | None, + Field(...), ] = None """ The content parts of the message @@ -258,7 +359,8 @@ class Message(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[Content | ContentModel7 | ContentModel] | None, + Field(...), ] = None """ The content parts of the message @@ -305,7 +407,8 @@ class MessageModel(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[ContentModel3 | ContentModel7 | ContentModel4] | None, + Field(...), ] = None """ The content parts of the message @@ -405,6 +508,15 @@ class SingleChatOutput(BaseChatOutput): message: MessageModel +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str + + class TokenLogProb(BaseTokenLogProb): model_config = ConfigDict( populate_by_name=True, diff --git a/agents-api/agents_api/autogen/Entries.py b/agents-api/agents_api/autogen/Entries.py index f001fc880..de37e77d8 100644 --- a/agents-api/agents_api/autogen/Entries.py +++ b/agents-api/agents_api/autogen/Entries.py @@ -28,7 +28,7 @@ class BaseEntry(BaseModel): """ name: str | None = None content: ( - list[Content | ContentModel] + list[Content | ContentModel3 | ContentModel] | Tool | ChosenFunctionCall | ChosenComputer20241022 @@ -37,7 +37,7 @@ class BaseEntry(BaseModel): | str | ToolResponse | list[ - list[Content | ContentModel] + list[ContentModel1 | ContentModel3 | ContentModel2] | Tool | ChosenFunctionCall | ChosenComputer20241022 @@ -95,7 +95,57 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + class ContentModel(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel1(Content): + pass + + +class ContentModel2(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + +class ContentModel3(BaseModel): model_config = ConfigDict( populate_by_name=True, ) @@ -168,3 +218,12 @@ class Relation(BaseModel): head: UUID relation: str tail: UUID + + +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str diff --git a/agents-api/agents_api/autogen/Tasks.py b/agents-api/agents_api/autogen/Tasks.py index e62e6d3c3..8e98caaab 100644 --- a/agents-api/agents_api/autogen/Tasks.py +++ b/agents-api/agents_api/autogen/Tasks.py @@ -9,6 +9,7 @@ from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, StrictBool from .Chat import ChatSettings +from .Common import JinjaTemplate from .Tools import ( ChosenBash20241022, ChosenComputer20241022, @@ -85,6 +86,26 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + class ContentModel(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -99,14 +120,40 @@ class ContentModel(BaseModel): """ -class ContentModel1(Content): +class ContentModel1(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel2(Content): pass -class ContentModel2(ContentModel): +class ContentModel3(ContentModel): pass +class ContentModel4(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + class CreateTaskRequest(BaseModel): """ Payload for creating a task @@ -655,7 +702,8 @@ class PromptItem(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - list[str] | list[Content | ContentModel] | str | None, Field(...) + list[str] | list[Content | ContentModel | ContentModel1] | str | None, + Field(...), ] """ The content parts of the message @@ -861,6 +909,18 @@ class SleepStep(BaseModel): """ +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str + """ + A valid jinja template. + """ + + class SwitchStep(BaseModel): model_config = ConfigDict( populate_by_name=True, diff --git a/agents-api/agents_api/autogen/Tools.py b/agents-api/agents_api/autogen/Tools.py index a48e2255e..cb548c0e3 100644 --- a/agents-api/agents_api/autogen/Tools.py +++ b/agents-api/agents_api/autogen/Tools.py @@ -12,6 +12,7 @@ BaseModel, ConfigDict, Field, + RootModel, StrictBool, ) @@ -198,6 +199,8 @@ class BaseIntegrationDef(BaseModel): "email", "remote_browser", "llama_parse", + "ffmpeg", + "cloudinary", ] """ The provider of the integration @@ -235,6 +238,8 @@ class BaseIntegrationDefUpdate(BaseModel): "email", "remote_browser", "llama_parse", + "ffmpeg", + "cloudinary", ] | None ) = None @@ -652,6 +657,154 @@ class ChosenTextEditor20241022(BaseModel): """ +class CloudinaryEditArguments(BaseModel): + """ + Arguments for Cloudinary media edit + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + public_id: str + """ + The file Public ID in Cloudinary + """ + transformation: list[dict[str, Any]] + """ + The transformation to apply to the file + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + + +class CloudinaryEditArgumentsUpdate(BaseModel): + """ + Arguments for Cloudinary media edit + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + public_id: str | None = None + """ + The file Public ID in Cloudinary + """ + transformation: list[dict[str, Any]] | None = None + """ + The transformation to apply to the file + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + + +class CloudinarySetup(BaseModel): + """ + Setup parameters for Cloudinary integration + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cloudinary_api_key: str + """ + The API key for Cloudinary + """ + cloudinary_api_secret: str + """ + The API secret for Cloudinary + """ + cloudinary_cloud_name: str + """ + The Cloud name for Cloudinary + """ + params: dict[str, Any] | None = None + """ + Additional parameters for the Cloudinary API + """ + + +class CloudinarySetupUpdate(BaseModel): + """ + Setup parameters for Cloudinary integration + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cloudinary_api_key: str | None = None + """ + The API key for Cloudinary + """ + cloudinary_api_secret: str | None = None + """ + The API secret for Cloudinary + """ + cloudinary_cloud_name: str | None = None + """ + The Cloud name for Cloudinary + """ + params: dict[str, Any] | None = None + """ + Additional parameters for the Cloudinary API + """ + + +class CloudinaryUploadArguments(BaseModel): + """ + Arguments for Cloudinary media upload + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + file: str + """ + The URL of the file upload + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + public_id: str | None = None + """ + Optional public ID for the uploaded file + """ + upload_params: dict[str, Any] | None = None + """ + Optional upload parameters + """ + + +class CloudinaryUploadArgumentsUpdate(BaseModel): + """ + Arguments for Cloudinary media upload + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + file: str | None = None + """ + The URL of the file upload + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + public_id: str | None = None + """ + Optional public ID for the uploaded file + """ + upload_params: dict[str, Any] | None = None + """ + Optional upload parameters + """ + + class Computer20241022Def(BaseModel): """ Anthropic new tools @@ -731,6 +884,9 @@ class CreateToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -922,6 +1078,94 @@ class EmailSetupUpdate(BaseModel): """ +class FfmpegIntegrationDef(BaseIntegrationDef): + """ + Ffmpeg integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["ffmpeg"] = "ffmpeg" + """ + The provider must be "ffmpeg" + """ + method: str | None = None + """ + The specific method of the integration to call + """ + setup: Any | None = None + """ + The setup parameters for Ffmpeg + """ + arguments: FfmpegSearchArguments | None = None + """ + The arguments for Ffmpeg Search + """ + + +class FfmpegIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + Ffmpeg integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["ffmpeg"] = "ffmpeg" + """ + The provider must be "ffmpeg" + """ + method: str | None = None + """ + The specific method of the integration to call + """ + setup: Any | None = None + """ + The setup parameters for Ffmpeg + """ + arguments: FfmpegSearchArgumentsUpdate | None = None + """ + The arguments for Ffmpeg Search + """ + + +class FfmpegSearchArguments(BaseModel): + """ + Arguments for Ffmpeg CMD + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cmd: str + """ + The bash command string + """ + file: str | None = None + """ + The base64 string of the file + """ + + +class FfmpegSearchArgumentsUpdate(BaseModel): + """ + Arguments for Ffmpeg CMD + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cmd: str | None = None + """ + The bash command string + """ + file: str | None = None + """ + The base64 string of the file + """ + + class FunctionCallOption(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -1165,6 +1409,9 @@ class PatchToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDefUpdate | RemoteBrowserIntegrationDefUpdate | LlamaParseIntegrationDefUpdate + | FfmpegIntegrationDefUpdate + | CloudinaryUploadIntegrationDefUpdate + | CloudinaryEditIntegrationDefUpdate | None ) = None """ @@ -1589,6 +1836,9 @@ class Tool(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -1679,6 +1929,9 @@ class UpdateToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -1954,6 +2207,32 @@ class BaseBrowserbaseIntegrationDefUpdate(BaseIntegrationDefUpdate): arguments: Any | None = None +class BaseCloudinaryIntegrationDef(BaseIntegrationDef): + """ + Base Cloudinary integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["cloudinary"] = "cloudinary" + setup: CloudinarySetup | None = None + method: Literal["media_upload", "media_edit"] | None = None + + +class BaseCloudinaryIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + Base Cloudinary integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["cloudinary"] = "cloudinary" + setup: CloudinarySetupUpdate | None = None + method: Literal["media_upload", "media_edit"] | None = None + + class BrowserbaseCompleteSessionIntegrationDef(BaseBrowserbaseIntegrationDef): """ browserbase complete session integration definition @@ -2192,3 +2471,51 @@ class BrowserbaseListSessionsIntegrationDefUpdate(BaseBrowserbaseIntegrationDefU """ The arguments for the method """ + + +class CloudinaryEditIntegrationDef(BaseCloudinaryIntegrationDef): + """ + Cloudinary edit integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_edit"] = "media_edit" + arguments: CloudinaryEditArguments | None = None + + +class CloudinaryEditIntegrationDefUpdate(BaseCloudinaryIntegrationDefUpdate): + """ + Cloudinary edit integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_edit"] = "media_edit" + arguments: CloudinaryEditArgumentsUpdate | None = None + + +class CloudinaryUploadIntegrationDef(BaseCloudinaryIntegrationDef): + """ + Cloudinary upload integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_upload"] = "media_upload" + arguments: CloudinaryUploadArguments | None = None + + +class CloudinaryUploadIntegrationDefUpdate(BaseCloudinaryIntegrationDefUpdate): + """ + Cloudinary upload integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_upload"] = "media_upload" + arguments: CloudinaryUploadArgumentsUpdate | None = None diff --git a/agents-api/agents_api/clients/temporal.py b/agents-api/agents_api/clients/temporal.py index 5737bd97e..316e0d256 100644 --- a/agents-api/agents_api/clients/temporal.py +++ b/agents-api/agents_api/clients/temporal.py @@ -10,8 +10,10 @@ ) from ..autogen.openapi_model import TransitionTarget +from ..common.protocol.remote import RemoteList from ..common.protocol.tasks import ExecutionInput from ..common.retry_policies import DEFAULT_RETRY_POLICY +from ..common.storage_handler import store_in_blob_store_if_large from ..env import ( temporal_client_cert, temporal_namespace, @@ -58,7 +60,9 @@ async def run_task_execution_workflow( previous_inputs: list[dict] = previous_inputs or [] client = client or (await get_client()) + execution_id = execution_input.execution.id execution_id_key = SearchAttributeKey.for_keyword("CustomStringField") + execution_input.arguments = store_in_blob_store_if_large(execution_input.arguments) return await client.start_workflow( TaskExecutionWorkflow.run, @@ -69,9 +73,7 @@ async def run_task_execution_workflow( retry_policy=DEFAULT_RETRY_POLICY, search_attributes=TypedSearchAttributes( [ - SearchAttributePair( - execution_id_key, str(execution_input.execution.id) - ), + SearchAttributePair(execution_id_key, str(execution_id)), ] ), ) diff --git a/agents-api/agents_api/common/protocol/tasks.py b/agents-api/agents_api/common/protocol/tasks.py index 6a379b6a4..2d5dd8946 100644 --- a/agents-api/agents_api/common/protocol/tasks.py +++ b/agents-api/agents_api/common/protocol/tasks.py @@ -75,7 +75,14 @@ valid_transitions: dict[TransitionType, list[TransitionType]] = { # Start state "init": ["wait", "error", "step", "cancelled", "init_branch", "finish"], - "init_branch": ["wait", "error", "step", "cancelled", "finish_branch"], + "init_branch": [ + "wait", + "error", + "step", + "cancelled", + "init_branch", + "finish_branch", + ], # End states "finish": [], "error": [], @@ -137,7 +144,7 @@ class ExecutionInput(BaseModel): task: TaskSpecDef agent: Agent agent_tools: list[Tool | CreateToolRequest] - arguments: dict[str, Any] + arguments: dict[str, Any] | RemoteObject # Not used at the moment user: User | None = None diff --git a/agents-api/agents_api/workflows/task_execution/helpers.py b/agents-api/agents_api/workflows/task_execution/helpers.py index 7fbc2c008..6d47b3a1f 100644 --- a/agents-api/agents_api/workflows/task_execution/helpers.py +++ b/agents-api/agents_api/workflows/task_execution/helpers.py @@ -66,7 +66,10 @@ async def execute_switch_branch( case_wf_name = f"`{context.cursor.workflow}`[{context.cursor.step}].case" case_task = execution_input.task.model_copy() - case_task.workflows = [Workflow(name=case_wf_name, steps=[chosen_branch.then])] + case_task.workflows = [ + Workflow(name=case_wf_name, steps=[chosen_branch.then]), + *case_task.workflows, + ] case_execution_input = execution_input.model_copy() case_execution_input.task = case_task @@ -99,7 +102,10 @@ async def execute_if_else_branch( if_else_wf_name += ".then" if condition else ".else" if_else_task = execution_input.task.model_copy() - if_else_task.workflows = [Workflow(name=if_else_wf_name, steps=[chosen_branch])] + if_else_task.workflows = [ + Workflow(name=if_else_wf_name, steps=[chosen_branch]), + *if_else_task.workflows, + ] if_else_execution_input = execution_input.model_copy() if_else_execution_input.task = if_else_task @@ -132,7 +138,10 @@ async def execute_foreach_step( f"`{context.cursor.workflow}`[{context.cursor.step}].foreach[{i}]" ) foreach_task = execution_input.task.model_copy() - foreach_task.workflows = [Workflow(name=foreach_wf_name, steps=[do_step])] + foreach_task.workflows = [ + Workflow(name=foreach_wf_name, steps=[do_step]), + *foreach_task.workflows, + ] foreach_execution_input = execution_input.model_copy() foreach_execution_input.task = foreach_task @@ -170,7 +179,10 @@ async def execute_map_reduce_step( f"`{context.cursor.workflow}`[{context.cursor.step}].mapreduce[{i}]" ) map_reduce_task = execution_input.task.model_copy() - map_reduce_task.workflows = [Workflow(name=workflow_name, steps=[map_defn])] + map_reduce_task.workflows = [ + Workflow(name=workflow_name, steps=[map_defn]), + *map_reduce_task.workflows, + ] map_reduce_execution_input = execution_input.model_copy() map_reduce_execution_input.task = map_reduce_task @@ -234,7 +246,10 @@ async def execute_map_reduce_step_parallel( # Note: Added PAR: prefix to easily identify parallel batches in logs workflow_name = f"PAR:`{context.cursor.workflow}`[{context.cursor.step}].mapreduce[{i}][{j}]" map_reduce_task = execution_input.task.model_copy() - map_reduce_task.workflows = [Workflow(name=workflow_name, steps=[map_defn])] + map_reduce_task.workflows = [ + Workflow(name=workflow_name, steps=[map_defn]), + *map_reduce_task.workflows, + ] map_reduce_execution_input = execution_input.model_copy() map_reduce_execution_input.task = map_reduce_task diff --git a/agents-api/gunicorn_conf.py b/agents-api/gunicorn_conf.py index f50b39c3d..d7c197e6d 100644 --- a/agents-api/gunicorn_conf.py +++ b/agents-api/gunicorn_conf.py @@ -1,7 +1,11 @@ import multiprocessing +import os + +TESTING = os.getenv("TESTING", "false").lower() == "true" +DEBUG = os.getenv("DEBUG", "false").lower() == "true" # Gunicorn config variables -workers = multiprocessing.cpu_count() - 1 +workers = multiprocessing.cpu_count() - 1 if not (TESTING or DEBUG) else 1 worker_class = "uvicorn.workers.UvicornWorker" bind = "0.0.0.0:8080" keepalive = 120 diff --git a/integrations-service/Dockerfile b/integrations-service/Dockerfile index 25c0abb12..d9a88d876 100644 --- a/integrations-service/Dockerfile +++ b/integrations-service/Dockerfile @@ -5,6 +5,19 @@ FROM python:3.12-slim WORKDIR /app +# Install system dependencies and FFmpeg with all required libraries +RUN apt-get update && apt-get install -y \ + ffmpeg \ + libavcodec-extra \ + libavformat-dev \ + libavutil-dev \ + libswscale-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Verify FFmpeg installation +RUN ffmpeg -version + # Install Poetry RUN pip install poetry @@ -18,5 +31,9 @@ RUN poetry config virtualenvs.create false \ # Copy project COPY . ./ -# Run the application +# Set proper signal handling +ENV PYTHONUNBUFFERED=1 +ENV GUNICORN_CMD_ARGS="--capture-output --enable-stdio-inheritance" + +# Run the application with proper signal handling ENTRYPOINT ["gunicorn", "integrations.web:app", "-c", "gunicorn_conf.py"] \ No newline at end of file diff --git a/integrations-service/gunicorn_conf.py b/integrations-service/gunicorn_conf.py index 30f5a2e4f..100b816dc 100644 --- a/integrations-service/gunicorn_conf.py +++ b/integrations-service/gunicorn_conf.py @@ -1,10 +1,38 @@ import multiprocessing +import os + +TESTING = os.getenv("TESTING", "false").lower() == "true" +DEBUG = os.getenv("DEBUG", "false").lower() == "true" # Gunicorn config variables -workers = multiprocessing.cpu_count() - 1 +workers = multiprocessing.cpu_count() - 1 if not (TESTING or DEBUG) else 1 worker_class = "uvicorn.workers.UvicornWorker" bind = "0.0.0.0:8000" keepalive = 120 timeout = 120 errorlog = "-" accesslog = "-" +loglevel = "info" +graceful_timeout = 30 +max_requests = 1000 +max_requests_jitter = 50 +preload_app = False + + +def when_ready(server): + """Run when server is ready to handle requests.""" + # Ensure proper permissions for any required directories + for directory in ["logs", "run"]: + path = os.path.join(os.getcwd(), directory) + if not os.path.exists(path): + os.makedirs(path, mode=0o755) + + +def on_starting(server): + """Run when server starts.""" + server.log.setup(server.app.cfg) + + +def worker_exit(server, worker): + """Clean up on worker exit.""" + server.log.info(f"Worker {worker.pid} exiting gracefully") diff --git a/integrations-service/integrations/autogen/Chat.py b/integrations-service/integrations/autogen/Chat.py index d013ae8b7..042f9164d 100644 --- a/integrations-service/integrations/autogen/Chat.py +++ b/integrations-service/integrations/autogen/Chat.py @@ -165,7 +165,107 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + +class ContentItemModel3(Content): + pass + + +class ContentItemModel4(ContentItemModel): + pass + + +class ContentItemModel5(Content): + pass + + +class ContentItemModel6(ContentItemModel): + pass + + class ContentModel(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel1(Content): + pass + + +class ContentModel2(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + +class ContentModel3(Content): + pass + + +class ContentModel4(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel3] | list[ContentItemModel4] + + +class ContentModel5(Content): + pass + + +class ContentModel6(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel5] | list[ContentItemModel6] + + +class ContentModel7(BaseModel): model_config = ConfigDict( populate_by_name=True, ) @@ -193,7 +293,8 @@ class Delta(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[ContentModel1 | ContentModel7 | ContentModel2] | None, + Field(...), ] = None """ The content parts of the message @@ -258,7 +359,8 @@ class Message(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[Content | ContentModel7 | ContentModel] | None, + Field(...), ] = None """ The content parts of the message @@ -305,7 +407,8 @@ class MessageModel(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - str | list[str] | list[Content | ContentModel] | None, Field(...) + str | list[str] | list[ContentModel3 | ContentModel7 | ContentModel4] | None, + Field(...), ] = None """ The content parts of the message @@ -405,6 +508,15 @@ class SingleChatOutput(BaseChatOutput): message: MessageModel +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str + + class TokenLogProb(BaseTokenLogProb): model_config = ConfigDict( populate_by_name=True, diff --git a/integrations-service/integrations/autogen/Entries.py b/integrations-service/integrations/autogen/Entries.py index f001fc880..de37e77d8 100644 --- a/integrations-service/integrations/autogen/Entries.py +++ b/integrations-service/integrations/autogen/Entries.py @@ -28,7 +28,7 @@ class BaseEntry(BaseModel): """ name: str | None = None content: ( - list[Content | ContentModel] + list[Content | ContentModel3 | ContentModel] | Tool | ChosenFunctionCall | ChosenComputer20241022 @@ -37,7 +37,7 @@ class BaseEntry(BaseModel): | str | ToolResponse | list[ - list[Content | ContentModel] + list[ContentModel1 | ContentModel3 | ContentModel2] | Tool | ChosenFunctionCall | ChosenComputer20241022 @@ -95,7 +95,57 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + class ContentModel(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel1(Content): + pass + + +class ContentModel2(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + +class ContentModel3(BaseModel): model_config = ConfigDict( populate_by_name=True, ) @@ -168,3 +218,12 @@ class Relation(BaseModel): head: UUID relation: str tail: UUID + + +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str diff --git a/integrations-service/integrations/autogen/Tasks.py b/integrations-service/integrations/autogen/Tasks.py index e62e6d3c3..8e98caaab 100644 --- a/integrations-service/integrations/autogen/Tasks.py +++ b/integrations-service/integrations/autogen/Tasks.py @@ -9,6 +9,7 @@ from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, StrictBool from .Chat import ChatSettings +from .Common import JinjaTemplate from .Tools import ( ChosenBash20241022, ChosenComputer20241022, @@ -85,6 +86,26 @@ class Content(BaseModel): """ +class ContentItem(Content): + pass + + +class ContentItemModel(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["image"] = "image" + source: Source + + +class ContentItemModel1(Content): + pass + + +class ContentItemModel2(ContentItemModel): + pass + + class ContentModel(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -99,14 +120,40 @@ class ContentModel(BaseModel): """ -class ContentModel1(Content): +class ContentModel1(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItem] | list[ContentItemModel] + + +class ContentModel2(Content): pass -class ContentModel2(ContentModel): +class ContentModel3(ContentModel): pass +class ContentModel4(BaseModel): + """ + Anthropic image content part + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + tool_use_id: str + type: Literal["tool_result"] = "tool_result" + content: list[ContentItemModel1] | list[ContentItemModel2] + + class CreateTaskRequest(BaseModel): """ Payload for creating a task @@ -655,7 +702,8 @@ class PromptItem(BaseModel): """ tool_call_id: str | None = None content: Annotated[ - list[str] | list[Content | ContentModel] | str | None, Field(...) + list[str] | list[Content | ContentModel | ContentModel1] | str | None, + Field(...), ] """ The content parts of the message @@ -861,6 +909,18 @@ class SleepStep(BaseModel): """ +class Source(BaseModel): + model_config = ConfigDict( + populate_by_name=True, + ) + type: Literal["base64"] = "base64" + media_type: str + data: str + """ + A valid jinja template. + """ + + class SwitchStep(BaseModel): model_config = ConfigDict( populate_by_name=True, diff --git a/integrations-service/integrations/autogen/Tools.py b/integrations-service/integrations/autogen/Tools.py index a48e2255e..cb548c0e3 100644 --- a/integrations-service/integrations/autogen/Tools.py +++ b/integrations-service/integrations/autogen/Tools.py @@ -12,6 +12,7 @@ BaseModel, ConfigDict, Field, + RootModel, StrictBool, ) @@ -198,6 +199,8 @@ class BaseIntegrationDef(BaseModel): "email", "remote_browser", "llama_parse", + "ffmpeg", + "cloudinary", ] """ The provider of the integration @@ -235,6 +238,8 @@ class BaseIntegrationDefUpdate(BaseModel): "email", "remote_browser", "llama_parse", + "ffmpeg", + "cloudinary", ] | None ) = None @@ -652,6 +657,154 @@ class ChosenTextEditor20241022(BaseModel): """ +class CloudinaryEditArguments(BaseModel): + """ + Arguments for Cloudinary media edit + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + public_id: str + """ + The file Public ID in Cloudinary + """ + transformation: list[dict[str, Any]] + """ + The transformation to apply to the file + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + + +class CloudinaryEditArgumentsUpdate(BaseModel): + """ + Arguments for Cloudinary media edit + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + public_id: str | None = None + """ + The file Public ID in Cloudinary + """ + transformation: list[dict[str, Any]] | None = None + """ + The transformation to apply to the file + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + + +class CloudinarySetup(BaseModel): + """ + Setup parameters for Cloudinary integration + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cloudinary_api_key: str + """ + The API key for Cloudinary + """ + cloudinary_api_secret: str + """ + The API secret for Cloudinary + """ + cloudinary_cloud_name: str + """ + The Cloud name for Cloudinary + """ + params: dict[str, Any] | None = None + """ + Additional parameters for the Cloudinary API + """ + + +class CloudinarySetupUpdate(BaseModel): + """ + Setup parameters for Cloudinary integration + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cloudinary_api_key: str | None = None + """ + The API key for Cloudinary + """ + cloudinary_api_secret: str | None = None + """ + The API secret for Cloudinary + """ + cloudinary_cloud_name: str | None = None + """ + The Cloud name for Cloudinary + """ + params: dict[str, Any] | None = None + """ + Additional parameters for the Cloudinary API + """ + + +class CloudinaryUploadArguments(BaseModel): + """ + Arguments for Cloudinary media upload + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + file: str + """ + The URL of the file upload + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + public_id: str | None = None + """ + Optional public ID for the uploaded file + """ + upload_params: dict[str, Any] | None = None + """ + Optional upload parameters + """ + + +class CloudinaryUploadArgumentsUpdate(BaseModel): + """ + Arguments for Cloudinary media upload + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + file: str | None = None + """ + The URL of the file upload + """ + return_base64: StrictBool = False + """ + Return base64 encoded file + """ + public_id: str | None = None + """ + Optional public ID for the uploaded file + """ + upload_params: dict[str, Any] | None = None + """ + Optional upload parameters + """ + + class Computer20241022Def(BaseModel): """ Anthropic new tools @@ -731,6 +884,9 @@ class CreateToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -922,6 +1078,94 @@ class EmailSetupUpdate(BaseModel): """ +class FfmpegIntegrationDef(BaseIntegrationDef): + """ + Ffmpeg integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["ffmpeg"] = "ffmpeg" + """ + The provider must be "ffmpeg" + """ + method: str | None = None + """ + The specific method of the integration to call + """ + setup: Any | None = None + """ + The setup parameters for Ffmpeg + """ + arguments: FfmpegSearchArguments | None = None + """ + The arguments for Ffmpeg Search + """ + + +class FfmpegIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + Ffmpeg integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["ffmpeg"] = "ffmpeg" + """ + The provider must be "ffmpeg" + """ + method: str | None = None + """ + The specific method of the integration to call + """ + setup: Any | None = None + """ + The setup parameters for Ffmpeg + """ + arguments: FfmpegSearchArgumentsUpdate | None = None + """ + The arguments for Ffmpeg Search + """ + + +class FfmpegSearchArguments(BaseModel): + """ + Arguments for Ffmpeg CMD + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cmd: str + """ + The bash command string + """ + file: str | None = None + """ + The base64 string of the file + """ + + +class FfmpegSearchArgumentsUpdate(BaseModel): + """ + Arguments for Ffmpeg CMD + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + cmd: str | None = None + """ + The bash command string + """ + file: str | None = None + """ + The base64 string of the file + """ + + class FunctionCallOption(BaseModel): model_config = ConfigDict( populate_by_name=True, @@ -1165,6 +1409,9 @@ class PatchToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDefUpdate | RemoteBrowserIntegrationDefUpdate | LlamaParseIntegrationDefUpdate + | FfmpegIntegrationDefUpdate + | CloudinaryUploadIntegrationDefUpdate + | CloudinaryEditIntegrationDefUpdate | None ) = None """ @@ -1589,6 +1836,9 @@ class Tool(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -1679,6 +1929,9 @@ class UpdateToolRequest(BaseModel): | BrowserbaseGetSessionConnectUrlIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryUploadIntegrationDef + | CloudinaryEditIntegrationDef | None ) = None """ @@ -1954,6 +2207,32 @@ class BaseBrowserbaseIntegrationDefUpdate(BaseIntegrationDefUpdate): arguments: Any | None = None +class BaseCloudinaryIntegrationDef(BaseIntegrationDef): + """ + Base Cloudinary integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["cloudinary"] = "cloudinary" + setup: CloudinarySetup | None = None + method: Literal["media_upload", "media_edit"] | None = None + + +class BaseCloudinaryIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + Base Cloudinary integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["cloudinary"] = "cloudinary" + setup: CloudinarySetupUpdate | None = None + method: Literal["media_upload", "media_edit"] | None = None + + class BrowserbaseCompleteSessionIntegrationDef(BaseBrowserbaseIntegrationDef): """ browserbase complete session integration definition @@ -2192,3 +2471,51 @@ class BrowserbaseListSessionsIntegrationDefUpdate(BaseBrowserbaseIntegrationDefU """ The arguments for the method """ + + +class CloudinaryEditIntegrationDef(BaseCloudinaryIntegrationDef): + """ + Cloudinary edit integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_edit"] = "media_edit" + arguments: CloudinaryEditArguments | None = None + + +class CloudinaryEditIntegrationDefUpdate(BaseCloudinaryIntegrationDefUpdate): + """ + Cloudinary edit integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_edit"] = "media_edit" + arguments: CloudinaryEditArgumentsUpdate | None = None + + +class CloudinaryUploadIntegrationDef(BaseCloudinaryIntegrationDef): + """ + Cloudinary upload integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_upload"] = "media_upload" + arguments: CloudinaryUploadArguments | None = None + + +class CloudinaryUploadIntegrationDefUpdate(BaseCloudinaryIntegrationDefUpdate): + """ + Cloudinary upload integration definition + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + method: Literal["media_upload"] = "media_upload" + arguments: CloudinaryUploadArgumentsUpdate | None = None diff --git a/integrations-service/integrations/models/__init__.py b/integrations-service/integrations/models/__init__.py index fdfc24951..2879a6fd3 100644 --- a/integrations-service/integrations/models/__init__.py +++ b/integrations-service/integrations/models/__init__.py @@ -33,7 +33,14 @@ from .browserbase import ( BrowserbaseListSessionsOutput as BrowserbaseListSessionsOutput, ) +from .cloudinary import ( + CloudinaryEditOutput as CloudinaryEditOutput, +) +from .cloudinary import ( + CloudinaryUploadOutput as CloudinaryUploadOutput, +) from .email import EmailOutput as EmailOutput +from .ffmpeg import FfmpegSearchOutput as FfmpegSearchOutput from .llama_parse import LlamaParseFetchOutput as LlamaParseFetchOutput from .remote_browser import RemoteBrowserOutput as RemoteBrowserOutput from .spider import SpiderFetchOutput as SpiderFetchOutput diff --git a/integrations-service/integrations/models/cloudinary.py b/integrations-service/integrations/models/cloudinary.py new file mode 100644 index 000000000..4ad59f4bf --- /dev/null +++ b/integrations-service/integrations/models/cloudinary.py @@ -0,0 +1,23 @@ +from typing import Optional + +from pydantic import Field + +from .base_models import BaseOutput + + +class CloudinaryUploadOutput(BaseOutput): + url: str = Field(..., description="The URL of the uploaded file") + public_id: str = Field(..., description="The public ID of the uploaded file") + base64: Optional[str] = Field( + None, description="The base64 encoded file if return_base64 is true" + ) + meta_data: Optional[dict] = Field( + None, description="Additional metadata from the upload response" + ) + + +class CloudinaryEditOutput(BaseOutput): + transformed_url: str = Field(..., description="The transformed URL") + base64: Optional[str] = Field( + None, description="The base64 encoded file if return_base64 is true" + ) diff --git a/integrations-service/integrations/models/execution.py b/integrations-service/integrations/models/execution.py index 40ba3fdba..693d9fa2d 100644 --- a/integrations-service/integrations/models/execution.py +++ b/integrations-service/integrations/models/execution.py @@ -16,8 +16,12 @@ BrowserbaseGetSessionLiveUrlsArguments, BrowserbaseListSessionsArguments, BrowserbaseSetup, + CloudinaryEditArguments, + CloudinarySetup, + CloudinaryUploadArguments, EmailArguments, EmailSetup, + FfmpegSearchArguments, LlamaParseFetchArguments, LlamaParseSetup, RemoteBrowserArguments, @@ -39,7 +43,9 @@ BrowserbaseGetSessionOutput, BrowserbaseListSessionsOutput, ) +from .cloudinary import CloudinaryEditOutput, CloudinaryUploadOutput from .email import EmailOutput +from .ffmpeg import FfmpegSearchOutput from .llama_parse import LlamaParseFetchOutput from .remote_browser import RemoteBrowserOutput from .spider import SpiderFetchOutput @@ -54,6 +60,7 @@ class ExecutionError(BaseModel): """ +# Setup configurations ExecutionSetup = Union[ EmailSetup, SpiderSetup, @@ -62,8 +69,10 @@ class ExecutionError(BaseModel): BrowserbaseSetup, RemoteBrowserSetup, LlamaParseSetup, + CloudinarySetup, ] +# Argument configurations ExecutionArguments = Union[ SpiderFetchArguments, WeatherGetArguments, @@ -80,6 +89,9 @@ class ExecutionError(BaseModel): BrowserbaseListSessionsArguments, RemoteBrowserArguments, LlamaParseFetchArguments, + FfmpegSearchArguments, + CloudinaryUploadArguments, + CloudinaryEditArguments, ] ExecutionResponse = Union[ @@ -98,6 +110,9 @@ class ExecutionError(BaseModel): BrowserbaseListSessionsOutput, RemoteBrowserOutput, LlamaParseFetchOutput, + FfmpegSearchOutput, + CloudinaryEditOutput, + CloudinaryUploadOutput, ExecutionError, ] diff --git a/integrations-service/integrations/models/ffmpeg.py b/integrations-service/integrations/models/ffmpeg.py new file mode 100644 index 000000000..ad773228c --- /dev/null +++ b/integrations-service/integrations/models/ffmpeg.py @@ -0,0 +1,15 @@ +from typing import Optional + +from pydantic import Field + +from .base_models import BaseOutput + + +class FfmpegSearchOutput(BaseOutput): + fileoutput: Optional[str] = Field( + None, description="The output file from the Ffmpeg command" + ) + result: bool = Field(..., description="Whether the Ffmpeg command was successful") + mime_type: Optional[str] = Field( + None, description="The MIME type of the output file" + ) diff --git a/integrations-service/integrations/providers.py b/integrations-service/integrations/providers.py index b467641bb..0589e4449 100644 --- a/integrations-service/integrations/providers.py +++ b/integrations-service/integrations/providers.py @@ -1,5 +1,7 @@ from .autogen.Tools import ( + # Arguments imports BraveSearchArguments, + # Setup imports BraveSearchSetup, BrowserbaseCompleteSessionArguments, BrowserbaseCreateSessionArguments, @@ -7,11 +9,14 @@ BrowserbaseGetSessionArguments, BrowserbaseGetSessionConnectUrlArguments, BrowserbaseGetSessionLiveUrlsArguments, - # WikipediaSearchSetup, BrowserbaseListSessionsArguments, BrowserbaseSetup, + CloudinaryEditArguments, + CloudinarySetup, + CloudinaryUploadArguments, EmailArguments, EmailSetup, + FfmpegSearchArguments, LlamaParseFetchArguments, LlamaParseSetup, RemoteBrowserArguments, @@ -33,7 +38,10 @@ BrowserbaseGetSessionLiveUrlsOutput, BrowserbaseGetSessionOutput, BrowserbaseListSessionsOutput, + CloudinaryEditOutput, + CloudinaryUploadOutput, EmailOutput, + FfmpegSearchOutput, LlamaParseFetchOutput, ProviderInfo, RemoteBrowserOutput, @@ -227,6 +235,50 @@ ), ) +ffmpeg = BaseProvider( + provider="ffmpeg", + setup=None, + methods=[ + BaseProviderMethod( + method="bash_cmd", + description="Run FFmpeg bash command", + arguments=FfmpegSearchArguments, + output=FfmpegSearchOutput, + ), + ], + info=ProviderInfo( + url="https://ffmpeg.org/", + docs="https://ffmpeg.org/documentation.html", + icon="https://upload.wikimedia.org/wikipedia/commons/5/5f/FFmpeg_Logo_new.svg", + friendly_name="Ffmpeg", + ), +) + +cloudinary = BaseProvider( + provider="cloudinary", + setup=CloudinarySetup, + methods=[ + BaseProviderMethod( + method="media_edit", + description="Edit media in Cloudinary", + arguments=CloudinaryEditArguments, + output=CloudinaryEditOutput, + ), + BaseProviderMethod( + method="media_upload", + description="Upload media to Cloudinary", + arguments=CloudinaryUploadArguments, + output=CloudinaryUploadOutput, + ), + ], + info=ProviderInfo( + url="https://cloudinary.com/", + docs="https://cloudinary.com/documentation/python_quickstart", + icon="https://cloudinary.com/favicon.ico", + friendly_name="Cloudinary", + ), +) + available_providers: dict[str, BaseProvider] = { "wikipedia": wikipedia, "weather": weather, @@ -236,4 +288,6 @@ "browserbase": browserbase, "remote_browser": remote_browser, "llama_parse": llama_parse, + "ffmpeg": ffmpeg, + "cloudinary": cloudinary, } diff --git a/integrations-service/integrations/utils/integrations/cloudinary.py b/integrations-service/integrations/utils/integrations/cloudinary.py new file mode 100644 index 000000000..5562ab968 --- /dev/null +++ b/integrations-service/integrations/utils/integrations/cloudinary.py @@ -0,0 +1,131 @@ +import base64 + +import aiohttp +import cloudinary +import cloudinary.uploader +from beartype import beartype +from tenacity import retry, stop_after_attempt, wait_exponential + +from ...autogen.Tools import ( + CloudinaryEditArguments, + CloudinarySetup, + CloudinaryUploadArguments, +) +from ...models.cloudinary import CloudinaryEditOutput, CloudinaryUploadOutput + + +@beartype +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) +async def media_upload( + setup: CloudinarySetup, arguments: CloudinaryUploadArguments +) -> CloudinaryUploadOutput: + """ + Upload media to Cloudinary. + """ + assert isinstance(setup, CloudinarySetup), "Invalid setup" + assert isinstance(arguments, CloudinaryUploadArguments), "Invalid arguments" + + try: + # Configure Cloudinary with credentials + cloudinary.config( + cloud_name=setup.cloudinary_cloud_name, + api_key=setup.cloudinary_api_key, + api_secret=setup.cloudinary_api_secret, + **(setup.params or {}), + ) + + # Upload the file + upload_params = arguments.upload_params or {} + if arguments.public_id: + upload_params["public_id"] = arguments.public_id + + result = cloudinary.uploader.upload(arguments.file, **upload_params) + + meta_data = { + key: value + for key, value in result.items() + if key not in ["secure_url", "public_id"] + } + + if arguments.return_base64: + async with aiohttp.ClientSession() as session: + async with session.get(result["secure_url"]) as response: + if response.status == 200: + content = await response.read() + base64_encoded = base64.b64encode(content).decode("utf-8") + result["base64"] = base64_encoded + else: + raise RuntimeError( + f"Failed to download file from URL: {result['secure_url']}" + ) + return CloudinaryUploadOutput( + url=result["secure_url"], + public_id=result["public_id"], + meta_data=meta_data, + base64=result["base64"] if arguments.return_base64 else None, + ) + + except cloudinary.exceptions.Error as e: + raise RuntimeError(f"Cloudinary error occurred: {e}") + except Exception as e: + raise RuntimeError(f"An unexpected error occurred: {e}") + + +@beartype +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) +async def media_edit( + setup: CloudinarySetup, arguments: CloudinaryEditArguments +) -> CloudinaryEditOutput: + """ + Edit media in Cloudinary. + """ + assert isinstance(setup, CloudinarySetup), "Invalid setup" + assert isinstance(arguments, CloudinaryEditArguments), "Invalid arguments" + + try: + # Configure Cloudinary with credentials + cloudinary.config( + cloud_name=setup.cloudinary_cloud_name, + api_key=setup.cloudinary_api_key, + api_secret=setup.cloudinary_api_secret, + **(setup.params or {}), + ) + + # Generate transformed URL + transformed_url = cloudinary.utils.cloudinary_url( + arguments.public_id, transformation=arguments.transformation + ) + if not transformed_url or not transformed_url[0]: + return CloudinaryEditOutput( + transformed_url="The transformation failed", + base64=None, + ) + if arguments.return_base64: + async with aiohttp.ClientSession() as session: + async with session.get(transformed_url[0]) as response: + if response.status == 200: + content = await response.read() + base64_encoded = base64.b64encode(content).decode("utf-8") + transformed_url_base64 = base64_encoded + else: + raise RuntimeError( + f"Failed to download file from URL: {transformed_url[0]}" + ) + + return CloudinaryEditOutput( + transformed_url=transformed_url[0], + base64=transformed_url_base64 if arguments.return_base64 else None, + ) + + except cloudinary.exceptions.Error as e: + raise RuntimeError(f"Cloudinary error occurred: {e}") + except Exception as e: + raise RuntimeError(f"An unexpected error occurred: {e}") diff --git a/integrations-service/integrations/utils/integrations/ffmpeg.py b/integrations-service/integrations/utils/integrations/ffmpeg.py new file mode 100644 index 000000000..3921e9736 --- /dev/null +++ b/integrations-service/integrations/utils/integrations/ffmpeg.py @@ -0,0 +1,145 @@ +import asyncio +import base64 +import os +import shutil +import tempfile +from functools import lru_cache +from typing import Tuple + +from beartype import beartype +from tenacity import retry, stop_after_attempt, wait_exponential + +from ...autogen.Tools import FfmpegSearchArguments +from ...models import FfmpegSearchOutput + + +# Cache for format validation +@lru_cache(maxsize=128) +def _sync_validate_format(binary_prefix: bytes) -> Tuple[bool, str]: + """Cached synchronous implementation of format validation""" + signatures = { + # Video formats + b"\x66\x74\x79\x70\x69\x73\x6f\x6d": "video/mp4", # MP4 + b"\x66\x74\x79\x70\x4d\x53\x4e\x56": "video/mp4", # MP4 + b"\x00\x00\x00\x1c\x66\x74\x79\x70": "video/mp4", # MP4 + b"\x1a\x45\xdf\xa3": "video/webm", # WebM + b"\x00\x00\x01\x00": "video/avi", # AVI + b"\x30\x26\xb2\x75": "video/wmv", # WMV + # Audio formats + b"\x49\x44\x33": "audio/mpeg", # MP3 + b"\xff\xfb": "audio/mpeg", # MP3 + b"\x52\x49\x46\x46": "audio/wav", # WAV + b"\x4f\x67\x67\x53": "audio/ogg", # OGG + b"\x66\x4c\x61\x43": "audio/flac", # FLAC + # Image formats + b"\xff\xd8\xff": "image/jpeg", # JPEG + b"\x89\x50\x4e\x47": "image/png", # PNG + b"\x47\x49\x46": "image/gif", # GIF + b"\x49\x49\x2a\x00": "image/tiff", # TIFF + b"\x42\x4d": "image/bmp", # BMP + } + + for signature, mime_type in signatures.items(): + if binary_prefix.startswith(signature): + return True, mime_type + + return False, "application/octet-stream" + + +async def validate_format(binary_data: bytes) -> Tuple[bool, str]: + """Validate file format using file signatures""" + # Only check first 16 bytes for efficiency + binary_prefix = binary_data[:16] + return _sync_validate_format(binary_prefix) + + +@beartype +@retry( + wait=wait_exponential(multiplier=1, min=4, max=10), + reraise=True, + stop=stop_after_attempt(4), +) +async def bash_cmd(arguments: FfmpegSearchArguments) -> FfmpegSearchOutput: + """Execute a FFmpeg bash command using base64-encoded input data.""" + try: + assert isinstance(arguments, FfmpegSearchArguments), "Invalid arguments" + + # Decode base64 input + try: + input_data = base64.b64decode(arguments.file) + except: + return FfmpegSearchOutput( + fileoutput="Error: Invalid base64 input", result=False, mime_type=None + ) + + # Validate input format + is_valid, input_mime = await validate_format(input_data) + + if not is_valid: + return FfmpegSearchOutput( + fileoutput="Error: Unsupported input file format", + result=False, + mime_type=None, + ) + + # Create temporary directory + temp_dir = tempfile.mkdtemp() + + # Get the output filename from the last argument of the FFmpeg command + cmd_parts = arguments.cmd.split() + output_filename = cmd_parts[-1] # e.g., "output.mp4" + output_path = os.path.join(temp_dir, output_filename) + + # Modify FFmpeg command + for i, part in enumerate(cmd_parts): + if part == "-i" and i + 1 < len(cmd_parts): + cmd_parts[i + 1] = "pipe:0" + cmd_parts[-1] = output_path # Replace the last argument with full output path + cmd = " ".join(cmd_parts) + + # Execute FFmpeg + process = await asyncio.create_subprocess_shell( + cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + # Process FFmpeg output + stdout, stderr = await process.communicate(input=input_data) + success = process.returncode == 0 + + if success and os.path.exists(output_path): + # Read the output file + with open(output_path, "rb") as f: + output_data = f.read() + # Convert to base64 + output_base64 = base64.b64encode(output_data).decode("utf-8") + + _, output_mime = await validate_format(output_data) + + # Clean up + shutil.rmtree(temp_dir) + + return FfmpegSearchOutput( + fileoutput=output_base64, + result=True, + mime_type=output_mime, + ) + + # Clean up in case of failure + shutil.rmtree(temp_dir) + error_msg = stderr.decode() if stderr else "Unknown error occurred" + return FfmpegSearchOutput( + fileoutput=f"Error: FFmpeg processing failed - {error_msg}", + result=False, + mime_type=None, + ) + + except Exception as e: + # Clean up in case of exception + if "temp_dir" in locals(): + shutil.rmtree(temp_dir) + return FfmpegSearchOutput( + fileoutput=f"Error: {str(e)}", result=False, mime_type=None + ) diff --git a/integrations-service/integrations/utils/integrations/remote_browser.py b/integrations-service/integrations/utils/integrations/remote_browser.py index 11e3e24a3..21ecc3a59 100644 --- a/integrations-service/integrations/utils/integrations/remote_browser.py +++ b/integrations-service/integrations/utils/integrations/remote_browser.py @@ -104,26 +104,26 @@ async def _reset_mouse(self) -> None: window.$$julep$$_initialized = true; """) - @staticmethod - def _with_error_and_screenshot(f): - @wraps(f) - async def wrapper(self: "PlaywrightActions", *args, **kwargs): - try: - result: RemoteBrowserOutput = await f(self, *args, **kwargs) - await self._wait_for_load() + # @staticmethod + # def _with_error_and_screenshot(f): + # @wraps(f) + # async def wrapper(self: "PlaywrightActions", *args, **kwargs): + # try: + # result: RemoteBrowserOutput = await f(self, *args, **kwargs) + # await self._wait_for_load() - screenshot: RemoteBrowserOutput = await self.take_screenshot() + # screenshot: RemoteBrowserOutput = await self.take_screenshot() - return RemoteBrowserOutput( - output=result.output, - base64_image=screenshot.base64_image, - system=result.system or f.__name__, - ) + # return RemoteBrowserOutput( + # output=result.output, + # base64_image=screenshot.base64_image, + # system=result.system or f.__name__, + # ) - except Exception as e: - return RemoteBrowserOutput(error=str(e)) + # except Exception as e: + # return RemoteBrowserOutput(error=str(e)) - return wrapper + # return wrapper async def _get_screen_size(self) -> tuple[int, int]: """Get the current browser viewport size""" @@ -198,7 +198,6 @@ def _overlay_cursor(self, screenshot_bytes: bytes, x: int, y: int) -> bytes: # --- # Actions - @_with_error_and_screenshot async def navigate(self, url: str) -> RemoteBrowserOutput: """Navigate to a specific URL""" await self.page.goto(url) @@ -208,7 +207,6 @@ async def navigate(self, url: str) -> RemoteBrowserOutput: output=url, ) - @_with_error_and_screenshot async def refresh(self) -> RemoteBrowserOutput: """Refresh the current page""" await self.page.reload() @@ -218,7 +216,6 @@ async def refresh(self) -> RemoteBrowserOutput: output="Refreshed page", ) - @_with_error_and_screenshot async def cursor_position(self) -> RemoteBrowserOutput: """Get current mouse coordinates""" x, y = await self._get_mouse_coordinates() @@ -226,11 +223,20 @@ async def cursor_position(self) -> RemoteBrowserOutput: output=f"X={x}, Y={y}", ) - @_with_error_and_screenshot async def press_key(self, key_combination: str) -> RemoteBrowserOutput: """Press a key or key combination""" # Split combination into individual keys keys = key_combination.split("+") + keys = [ + "Enter" + if k == "Return" + else "Control" + if k == "ctrl" + else "PageDown" + if k == "Page_Down" + else k + for k in keys + ] # Press modifier keys first for key in keys[:-1]: @@ -247,7 +253,6 @@ async def press_key(self, key_combination: str) -> RemoteBrowserOutput: output=f"Pressed {key_combination}", ) - @_with_error_and_screenshot async def type_text(self, text: str) -> RemoteBrowserOutput: """Type a string of text""" await self.page.keyboard.type(text) @@ -256,7 +261,6 @@ async def type_text(self, text: str) -> RemoteBrowserOutput: output=f"Typed {text}", ) - @_with_error_and_screenshot async def mouse_move(self, coordinate: tuple[int, int]) -> RemoteBrowserOutput: """Move mouse to specified coordinates""" await self.mouse.move(*coordinate) @@ -265,7 +269,6 @@ async def mouse_move(self, coordinate: tuple[int, int]) -> RemoteBrowserOutput: output=f"Moved mouse to {coordinate}", ) - @_with_error_and_screenshot async def left_click(self) -> RemoteBrowserOutput: """Perform left mouse click""" x, y = await self._get_mouse_coordinates() @@ -275,7 +278,6 @@ async def left_click(self) -> RemoteBrowserOutput: output="Left clicked", ) - @_with_error_and_screenshot async def left_click_drag(self, coordinate: tuple[int, int]) -> RemoteBrowserOutput: """Click and drag to specified coordinates""" await self.mouse.down() @@ -286,7 +288,6 @@ async def left_click_drag(self, coordinate: tuple[int, int]) -> RemoteBrowserOut output=f"Left clicked and dragged to {coordinate}", ) - @_with_error_and_screenshot async def right_click(self) -> RemoteBrowserOutput: """Perform right mouse click""" x, y = await self._get_mouse_coordinates() @@ -296,7 +297,6 @@ async def right_click(self) -> RemoteBrowserOutput: output="Right clicked", ) - @_with_error_and_screenshot async def middle_click(self) -> RemoteBrowserOutput: """Perform middle mouse click""" x, y = await self._get_mouse_coordinates() @@ -306,7 +306,6 @@ async def middle_click(self) -> RemoteBrowserOutput: output="Middle clicked", ) - @_with_error_and_screenshot async def double_click(self) -> RemoteBrowserOutput: """Perform double click""" x, y = await self._get_mouse_coordinates() diff --git a/integrations-service/poetry.lock b/integrations-service/poetry.lock index c106b4b4f..cbf0da504 100644 --- a/integrations-service/poetry.lock +++ b/integrations-service/poetry.lock @@ -517,6 +517,24 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "cloudinary" +version = "1.41.0" +description = "Python and Django SDK for Cloudinary" +optional = false +python-versions = "*" +files = [ + {file = "cloudinary-1.41.0.tar.gz", hash = "sha256:e189739a796a7d2ad15c19971741d33a9300816b16c0282b4b14ccf1dd2948c0"}, +] + +[package.dependencies] +certifi = "*" +six = "*" +urllib3 = ">=1.26.5" + +[package.extras] +dev = ["tox"] + [[package]] name = "colorama" version = "0.4.6" @@ -4062,4 +4080,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = ">=3.12,<3.13" -content-hash = "256945c59e69f4d19642f34ff712bd8d585832a7fc32548cd9d14e8813fa595a" +content-hash = "f395084232b0cbfaeb8c1fe90c971dbad935d5ef479178b672adaaed6c85e769" diff --git a/integrations-service/pyproject.toml b/integrations-service/pyproject.toml index b8de2236a..571846b9e 100644 --- a/integrations-service/pyproject.toml +++ b/integrations-service/pyproject.toml @@ -30,6 +30,7 @@ httpx = "^0.27.2" pillow = "^11.0.0" llama-index = "^0.11.22" llama-parse = "^0.5.13" +cloudinary = "^1.41.0" [tool.poe.tasks] format = "ruff format" diff --git a/llm-proxy/docker-compose.yml b/llm-proxy/docker-compose.yml index 0b084c385..0b9800b02 100644 --- a/llm-proxy/docker-compose.yml +++ b/llm-proxy/docker-compose.yml @@ -13,7 +13,7 @@ x--litellm-base: &litellm-base - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} - DATABASE_URL=${LITELLM_DATABASE_URL:-postgresql://${LITELLM_POSTGRES_USER:-llmproxy}:${LITELLM_POSTGRES_PASSWORD}@${LITELLM_POSTGRES_HOST:-litellm-db}:${LITELLM_POSTGRES_PORT:-5432}/${LITELLM_POSTGRES_DB:-litellm}?sslmode=${LITELLM_POSTGRES_SSLMODE:-prefer_ssl}} - REDIS_URL=${LITELLM_REDIS_URL:-${LITELLM_REDIS_PROTOCOL:-redis}://${LITELLM_REDIS_USER:-default}:${LITELLM_REDIS_PASSWORD:-${LITELLM_REDIS_PASSWORD}}@${LITELLM_REDIS_HOST:-litellm-redis}:${LITELLM_REDIS_PORT:-6379}} - + - GEMINI_API_KEY=${GEMINI_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - GROQ_API_KEY=${GROQ_API_KEY} diff --git a/llm-proxy/litellm-config.yaml b/llm-proxy/litellm-config.yaml index 3e0127036..8423b3bc2 100644 --- a/llm-proxy/litellm-config.yaml +++ b/llm-proxy/litellm-config.yaml @@ -6,6 +6,19 @@ model_list: # ------------------- # Gemini models + +- model_name: gemini-1.5-pro + litellm_params: + model: gemini/gemini-1.5-pro + api_key: os.environ/GEMINI_API_KEY + tags: ["paid"] + +- model_name: gemini-1.5-pro-latest + litellm_params: + model: gemini/gemini-1.5-pro-latest + api_key: os.environ/GEMINI_API_KEY + tags: ["paid"] + - model_name: gemini-1.5-pro litellm_params: model: vertex_ai_beta/gemini-1.5-pro diff --git a/typespec/entries/models.tsp b/typespec/entries/models.tsp index 61f47bce2..7f8c8b9fa 100644 --- a/typespec/entries/models.tsp +++ b/typespec/entries/models.tsp @@ -50,7 +50,25 @@ model ChatMLImageContentPart { type: "image_url" = "image_url"; } -alias ChatMLContentPart = ChatMLTextContentPart | ChatMLImageContentPart; +model ChatMLAnthropicImageSource { + type: "base64" = "base64"; + media_type: string; + data: T; +} + +model ChatMLAnthropicImageContentPart { + type: "image" = "image"; + source: ChatMLAnthropicImageSource; +} + +/** Anthropic image content part */ +model ChatMLAnthropicContentPart { + tool_use_id: string; + type: "tool_result" = "tool_result"; + content: ChatMLTextContentPart[] | ChatMLAnthropicImageContentPart[]; +} + +alias ChatMLContentPart = ChatMLTextContentPart | ChatMLImageContentPart | ChatMLAnthropicContentPart; model ChatMLMessage { /** The role of the message */ @@ -119,4 +137,4 @@ model History { session_id: Session.id; ...HasCreatedAt; -} +} \ No newline at end of file diff --git a/typespec/tools/cloudinary.tsp b/typespec/tools/cloudinary.tsp new file mode 100644 index 000000000..ec66962e1 --- /dev/null +++ b/typespec/tools/cloudinary.tsp @@ -0,0 +1,122 @@ +import "../common"; + +using Common; + +namespace Tools; + +/** Setup parameters for Cloudinary integration */ +model CloudinarySetup { + /** The API key for Cloudinary */ + cloudinary_api_key: string; + + /** The API secret for Cloudinary */ + cloudinary_api_secret: string; + + /** The Cloud name for Cloudinary */ + cloudinary_cloud_name: string; + + /** Additional parameters for the Cloudinary API */ + params?: Record; +} + +alias CloudinaryMethod = + | /** Upload media to Cloudinary */ + "media_upload" + | /** Edit media in Cloudinary */ + "media_edit"; + +/** Arguments for Cloudinary media upload */ +model CloudinaryUploadArguments { + /** The URL of the file upload */ + file: string; + + /** Return base64 encoded file */ + return_base64: boolean = false; + + /** Optional public ID for the uploaded file */ + public_id?: string; + + /** Optional upload parameters */ + upload_params?: Record; +} + +/** Arguments for Cloudinary media edit */ +model CloudinaryEditArguments { + /** The file Public ID in Cloudinary */ + public_id: string; + + /** The transformation to apply to the file */ + transformation: Array>; + + /** Return base64 encoded file */ + return_base64: boolean = false; +} + +/** Base Cloudinary integration definition */ +model BaseCloudinaryIntegrationDef extends BaseIntegrationDef { + provider: "cloudinary" = "cloudinary"; + setup?: CloudinarySetup; + method?: CloudinaryMethod; +} + +/** Cloudinary upload integration definition */ +model CloudinaryUploadIntegrationDef extends BaseCloudinaryIntegrationDef { + method: "media_upload" = "media_upload"; + arguments?: CloudinaryUploadArguments; +} + +/** Cloudinary edit integration definition */ +model CloudinaryEditIntegrationDef extends BaseCloudinaryIntegrationDef { + method: "media_edit" = "media_edit"; + arguments?: CloudinaryEditArguments; +} + +alias CloudinaryIntegrationDef = CloudinaryUploadIntegrationDef | CloudinaryEditIntegrationDef; + +/** Output for Cloudinary upload */ +model CloudinaryUploadOutput { + /** The URL of the uploaded file */ + url: string; + + /** The public ID of the uploaded file */ + public_id: string; + + /** The base64 encoded file */ + base64?: string; + + /** The metadata of the uploaded file */ + metadata: Record; +} + +/** Output for Cloudinary edit */ +model CloudinaryEditOutput { + /** The transformed URL from Cloudinary */ + transformed_url: string; + + /** The base64 encoded file */ + base64?: string; +} + +alias CloudinaryOutput = CloudinaryUploadOutput | CloudinaryEditOutput; + +/** Cloudinary Provider Card */ +model CloudinaryProviderCard extends BaseProviderCard { + provider: "cloudinary" = "cloudinary"; + setup: CloudinarySetup; + methods: ProviderMethod[] = #[ + #{ + method: "media_upload", + description: "Upload media to Cloudinary", + }, + #{ + method: "media_edit", + description: "Edit media in Cloudinary", + } + ]; + info: ProviderInfo = #{ + url: "https://cloudinary.com/", + docs: "https://cloudinary.com/documentation/python_quickstart", + icon: "https://cloudinary.com/favicon.ico", + friendly_name: "Cloudinary", + }; +} \ No newline at end of file diff --git a/typespec/tools/ffmpeg.tsp b/typespec/tools/ffmpeg.tsp new file mode 100644 index 000000000..a8f8baf03 --- /dev/null +++ b/typespec/tools/ffmpeg.tsp @@ -0,0 +1,56 @@ +import "../common"; + +using Common; + +namespace Tools; + +/** Arguments for Ffmpeg CMD */ +model FfmpegSearchArguments { + /** The bash command string */ + cmd: string; + + /** The base64 string of the file*/ + file?: string; + +} + +/** Ffmpeg integration definition */ +model FfmpegIntegrationDef extends BaseIntegrationDef { + /** The provider must be "ffmpeg" */ + provider: "ffmpeg" = "ffmpeg"; + + /** The specific method of the integration to call */ + method?: string; + + /** The setup parameters for Ffmpeg */ + setup?: null = null; + + /** The arguments for Ffmpeg Search */ + arguments?: FfmpegSearchArguments; +} + +/** Ffmpeg Provider Card */ +model FfmpegProviderCard extends BaseProviderCard { + provider: "ffmpeg" = "ffmpeg"; + setup: null = null; + methods: ProviderMethod[] = #[ + #{ + method: "bash_cmd", + description: "Run FFmpeg bash command", + } + ]; + info: ProviderInfo = #{ + url: "https://ffmpeg.org/", + docs: "https://ffmpeg.org/documentation.html", + icon: "https://upload.wikimedia.org/wikipedia/commons/5/5f/FFmpeg_Logo_new.svg", + friendly_name: "Ffmpeg", + }; +} + +/** Ffmpeg Search Output */ +model FfmpegSearchOutput { + /** The documents returned from the Ffmpeg search */ + file: string; + result: boolean; + mime_type: string; +} \ No newline at end of file diff --git a/typespec/tools/main.tsp b/typespec/tools/main.tsp index 6d09a3fb0..443329191 100644 --- a/typespec/tools/main.tsp +++ b/typespec/tools/main.tsp @@ -9,6 +9,8 @@ import "./wikipedia.tsp"; import "./browserbase"; import "./remote_browser.tsp"; import "./llama_parse.tsp"; +import "./ffmpeg.tsp"; +import "./cloudinary.tsp"; namespace Tools; diff --git a/typespec/tools/models.tsp b/typespec/tools/models.tsp index e7f5deb5e..c41154de0 100644 --- a/typespec/tools/models.tsp +++ b/typespec/tools/models.tsp @@ -24,6 +24,8 @@ alias integrationProvider = ( | "email" | "remote_browser" | "llama_parse" + | "ffmpeg" + | "cloudinary" ); enum ToolType { @@ -116,6 +118,8 @@ alias IntegrationDef = ( | BrowserbaseIntegrationDef | RemoteBrowserIntegrationDef | LlamaParseIntegrationDef + | FfmpegIntegrationDef + | CloudinaryIntegrationDef ); // diff --git a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml index 8b8afa84f..5c9a57c4c 100644 --- a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml +++ b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml @@ -1872,6 +1872,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -1972,6 +2031,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -2203,6 +2321,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -2353,6 +2530,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -2850,6 +3086,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part - $ref: '#/components/schemas/Tools.Tool' - $ref: '#/components/schemas/Tools.ChosenFunctionCall' - $ref: '#/components/schemas/Tools.ChosenComputer20241022' @@ -2902,6 +3197,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + type: string + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + type: string + description: Anthropic image content part - $ref: '#/components/schemas/Tools.Tool' - $ref: '#/components/schemas/Tools.ChosenFunctionCall' - $ref: '#/components/schemas/Tools.ChosenComputer20241022' @@ -4899,6 +5253,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + $ref: '#/components/schemas/Common.JinjaTemplate' + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + $ref: '#/components/schemas/Common.JinjaTemplate' + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -5049,6 +5462,65 @@ components: - image_url description: The type (fixed to 'image_url') default: image_url + - type: object + required: + - tool_use_id + - type + - content + properties: + tool_use_id: + type: string + type: + type: string + enum: + - tool_result + default: tool_result + content: + anyOf: + - type: array + items: + type: object + required: + - text + - type + properties: + text: + $ref: '#/components/schemas/Common.JinjaTemplate' + type: + type: string + enum: + - text + description: The type (fixed to 'text') + default: text + - type: array + items: + type: object + required: + - type + - source + properties: + type: + type: string + enum: + - image + default: image + source: + type: object + required: + - type + - media_type + - data + properties: + type: + type: string + enum: + - base64 + default: base64 + media_type: + type: string + data: + $ref: '#/components/schemas/Common.JinjaTemplate' + description: Anthropic image content part nullable: true description: The content parts of the message name: @@ -6200,6 +6672,44 @@ components: type: string readOnly: true description: The response tool value generated by the model + Tools.BaseCloudinaryIntegrationDef: + type: object + required: + - provider + properties: + provider: + type: string + enum: + - cloudinary + default: cloudinary + setup: + $ref: '#/components/schemas/Tools.CloudinarySetup' + method: + type: string + enum: + - media_upload + - media_edit + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDef' + description: Base Cloudinary integration definition + Tools.BaseCloudinaryIntegrationDefUpdate: + type: object + properties: + provider: + type: string + enum: + - cloudinary + default: cloudinary + setup: + $ref: '#/components/schemas/Tools.CloudinarySetupUpdate' + method: + type: string + enum: + - media_upload + - media_edit + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDefUpdate' + description: Base Cloudinary integration definition Tools.BaseIntegrationDef: type: object required: @@ -6217,6 +6727,8 @@ components: - email - remote_browser - llama_parse + - ffmpeg + - cloudinary description: The provider of the integration method: type: string @@ -6236,6 +6748,8 @@ components: browserbase: '#/components/schemas/Tools.BaseBrowserbaseIntegrationDef' remote_browser: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' llama_parse: '#/components/schemas/Tools.LlamaParseIntegrationDef' + ffmpeg: '#/components/schemas/Tools.FfmpegIntegrationDef' + cloudinary: '#/components/schemas/Tools.BaseCloudinaryIntegrationDef' description: Integration definition Tools.BaseIntegrationDefUpdate: type: object @@ -6252,6 +6766,8 @@ components: - email - remote_browser - llama_parse + - ffmpeg + - cloudinary description: The provider of the integration method: type: string @@ -6271,6 +6787,8 @@ components: browserbase: '#/components/schemas/Tools.BaseBrowserbaseIntegrationDefUpdate' remote_browser: '#/components/schemas/Tools.RemoteBrowserIntegrationDefUpdate' llama_parse: '#/components/schemas/Tools.LlamaParseIntegrationDefUpdate' + ffmpeg: '#/components/schemas/Tools.FfmpegIntegrationDefUpdate' + cloudinary: '#/components/schemas/Tools.BaseCloudinaryIntegrationDefUpdate' description: Integration definition Tools.Bash20241022Def: type: object @@ -6866,6 +7384,177 @@ components: type: integer format: uint16 description: The line range to view + Tools.CloudinaryEditArguments: + type: object + required: + - public_id + - transformation + - return_base64 + properties: + public_id: + type: string + description: The file Public ID in Cloudinary + transformation: + type: array + items: + type: object + additionalProperties: {} + description: The transformation to apply to the file + return_base64: + type: boolean + description: Return base64 encoded file + default: false + description: Arguments for Cloudinary media edit + Tools.CloudinaryEditArgumentsUpdate: + type: object + properties: + public_id: + type: string + description: The file Public ID in Cloudinary + transformation: + type: array + items: + type: object + additionalProperties: {} + description: The transformation to apply to the file + return_base64: + type: boolean + description: Return base64 encoded file + default: false + description: Arguments for Cloudinary media edit + Tools.CloudinaryEditIntegrationDef: + type: object + required: + - method + properties: + method: + type: string + enum: + - media_edit + default: media_edit + arguments: + $ref: '#/components/schemas/Tools.CloudinaryEditArguments' + allOf: + - $ref: '#/components/schemas/Tools.BaseCloudinaryIntegrationDef' + description: Cloudinary edit integration definition + Tools.CloudinaryEditIntegrationDefUpdate: + type: object + properties: + method: + type: string + enum: + - media_edit + default: media_edit + arguments: + $ref: '#/components/schemas/Tools.CloudinaryEditArgumentsUpdate' + allOf: + - $ref: '#/components/schemas/Tools.BaseCloudinaryIntegrationDefUpdate' + description: Cloudinary edit integration definition + Tools.CloudinarySetup: + type: object + required: + - cloudinary_api_key + - cloudinary_api_secret + - cloudinary_cloud_name + properties: + cloudinary_api_key: + type: string + description: The API key for Cloudinary + cloudinary_api_secret: + type: string + description: The API secret for Cloudinary + cloudinary_cloud_name: + type: string + description: The Cloud name for Cloudinary + params: + type: object + additionalProperties: {} + description: Additional parameters for the Cloudinary API + description: Setup parameters for Cloudinary integration + Tools.CloudinarySetupUpdate: + type: object + properties: + cloudinary_api_key: + type: string + description: The API key for Cloudinary + cloudinary_api_secret: + type: string + description: The API secret for Cloudinary + cloudinary_cloud_name: + type: string + description: The Cloud name for Cloudinary + params: + type: object + additionalProperties: {} + description: Additional parameters for the Cloudinary API + description: Setup parameters for Cloudinary integration + Tools.CloudinaryUploadArguments: + type: object + required: + - file + - return_base64 + properties: + file: + type: string + description: The URL of the file upload + return_base64: + type: boolean + description: Return base64 encoded file + default: false + public_id: + type: string + description: Optional public ID for the uploaded file + upload_params: + type: object + additionalProperties: {} + description: Optional upload parameters + description: Arguments for Cloudinary media upload + Tools.CloudinaryUploadArgumentsUpdate: + type: object + properties: + file: + type: string + description: The URL of the file upload + return_base64: + type: boolean + description: Return base64 encoded file + default: false + public_id: + type: string + description: Optional public ID for the uploaded file + upload_params: + type: object + additionalProperties: {} + description: Optional upload parameters + description: Arguments for Cloudinary media upload + Tools.CloudinaryUploadIntegrationDef: + type: object + required: + - method + properties: + method: + type: string + enum: + - media_upload + default: media_upload + arguments: + $ref: '#/components/schemas/Tools.CloudinaryUploadArguments' + allOf: + - $ref: '#/components/schemas/Tools.BaseCloudinaryIntegrationDef' + description: Cloudinary upload integration definition + Tools.CloudinaryUploadIntegrationDefUpdate: + type: object + properties: + method: + type: string + enum: + - media_upload + default: media_upload + arguments: + $ref: '#/components/schemas/Tools.CloudinaryUploadArgumentsUpdate' + allOf: + - $ref: '#/components/schemas/Tools.BaseCloudinaryIntegrationDefUpdate' + description: Cloudinary upload integration definition Tools.Computer20241022Action: type: string enum: @@ -6983,6 +7672,9 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' - $ref: '#/components/schemas/Tools.LlamaParseIntegrationDef' + - $ref: '#/components/schemas/Tools.FfmpegIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryUploadIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryEditIntegrationDef' description: The integration to call system: allOf: @@ -7147,6 +7839,76 @@ components: type: string description: The password of the email server description: Setup parameters for Email integration + Tools.FfmpegIntegrationDef: + type: object + required: + - provider + properties: + provider: + type: string + enum: + - ffmpeg + description: The provider must be "ffmpeg" + default: ffmpeg + method: + type: string + description: The specific method of the integration to call + setup: + nullable: true + description: The setup parameters for Ffmpeg + default: null + arguments: + allOf: + - $ref: '#/components/schemas/Tools.FfmpegSearchArguments' + description: The arguments for Ffmpeg Search + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDef' + description: Ffmpeg integration definition + Tools.FfmpegIntegrationDefUpdate: + type: object + properties: + provider: + type: string + enum: + - ffmpeg + description: The provider must be "ffmpeg" + default: ffmpeg + method: + type: string + description: The specific method of the integration to call + setup: + nullable: true + description: The setup parameters for Ffmpeg + default: null + arguments: + allOf: + - $ref: '#/components/schemas/Tools.FfmpegSearchArgumentsUpdate' + description: The arguments for Ffmpeg Search + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDefUpdate' + description: Ffmpeg integration definition + Tools.FfmpegSearchArguments: + type: object + required: + - cmd + properties: + cmd: + type: string + description: The bash command string + file: + type: string + description: The base64 string of the file + description: Arguments for Ffmpeg CMD + Tools.FfmpegSearchArgumentsUpdate: + type: object + properties: + cmd: + type: string + description: The bash command string + file: + type: string + description: The base64 string of the file + description: Arguments for Ffmpeg CMD Tools.FunctionCallOption: type: object required: @@ -7345,6 +8107,9 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDefUpdate' - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDefUpdate' - $ref: '#/components/schemas/Tools.LlamaParseIntegrationDefUpdate' + - $ref: '#/components/schemas/Tools.FfmpegIntegrationDefUpdate' + - $ref: '#/components/schemas/Tools.CloudinaryUploadIntegrationDefUpdate' + - $ref: '#/components/schemas/Tools.CloudinaryEditIntegrationDefUpdate' description: The integration to call system: allOf: @@ -7749,6 +8514,9 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' - $ref: '#/components/schemas/Tools.LlamaParseIntegrationDef' + - $ref: '#/components/schemas/Tools.FfmpegIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryUploadIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryEditIntegrationDef' description: The integration to call system: allOf: @@ -7841,6 +8609,9 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' - $ref: '#/components/schemas/Tools.LlamaParseIntegrationDef' + - $ref: '#/components/schemas/Tools.FfmpegIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryUploadIntegrationDef' + - $ref: '#/components/schemas/Tools.CloudinaryEditIntegrationDef' description: The integration to call system: allOf: