From 2c0a2f9a1b9a1e3e16801b18c058bff344519630 Mon Sep 17 00:00:00 2001 From: Diwank Singh Tomer Date: Tue, 29 Oct 2024 23:03:40 -0400 Subject: [PATCH] feat(typespec): Add types for remote_browser tools Signed-off-by: Diwank Singh Tomer --- agents-api/agents_api/autogen/Tools.py | 153 +++++++++++++++++- .../integrations/autogen/Tools.py | 153 +++++++++++++++++- typespec/tools/main.tsp | 1 + typespec/tools/models.tsp | 2 + typespec/tools/remote_browser.tsp | 114 +++++++++++++ .../@typespec/openapi3/openapi-1.0.0.yaml | 129 +++++++++++++++ 6 files changed, 548 insertions(+), 4 deletions(-) create mode 100644 typespec/tools/remote_browser.tsp diff --git a/agents-api/agents_api/autogen/Tools.py b/agents-api/agents_api/autogen/Tools.py index 1a5a069b0..cc0fc8266 100644 --- a/agents-api/agents_api/autogen/Tools.py +++ b/agents-api/agents_api/autogen/Tools.py @@ -174,7 +174,14 @@ class BaseIntegrationDef(BaseModel): populate_by_name=True, ) provider: Literal[ - "dummy", "weather", "wikipedia", "spider", "brave", "browserbase", "email" + "dummy", + "weather", + "wikipedia", + "spider", + "brave", + "browserbase", + "email", + "remote_browser", ] """ The provider of the integration @@ -203,7 +210,14 @@ class BaseIntegrationDefUpdate(BaseModel): ) provider: ( Literal[ - "dummy", "weather", "wikipedia", "spider", "brave", "browserbase", "email" + "dummy", + "weather", + "wikipedia", + "spider", + "brave", + "browserbase", + "email", + "remote_browser", ] | None ) = None @@ -696,6 +710,7 @@ class CreateToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ @@ -961,6 +976,7 @@ class PatchToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDefUpdate | BrowserbaseGetSessionLiveUrlsIntegrationDefUpdate | BrowserbaseGetSessionConnectUrlIntegrationDefUpdate + | RemoteBrowserIntegrationDefUpdate | None ) = None """ @@ -982,6 +998,137 @@ class PatchToolRequest(BaseModel): bash_20241022: Bash20241022DefUpdate | None = None +class RemoteBrowserArguments(BaseModel): + """ + The arguments for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + action: Literal[ + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", + "navigate", + "refresh", + "wait_for_load", + ] + """ + The action to perform + """ + text: str | None = None + """ + The text + """ + coordinate: list | None = None + """ + The coordinate to move the mouse to + """ + + +class RemoteBrowserArgumentsUpdate(BaseModel): + """ + The arguments for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + action: ( + Literal[ + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", + "navigate", + "refresh", + "wait_for_load", + ] + | None + ) = None + """ + The action to perform + """ + text: str | None = None + """ + The text + """ + coordinate: list | None = None + """ + The coordinate to move the mouse to + """ + + +class RemoteBrowserIntegrationDef(BaseIntegrationDef): + """ + The integration definition for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["remote_browser"] = "remote_browser" + setup: RemoteBrowserSetup + method: Literal["perform_action"] = "perform_action" + arguments: RemoteBrowserArguments + + +class RemoteBrowserIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + The integration definition for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["remote_browser"] = "remote_browser" + setup: RemoteBrowserSetupUpdate | None = None + method: Literal["perform_action"] = "perform_action" + arguments: RemoteBrowserArgumentsUpdate | None = None + + +class RemoteBrowserSetup(BaseModel): + """ + The setup parameters for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + connect_url: AnyUrl + """ + The connection URL for the remote browser + """ + + +class RemoteBrowserSetupUpdate(BaseModel): + """ + The setup parameters for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + connect_url: AnyUrl | None = None + """ + The connection URL for the remote browser + """ + + class SpiderFetchArguments(BaseModel): """ Arguments for Spider integration @@ -1240,6 +1387,7 @@ class Tool(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ @@ -1316,6 +1464,7 @@ class UpdateToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ diff --git a/integrations-service/integrations/autogen/Tools.py b/integrations-service/integrations/autogen/Tools.py index 1a5a069b0..cc0fc8266 100644 --- a/integrations-service/integrations/autogen/Tools.py +++ b/integrations-service/integrations/autogen/Tools.py @@ -174,7 +174,14 @@ class BaseIntegrationDef(BaseModel): populate_by_name=True, ) provider: Literal[ - "dummy", "weather", "wikipedia", "spider", "brave", "browserbase", "email" + "dummy", + "weather", + "wikipedia", + "spider", + "brave", + "browserbase", + "email", + "remote_browser", ] """ The provider of the integration @@ -203,7 +210,14 @@ class BaseIntegrationDefUpdate(BaseModel): ) provider: ( Literal[ - "dummy", "weather", "wikipedia", "spider", "brave", "browserbase", "email" + "dummy", + "weather", + "wikipedia", + "spider", + "brave", + "browserbase", + "email", + "remote_browser", ] | None ) = None @@ -696,6 +710,7 @@ class CreateToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ @@ -961,6 +976,7 @@ class PatchToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDefUpdate | BrowserbaseGetSessionLiveUrlsIntegrationDefUpdate | BrowserbaseGetSessionConnectUrlIntegrationDefUpdate + | RemoteBrowserIntegrationDefUpdate | None ) = None """ @@ -982,6 +998,137 @@ class PatchToolRequest(BaseModel): bash_20241022: Bash20241022DefUpdate | None = None +class RemoteBrowserArguments(BaseModel): + """ + The arguments for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + action: Literal[ + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", + "navigate", + "refresh", + "wait_for_load", + ] + """ + The action to perform + """ + text: str | None = None + """ + The text + """ + coordinate: list | None = None + """ + The coordinate to move the mouse to + """ + + +class RemoteBrowserArgumentsUpdate(BaseModel): + """ + The arguments for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + action: ( + Literal[ + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "screenshot", + "cursor_position", + "navigate", + "refresh", + "wait_for_load", + ] + | None + ) = None + """ + The action to perform + """ + text: str | None = None + """ + The text + """ + coordinate: list | None = None + """ + The coordinate to move the mouse to + """ + + +class RemoteBrowserIntegrationDef(BaseIntegrationDef): + """ + The integration definition for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["remote_browser"] = "remote_browser" + setup: RemoteBrowserSetup + method: Literal["perform_action"] = "perform_action" + arguments: RemoteBrowserArguments + + +class RemoteBrowserIntegrationDefUpdate(BaseIntegrationDefUpdate): + """ + The integration definition for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + provider: Literal["remote_browser"] = "remote_browser" + setup: RemoteBrowserSetupUpdate | None = None + method: Literal["perform_action"] = "perform_action" + arguments: RemoteBrowserArgumentsUpdate | None = None + + +class RemoteBrowserSetup(BaseModel): + """ + The setup parameters for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + connect_url: AnyUrl + """ + The connection URL for the remote browser + """ + + +class RemoteBrowserSetupUpdate(BaseModel): + """ + The setup parameters for the remote browser + """ + + model_config = ConfigDict( + populate_by_name=True, + ) + connect_url: AnyUrl | None = None + """ + The connection URL for the remote browser + """ + + class SpiderFetchArguments(BaseModel): """ Arguments for Spider integration @@ -1240,6 +1387,7 @@ class Tool(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ @@ -1316,6 +1464,7 @@ class UpdateToolRequest(BaseModel): | BrowserbaseUpdateSessionIntegrationDef | BrowserbaseGetSessionLiveUrlsIntegrationDef | BrowserbaseGetSessionConnectUrlIntegrationDef + | RemoteBrowserIntegrationDef | None ) = None """ diff --git a/typespec/tools/main.tsp b/typespec/tools/main.tsp index 38d4d7d57..1630ef965 100644 --- a/typespec/tools/main.tsp +++ b/typespec/tools/main.tsp @@ -7,6 +7,7 @@ import "./spider.tsp"; import "./weather.tsp"; import "./wikipedia.tsp"; import "./browserbase"; +import "./remote_browser.tsp"; namespace Tools; diff --git a/typespec/tools/models.tsp b/typespec/tools/models.tsp index dd41038b1..91ea7eb1c 100644 --- a/typespec/tools/models.tsp +++ b/typespec/tools/models.tsp @@ -22,6 +22,7 @@ alias integrationProvider = ( | "brave" | "browserbase" | "email" + | "remote_browser" ); enum ToolType { @@ -112,6 +113,7 @@ alias IntegrationDef = ( | WikipediaIntegrationDef | WeatherIntegrationDef | BrowserbaseIntegrationDef + | RemoteBrowserIntegrationDef ); // diff --git a/typespec/tools/remote_browser.tsp b/typespec/tools/remote_browser.tsp new file mode 100644 index 000000000..5719085c4 --- /dev/null +++ b/typespec/tools/remote_browser.tsp @@ -0,0 +1,114 @@ +import "../common"; + +using Common; + +namespace Tools; + +/* +async def perform_action( + self, + action: str, + coordinate: tuple[int, int] | None = None, + text: str | None = None, +) -> tuple[int, int] | str | None: + """Perform a specified automation action""" + + actions = { + # Anthropic + "key": lambda: text and self.press_key(text), + "type": lambda: text and self.type_text(text), + "mouse_move": lambda: coordinate and self.mouse_move(*coordinate), + "left_click": self.left_click, + "left_click_drag": lambda: coordinate + and self.left_click_drag(*coordinate), + "right_click": self.right_click, + "middle_click": self.middle_click, + "double_click": self.double_click, + "screenshot": self.take_screenshot, + "cursor_position": self.get_mouse_coordinates, + # Additional + "navigate": lambda: text and self.navigate(text), + "refresh": self.refresh, + "wait_for_load": self.wait_for_load, + } + + ... + +*/ + +alias RemoteBrowserAction = /** Press a key */ + + | "key" + | /** Type text */ + "type" + | /** Move the mouse to a coordinate */ + "mouse_move" + | /** Left click */ + "left_click" + | /** Left click and drag */ + "left_click_drag" + | /** Right click */ + "right_click" + | /** Middle click */ + "middle_click" + | /** Double click */ + "double_click" + | /** Take a screenshot */ + "screenshot" + | /** Get the cursor position */ + "cursor_position" + | /** Navigate to a URL */ + "navigate" + | /** Refresh the page */ + "refresh" + | /** Wait for the page to load */ + "wait_for_load"; + +/** The setup parameters for the remote browser */ +model RemoteBrowserSetup { + /** The connection URL for the remote browser */ + connect_url: url; +} + +/** The arguments for the remote browser */ +model RemoteBrowserArguments { + /** The action to perform */ + action: RemoteBrowserAction; + + /** The text */ + text?: string; + + /** The coordinate to move the mouse to */ + coordinate?: [uint16, uint16]; +} + +/** The output of the remote browser */ +model RemoteBrowserOutput { + /** The result of the action */ + result: unknown; +} + +/** The integration definition for the remote browser */ +model RemoteBrowserIntegrationDef extends BaseIntegrationDef { + provider: "remote_browser" = "remote_browser"; + setup: RemoteBrowserSetup; + method: "perform_action" = "perform_action"; + arguments: RemoteBrowserArguments; +} + +model RemoteBrowserProviderCard extends BaseProviderCard { + provider: "remote_browser" = "remote_browser"; + setup: RemoteBrowserSetup; + methods: ProviderMethod[] = #[ + #{ + method: "perform_action", + description: "Perform an action in the remote browser", + } + ]; + info: ProviderInfo = #{ + url: "https://playwright.dev/", + docs: "https://playwright.dev/docs/api/class-page", + icon: "https://playwright.dev/favicon.ico", + friendly_name: "Playwright", + }; +} diff --git a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml index e8fb1cf9c..97b211b62 100644 --- a/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml +++ b/typespec/tsp-output/@typespec/openapi3/openapi-1.0.0.yaml @@ -5903,6 +5903,7 @@ components: - brave - browserbase - email + - remote_browser description: The provider of the integration method: type: string @@ -5920,6 +5921,7 @@ components: wikipedia: '#/components/schemas/Tools.WikipediaIntegrationDef' weather: '#/components/schemas/Tools.WeatherIntegrationDef' browserbase: '#/components/schemas/Tools.BaseBrowserbaseIntegrationDef' + remote_browser: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' description: Integration definition Tools.BaseIntegrationDefUpdate: type: object @@ -5934,6 +5936,7 @@ components: - brave - browserbase - email + - remote_browser description: The provider of the integration method: type: string @@ -5951,6 +5954,7 @@ components: wikipedia: '#/components/schemas/Tools.WikipediaIntegrationDefUpdate' weather: '#/components/schemas/Tools.WeatherIntegrationDefUpdate' browserbase: '#/components/schemas/Tools.BaseBrowserbaseIntegrationDefUpdate' + remote_browser: '#/components/schemas/Tools.RemoteBrowserIntegrationDefUpdate' description: Integration definition Tools.Bash20241022Def: type: object @@ -6667,6 +6671,7 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseUpdateSessionIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionLiveUrlsIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' + - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' description: The integration to call system: allOf: @@ -6890,6 +6895,7 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseUpdateSessionIntegrationDefUpdate' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionLiveUrlsIntegrationDefUpdate' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDefUpdate' + - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDefUpdate' description: The integration to call system: allOf: @@ -6908,6 +6914,127 @@ components: bash_20241022: $ref: '#/components/schemas/Tools.Bash20241022DefUpdate' description: Payload for patching a tool + Tools.RemoteBrowserArguments: + type: object + required: + - action + properties: + action: + type: string + enum: + - key + - type + - mouse_move + - left_click + - left_click_drag + - right_click + - middle_click + - double_click + - screenshot + - cursor_position + - navigate + - refresh + - wait_for_load + description: The action to perform + text: + type: string + description: The text + coordinate: + type: array + items: {} + description: The coordinate to move the mouse to + description: The arguments for the remote browser + Tools.RemoteBrowserArgumentsUpdate: + type: object + properties: + action: + type: string + enum: + - key + - type + - mouse_move + - left_click + - left_click_drag + - right_click + - middle_click + - double_click + - screenshot + - cursor_position + - navigate + - refresh + - wait_for_load + description: The action to perform + text: + type: string + description: The text + coordinate: + type: array + items: {} + description: The coordinate to move the mouse to + description: The arguments for the remote browser + Tools.RemoteBrowserIntegrationDef: + type: object + required: + - provider + - setup + - method + - arguments + properties: + provider: + type: string + enum: + - remote_browser + default: remote_browser + setup: + $ref: '#/components/schemas/Tools.RemoteBrowserSetup' + method: + type: string + enum: + - perform_action + default: perform_action + arguments: + $ref: '#/components/schemas/Tools.RemoteBrowserArguments' + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDef' + description: The integration definition for the remote browser + Tools.RemoteBrowserIntegrationDefUpdate: + type: object + properties: + provider: + type: string + enum: + - remote_browser + default: remote_browser + setup: + $ref: '#/components/schemas/Tools.RemoteBrowserSetupUpdate' + method: + type: string + enum: + - perform_action + default: perform_action + arguments: + $ref: '#/components/schemas/Tools.RemoteBrowserArgumentsUpdate' + allOf: + - $ref: '#/components/schemas/Tools.BaseIntegrationDefUpdate' + description: The integration definition for the remote browser + Tools.RemoteBrowserSetup: + type: object + required: + - connect_url + properties: + connect_url: + type: string + format: uri + description: The connection URL for the remote browser + description: The setup parameters for the remote browser + Tools.RemoteBrowserSetupUpdate: + type: object + properties: + connect_url: + type: string + format: uri + description: The connection URL for the remote browser + description: The setup parameters for the remote browser Tools.SpiderFetchArguments: type: object required: @@ -7166,6 +7293,7 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseUpdateSessionIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionLiveUrlsIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' + - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' description: The integration to call system: allOf: @@ -7251,6 +7379,7 @@ components: - $ref: '#/components/schemas/Tools.BrowserbaseUpdateSessionIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionLiveUrlsIntegrationDef' - $ref: '#/components/schemas/Tools.BrowserbaseGetSessionConnectUrlIntegrationDef' + - $ref: '#/components/schemas/Tools.RemoteBrowserIntegrationDef' description: The integration to call system: allOf: