From e986b823ddaf7f2c4857329fb5aa21e77da5e39e Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Mon, 12 Aug 2024 13:12:41 -0700 Subject: [PATCH 1/5] add client tools to schema --- .changeset/fifty-bags-type.md | 5 ++++ packages/ai-constructs/API.md | 10 ++++--- .../event_tools_provider.test.ts | 2 +- .../event_tools_provider.ts | 4 +-- .../runtime/event-tools-provider/types.ts | 4 +-- .../src/conversation/runtime/types.ts | 26 ++++++++++++------- .../conversation_handler_project.ts | 2 +- 7 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 .changeset/fifty-bags-type.md diff --git a/.changeset/fifty-bags-type.md b/.changeset/fifty-bags-type.md new file mode 100644 index 0000000000..32b5e50e13 --- /dev/null +++ b/.changeset/fifty-bags-type.md @@ -0,0 +1,5 @@ +--- +'@aws-amplify/ai-constructs': patch +--- + +Add client tools diff --git a/packages/ai-constructs/API.md b/packages/ai-constructs/API.md index 4a82cd2261..49ad0bcf8e 100644 --- a/packages/ai-constructs/API.md +++ b/packages/ai-constructs/API.md @@ -68,16 +68,14 @@ type ConversationTurnEvent = { }; messages: Array; toolsConfiguration?: { - tools: Array<{ - name: string; - description: string; - inputSchema: ToolInputSchema; + dataTools?: Array; }; }>; + clientTools?: Array; }; }; @@ -104,6 +102,10 @@ declare namespace runtime { } } +// Warnings were encountered during analysis: +// +// src/conversation/runtime/types.ts:48:5 - (ae-forgotten-export) The symbol "ToolDefinition" needs to be exported by the entry point index.d.ts + // (No @packageDocumentation comment for this package) ``` diff --git a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.test.ts b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.test.ts index 871a97f812..6d18194b87 100644 --- a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.test.ts +++ b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.test.ts @@ -64,7 +64,7 @@ void describe('events tool provider', () => { responseMutationInputTypeName: '', responseMutationName: '', toolsConfiguration: { - tools: [toolDefinition1, toolDefinition2], + dataTools: [toolDefinition1, toolDefinition2], }, }; const queryFactory = new GraphQlQueryFactory(); diff --git a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.ts b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.ts index 33be8c0eb5..c008e7905d 100644 --- a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.ts +++ b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/event_tools_provider.ts @@ -16,10 +16,10 @@ export class ConversationTurnEventToolsProvider { getEventTools = (): Array => { const { toolsConfiguration, graphqlApiEndpoint } = this.event; - if (!toolsConfiguration || !toolsConfiguration.tools) { + if (!toolsConfiguration || !toolsConfiguration.dataTools) { return []; } - const tools = toolsConfiguration.tools?.map((tool) => { + const tools = toolsConfiguration.dataTools?.map((tool) => { const { name, description, inputSchema } = tool; const query = this.graphQlQueryFactory.createQuery(tool); return new GraphQlTool( diff --git a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/types.ts b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/types.ts index 41bde5a174..bb7cc2e955 100644 --- a/packages/ai-constructs/src/conversation/runtime/event-tools-provider/types.ts +++ b/packages/ai-constructs/src/conversation/runtime/event-tools-provider/types.ts @@ -1,5 +1,5 @@ import { ConversationTurnEvent } from '../types'; export type ConversationTurnEventToolConfiguration = NonNullable< - ConversationTurnEvent['toolsConfiguration'] ->['tools'][number]; + NonNullable['dataTools'] +>[number]; diff --git a/packages/ai-constructs/src/conversation/runtime/types.ts b/packages/ai-constructs/src/conversation/runtime/types.ts index f685ca066f..7f2d95d905 100644 --- a/packages/ai-constructs/src/conversation/runtime/types.ts +++ b/packages/ai-constructs/src/conversation/runtime/types.ts @@ -19,6 +19,12 @@ export type ConversationMessageContentBlock = { text: string; }; +export type ToolDefinition = { + name: string; + description: string; + inputSchema: ToolInputSchema; +}; + // Customers are not expected to create events themselves, therefore // definition of nested properties is inline. export type ConversationTurnEvent = { @@ -39,16 +45,16 @@ export type ConversationTurnEvent = { }; messages: Array; toolsConfiguration?: { - tools: Array<{ - name: string; - description: string; - inputSchema: ToolInputSchema; - graphqlRequestInputDescriptor: { - queryName: string; - selectionSet: string[]; - propertyTypes: Record; - }; - }>; + dataTools?: Array< + ToolDefinition & { + graphqlRequestInputDescriptor: { + queryName: string; + selectionSet: string[]; + propertyTypes: Record; + }; + } + >; + clientTools?: Array; }; }; diff --git a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts index 6895a5eaeb..4863cb922c 100644 --- a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts +++ b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts @@ -273,7 +273,7 @@ class ConversationHandlerTestProject extends TestProjectBase { systemPrompt: 'You are helpful bot.', }, toolsConfiguration: { - tools: [ + dataTools: [ { name: 'thermometer', description: 'Provides the current temperature for a given city.', From 5bc285232a21456bcf5e45bba154ddf2377dcbca Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Mon, 12 Aug 2024 14:35:28 -0700 Subject: [PATCH 2/5] api --- packages/ai-constructs/API.md | 17 +++++++++-------- .../src/conversation/runtime/index.ts | 2 ++ .../src/conversation/runtime/types.ts | 5 +---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/packages/ai-constructs/API.md b/packages/ai-constructs/API.md index 872447f954..38328305ea 100644 --- a/packages/ai-constructs/API.md +++ b/packages/ai-constructs/API.md @@ -83,10 +83,7 @@ type ConversationTurnEvent = { }; // @public (undocumented) -type ExecutableTool = { - name: string; - description: string; - inputSchema: ToolInputSchema; +type ExecutableTool = ToolDefinition & { execute: (input: DocumentType | undefined) => Promise; }; @@ -101,13 +98,17 @@ declare namespace runtime { ConversationMessageContentBlock, ConversationTurnEvent, ExecutableTool, - handleConversationTurnEvent + handleConversationTurnEvent, + ToolDefinition } } -// Warnings were encountered during analysis: -// -// src/conversation/runtime/types.ts:48:5 - (ae-forgotten-export) The symbol "ToolDefinition" needs to be exported by the entry point index.d.ts +// @public (undocumented) +type ToolDefinition = { + name: string; + description: string; + inputSchema: ToolInputSchema; +}; // (No @packageDocumentation comment for this package) diff --git a/packages/ai-constructs/src/conversation/runtime/index.ts b/packages/ai-constructs/src/conversation/runtime/index.ts index fd9e4c6bff..4667775e69 100644 --- a/packages/ai-constructs/src/conversation/runtime/index.ts +++ b/packages/ai-constructs/src/conversation/runtime/index.ts @@ -3,6 +3,7 @@ import { ConversationMessageContentBlock, ConversationTurnEvent, ExecutableTool, + ToolDefinition, } from './types.js'; import { handleConversationTurnEvent } from './conversation_turn_executor.js'; @@ -13,4 +14,5 @@ export { ConversationTurnEvent, ExecutableTool, handleConversationTurnEvent, + ToolDefinition, }; diff --git a/packages/ai-constructs/src/conversation/runtime/types.ts b/packages/ai-constructs/src/conversation/runtime/types.ts index 17386d596b..60c3cf2f54 100644 --- a/packages/ai-constructs/src/conversation/runtime/types.ts +++ b/packages/ai-constructs/src/conversation/runtime/types.ts @@ -61,9 +61,6 @@ export type ConversationTurnEvent = { }; }; -export type ExecutableTool = { - name: string; - description: string; - inputSchema: ToolInputSchema; +export type ExecutableTool = ToolDefinition & { execute: (input: DocumentType | undefined) => Promise; }; From 6cbbfc8dba52ba8a67fc4d4dfe0a612218a3e179 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Mon, 12 Aug 2024 16:58:01 -0700 Subject: [PATCH 3/5] logic and unit tests --- .../runtime/bedrock_converse_adapter.test.ts | 152 +++++++++++++++++- .../runtime/bedrock_converse_adapter.ts | 65 ++++++-- 2 files changed, 196 insertions(+), 21 deletions(-) diff --git a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.test.ts b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.test.ts index ff565e448a..4dff39b3f0 100644 --- a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.test.ts +++ b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.test.ts @@ -1,6 +1,6 @@ import { describe, it, mock } from 'node:test'; import assert from 'node:assert'; -import { ConversationTurnEvent, ExecutableTool } from './types'; +import { ConversationTurnEvent, ExecutableTool, ToolDefinition } from './types'; import { BedrockConverseAdapter } from './bedrock_converse_adapter'; import { BedrockRuntimeClient, @@ -101,7 +101,7 @@ void describe('Bedrock converse adapter', () => { assert.deepStrictEqual(bedrockRequest.input, expectedBedrockInput); }); - void it('uses tools while calling bedrock', async () => { + void it('uses executable tools while calling bedrock', async () => { const additionalToolOutput: ToolResultContentBlock = { text: 'additionalToolOutput', }; @@ -322,6 +322,26 @@ void describe('Bedrock converse adapter', () => { new BedrockConverseAdapter( { ...commonEvent, + toolsConfiguration: { + clientTools: [ + { + // this one overlaps with executable tools below + name: 'duplicateName3', + description: '', + inputSchema: { json: {} }, + }, + { + name: 'duplicateName4', + description: '', + inputSchema: { json: {} }, + }, + { + name: 'duplicateName4', + description: '', + inputSchema: { json: {} }, + }, + ], + }, }, [ { @@ -348,19 +368,25 @@ void describe('Bedrock converse adapter', () => { inputSchema: { json: {} }, execute: () => Promise.reject(new Error()), }, + { + name: 'duplicateName3', + description: '', + inputSchema: { json: {} }, + execute: () => Promise.reject(new Error()), + }, ] ), (error: Error) => { assert.strictEqual( error.message, - 'Tools must have unique names. Duplicate tools: duplicateName1, duplicateName2.' + 'Tools must have unique names. Duplicate tools: duplicateName1, duplicateName2, duplicateName3, duplicateName4.' ); return true; } ); }); - void it('tool error is reported to bedrock', async () => { + void it('executable tool error is reported to bedrock', async () => { const tool: ExecutableTool = { name: 'testTool', description: 'tool description', @@ -454,7 +480,7 @@ void describe('Bedrock converse adapter', () => { } as Message); }); - void it('tool error of unknown type is reported to bedrock', async () => { + void it('executable tool error of unknown type is reported to bedrock', async () => { const tool: ExecutableTool = { name: 'testTool', description: 'tool description', @@ -549,4 +575,120 @@ void describe('Bedrock converse adapter', () => { ], } as Message); }); + + void it('returns client tool input block when client tool is requested and ignores executable tools', async () => { + const additionalToolOutput: ToolResultContentBlock = { + text: 'additionalToolOutput', + }; + const additionalTool: ExecutableTool = { + name: 'additionalTool', + description: 'additional tool description', + inputSchema: { + json: { + required: ['additionalToolRequiredProperty'], + }, + }, + execute: () => Promise.resolve(additionalToolOutput), + }; + const clientTool: ToolDefinition = { + name: 'clientTool', + description: 'client tool description', + inputSchema: { + json: { + required: ['clientToolRequiredProperty'], + }, + }, + }; + + const event: ConversationTurnEvent = { + ...commonEvent, + toolsConfiguration: { + clientTools: [clientTool], + }, + }; + + const bedrockClient = new BedrockRuntimeClient(); + const bedrockResponseQueue: Array = []; + const clientToolUseBlock = { + toolUse: { + toolUseId: randomUUID().toString(), + name: clientTool.name, + input: 'clientToolInput', + }, + }; + const toolUseBedrockResponse: ConverseCommandOutput = { + $metadata: {}, + metrics: undefined, + output: { + message: { + role: 'assistant', + content: [ + { + toolUse: { + toolUseId: randomUUID().toString(), + name: additionalTool.name, + input: 'additionalToolInput', + }, + }, + clientToolUseBlock, + ], + }, + }, + stopReason: 'tool_use', + usage: undefined, + }; + bedrockResponseQueue.push(toolUseBedrockResponse); + + const bedrockClientSendMock = mock.method(bedrockClient, 'send', () => + Promise.resolve(bedrockResponseQueue.shift()) + ); + + const responseContent = await new BedrockConverseAdapter( + event, + [additionalTool], + bedrockClient + ).askBedrock(); + + assert.deepStrictEqual(responseContent, [clientToolUseBlock]); + + assert.strictEqual(bedrockClientSendMock.mock.calls.length, 1); + const expectedToolConfig: ToolConfiguration = { + tools: [ + { + toolSpec: { + name: additionalTool.name, + description: additionalTool.description, + inputSchema: additionalTool.inputSchema, + }, + }, + { + toolSpec: { + name: clientTool.name, + description: clientTool.description, + inputSchema: clientTool.inputSchema, + }, + }, + ], + }; + const expectedBedrockInputCommonProperties = { + modelId: event.modelConfiguration.modelId, + inferenceConfig: { + maxTokens: 2000, + temperature: 0, + }, + system: [ + { + text: event.modelConfiguration.systemPrompt, + }, + ], + toolConfig: expectedToolConfig, + }; + const bedrockRequest = bedrockClientSendMock.mock.calls[0] + .arguments[0] as unknown as ConverseCommand; + const expectedBedrockInput: ConverseCommandInput = { + messages: event.messages, + ...expectedBedrockInputCommonProperties, + }; + assert.deepStrictEqual(bedrockRequest.input, expectedBedrockInput); + }); }); diff --git a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts index d5c34946df..8f6b741d10 100644 --- a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts +++ b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts @@ -8,7 +8,11 @@ import { Tool, ToolConfiguration, } from '@aws-sdk/client-bedrock-runtime'; -import { ConversationTurnEvent, ExecutableTool } from './types.js'; +import { + ConversationTurnEvent, + ExecutableTool, + ToolDefinition, +} from './types.js'; import { ConversationTurnEventToolsProvider } from './event-tools-provider'; /** @@ -16,8 +20,12 @@ import { ConversationTurnEventToolsProvider } from './event-tools-provider'; * in order to produce final response that can be sent back to caller. */ export class BedrockConverseAdapter { - private readonly allTools: Array; - private readonly toolByName: Map = new Map(); + private readonly allTools: Array; + private readonly allExecutableTools: Array; + private readonly clientTools: Array; + private readonly executableToolByName: Map = + new Map(); + private readonly clientToolByName: Map = new Map(); /** * Creates Bedrock Converse Adapter. @@ -30,13 +38,27 @@ export class BedrockConverseAdapter { ), eventToolsProvider = new ConversationTurnEventToolsProvider(event) ) { - this.allTools = [...eventToolsProvider.getEventTools(), ...additionalTools]; + this.allExecutableTools = [ + ...eventToolsProvider.getEventTools(), + ...additionalTools, + ]; + this.clientTools = this.event.toolsConfiguration?.clientTools ?? []; + this.allTools = [...this.allExecutableTools, ...this.clientTools]; const duplicateTools = new Set(); - this.allTools.forEach((t) => { - if (this.toolByName.has(t.name)) { + this.allExecutableTools.forEach((t) => { + if (this.executableToolByName.has(t.name)) { duplicateTools.add(t.name); } - this.toolByName.set(t.name, t); + this.executableToolByName.set(t.name, t); + }); + this.clientTools.forEach((t) => { + if (this.executableToolByName.has(t.name)) { + duplicateTools.add(t.name); + } + if (this.clientToolByName.has(t.name)) { + duplicateTools.add(t.name); + } + this.clientToolByName.set(t.name, t); }); if (duplicateTools.size > 0) { throw new Error( @@ -74,13 +96,24 @@ export class BedrockConverseAdapter { if (bedrockResponse.stopReason === 'tool_use') { const responseContentBlocks = bedrockResponse.output?.message?.content ?? []; - for (const responseContentBlock of responseContentBlocks) { - if ('toolUse' in responseContentBlock) { - const toolUseBlock = - responseContentBlock as ContentBlock.ToolUseMember; - const toolMessage = await this.executeTool(toolUseBlock); - messages.push(toolMessage); - } + const toolUseBlocks = responseContentBlocks.filter( + (block) => 'toolUse' in block + ) as Array; + const clientToolUseBlocks = responseContentBlocks.filter( + (block) => + block.toolUse?.name && + this.clientToolByName.has(block.toolUse?.name) + ); + if (clientToolUseBlocks.length > 0) { + // For now if any of client tools is used we ignore executable tools + // and propagate result back to client. + return clientToolUseBlocks; + } + for (const responseContentBlock of toolUseBlocks) { + const toolUseBlock = + responseContentBlock as ContentBlock.ToolUseMember; + const toolMessage = await this.executeTool(toolUseBlock); + messages.push(toolMessage); } } } while (bedrockResponse.stopReason === 'tool_use'); @@ -89,7 +122,7 @@ export class BedrockConverseAdapter { }; private createToolConfiguration = (): ToolConfiguration | undefined => { - if (this.allTools.length === 0) { + if (this.allExecutableTools.length === 0) { return undefined; } @@ -112,7 +145,7 @@ export class BedrockConverseAdapter { if (!toolUseBlock.toolUse.name) { throw Error('Bedrock tool use response is missing a tool name'); } - const tool = this.toolByName.get(toolUseBlock.toolUse.name); + const tool = this.executableToolByName.get(toolUseBlock.toolUse.name); if (!tool) { throw Error( `Bedrock tool use response contains unknown tool '${toolUseBlock.toolUse.name}'` From 8a4364a85fbd9b352448b20a532bd73852602397 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Mon, 12 Aug 2024 18:03:23 -0700 Subject: [PATCH 4/5] e2e --- .../runtime/bedrock_converse_adapter.ts | 10 +-- .../conversation_handler_project.ts | 72 +++++++++++++++++++ 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts index 8f6b741d10..ab90e0ff02 100644 --- a/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts +++ b/packages/ai-constructs/src/conversation/runtime/bedrock_converse_adapter.ts @@ -21,7 +21,7 @@ import { ConversationTurnEventToolsProvider } from './event-tools-provider'; */ export class BedrockConverseAdapter { private readonly allTools: Array; - private readonly allExecutableTools: Array; + private readonly executableTools: Array; private readonly clientTools: Array; private readonly executableToolByName: Map = new Map(); @@ -38,14 +38,14 @@ export class BedrockConverseAdapter { ), eventToolsProvider = new ConversationTurnEventToolsProvider(event) ) { - this.allExecutableTools = [ + this.executableTools = [ ...eventToolsProvider.getEventTools(), ...additionalTools, ]; this.clientTools = this.event.toolsConfiguration?.clientTools ?? []; - this.allTools = [...this.allExecutableTools, ...this.clientTools]; + this.allTools = [...this.executableTools, ...this.clientTools]; const duplicateTools = new Set(); - this.allExecutableTools.forEach((t) => { + this.executableTools.forEach((t) => { if (this.executableToolByName.has(t.name)) { duplicateTools.add(t.name); } @@ -122,7 +122,7 @@ export class BedrockConverseAdapter { }; private createToolConfiguration = (): ToolConfiguration | undefined => { - if (this.allExecutableTools.length === 0) { + if (this.allTools.length === 0) { return undefined; } diff --git a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts index 7c8c5f5691..00c5e9cadb 100644 --- a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts +++ b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts @@ -203,6 +203,13 @@ class ConversationHandlerTestProject extends TestProjectBase { clientConfig.data.url, apolloClient ); + + await this.assertDefaultConversationHandlerCanExecuteTurnWithClientTool( + backendId, + authenticatedUserCredentials.accessToken, + clientConfig.data.url, + apolloClient + ); } private assertDefaultConversationHandlerCanExecuteTurn = async ( @@ -320,6 +327,71 @@ class ConversationHandlerTestProject extends TestProjectBase { ); }; + private assertDefaultConversationHandlerCanExecuteTurnWithClientTool = async ( + backendId: BackendIdentifier, + accessToken: string, + graphqlApiEndpoint: string, + apolloClient: ApolloClient + ): Promise => { + const defaultConversationHandlerFunction = ( + await this.resourceFinder.findByBackendIdentifier( + backendId, + 'AWS::Lambda::Function', + (name) => name.includes('default') + ) + )[0]; + + // send event + const event: ConversationTurnEvent = { + conversationId: randomUUID().toString(), + currentMessageId: randomUUID().toString(), + graphqlApiEndpoint: graphqlApiEndpoint, + messages: [ + { + role: 'user', + content: [ + { + text: 'What is the temperature in Seattle?', + }, + ], + }, + ], + request: { + headers: { authorization: accessToken }, + }, + toolsConfiguration: { + clientTools: [ + { + name: 'thermometer', + description: 'Provides the current temperature for a given city.', + inputSchema: { + json: { + type: 'object', + properties: { + city: { + type: 'string', + description: 'string', + }, + }, + required: [], + }, + }, + }, + ], + }, + ...commonEventProperties, + }; + const response = await this.executeConversationTurn( + event, + defaultConversationHandlerFunction, + apolloClient + ); + // Assert that tool was used. I.e. that LLM used value returned by the tool. + assert.match(response.content, /Seattle/); + assert.match(response.content, /toolUse/); + assert.match(response.content, /toolUseId/); + }; + private assertCustomConversationHandlerCanExecuteTurn = async ( backendId: BackendIdentifier, accessToken: string, From 5253ff9eef2a683b55c2c6347b7ed8c150ba5408 Mon Sep 17 00:00:00 2001 From: Kamil Sobol Date: Tue, 13 Aug 2024 11:33:44 -0700 Subject: [PATCH 5/5] pr feedback --- .../src/test-project-setup/conversation_handler_project.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts index 00c5e9cadb..00ead426c4 100644 --- a/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts +++ b/packages/integration-tests/src/test-project-setup/conversation_handler_project.ts @@ -386,10 +386,13 @@ class ConversationHandlerTestProject extends TestProjectBase { defaultConversationHandlerFunction, apolloClient ); - // Assert that tool was used. I.e. that LLM used value returned by the tool. - assert.match(response.content, /Seattle/); + // Assert that tool use content blocks are emitted in case LLM selects client tool. + // The content blocks are string serialized, but not as a proper JSON, + // hence string matching is employed below to detect some signals that tool use blocks kinds were emitted. assert.match(response.content, /toolUse/); assert.match(response.content, /toolUseId/); + // Assert that LLM attempts to pass parameter when asking for tool use. + assert.match(response.content, /city=Seattle/); }; private assertCustomConversationHandlerCanExecuteTurn = async (