From 5351267a11db11559eaadb3d609cbe8b15903dbe Mon Sep 17 00:00:00 2001 From: tomfrenken Date: Thu, 9 Jan 2025 17:24:35 +0100 Subject: [PATCH] add tests and docs --- packages/orchestration/README.md | 9 ++-- .../src/orchestration-client.test.ts | 2 + ...rchestration-stream-chunk-response.test.ts | 19 +++---- .../src/orchestration-utils.temp.ts | 4 ++ packages/orchestration/tsconfig.json | 2 +- ...sponse-token-usage-and-finish-reason.json} | 0 ...ion-stream-chunk-response-token-usage.json | 49 ------------------- 7 files changed, 19 insertions(+), 66 deletions(-) create mode 100644 packages/orchestration/src/orchestration-utils.temp.ts rename test-util/data/orchestration/{orchestration-chat-completion-stream-chunk-response-finish-reason.json => orchestration-chat-completion-stream-chunk-response-token-usage-and-finish-reason.json} (100%) delete mode 100644 test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage.json diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md index be2494d3e..7bcf268fe 100644 --- a/packages/orchestration/README.md +++ b/packages/orchestration/README.md @@ -175,9 +175,10 @@ Abort controller can be useful, e.g., when end-user wants to stop the stream or #### Stream Options The orchestration service offers multiple streaming options, which you can configure in addition to the LLM's streaming options. -There are two ways to add specific streaming options to your client, either at initalization, or dynamically when calling the stream API. +These include options like definining the maximum number of characters per chunk or modifying the output filter behavior. +There are two ways to add specific streaming options to your client, either at initialization of orchestration client, or when calling the stream API. -Dynamically setting these options after client initialization is particularly helpful when you've initialized a client with a config meant for regular chat completion and now want to switch to using streaming. +Setting streaming options dynamically could be useful if an initialized orchestration client will also be used for streaming. You can check the list of available stream options in the [orchestration service's documentation](https://help.sap.com/docs/sap-ai-core/sap-ai-core-service-guide/streaming). @@ -197,8 +198,8 @@ const response = orchestrationClient.stream( ); ``` -Usage metrics are collected by default, if you do not want to receive them, set include_usage to false. -If you don't want any streaming options as part of your call to the LLM, set options.llm = null. +Usage metrics are collected by default, if you do not want to receive them, set `include_usage` to `false`. +If you don't want any streaming options as part of your call to the LLM, set `streamOptions.llm` to `null`. > [!NOTE] > When initalizing a client with a JSON module config, providing streaming options is not possible. diff --git a/packages/orchestration/src/orchestration-client.test.ts b/packages/orchestration/src/orchestration-client.test.ts index 016b3a5c3..731be1eb2 100644 --- a/packages/orchestration/src/orchestration-client.test.ts +++ b/packages/orchestration/src/orchestration-client.test.ts @@ -435,4 +435,6 @@ describe('orchestration service client', () => { 'Could not parse JSON' ); }); + + // add test for executing streaming with options with a JSON client, check for warning log { let mockResponses: { - tokenUsageResponse: any; - finishReasonResponse: any; + tokenUsageAndFinishReasonResponse: any; deltaContentResponse: any; }; let orchestrationStreamChunkResponses: { @@ -15,13 +14,9 @@ describe('Orchestration chat completion stream chunk response', () => { beforeAll(async () => { mockResponses = { - tokenUsageResponse: await parseMockResponse( + tokenUsageAndFinishReasonResponse: await parseMockResponse( 'orchestration', - 'orchestration-chat-completion-stream-chunk-response-token-usage.json' - ), - finishReasonResponse: await parseMockResponse( - 'orchestration', - 'orchestration-chat-completion-stream-chunk-response-finish-reason.json' + 'orchestration-chat-completion-stream-chunk-response-token-usage-and-finish-reason.json' ), deltaContentResponse: await parseMockResponse( 'orchestration', @@ -30,10 +25,10 @@ describe('Orchestration chat completion stream chunk response', () => { }; orchestrationStreamChunkResponses = { tokenUsageResponse: new OrchestrationStreamChunkResponse( - mockResponses.tokenUsageResponse + mockResponses.tokenUsageAndFinishReasonResponse ), finishReasonResponse: new OrchestrationStreamChunkResponse( - mockResponses.finishReasonResponse + mockResponses.tokenUsageAndFinishReasonResponse ), deltaContentResponse: new OrchestrationStreamChunkResponse( mockResponses.deltaContentResponse @@ -44,10 +39,10 @@ describe('Orchestration chat completion stream chunk response', () => { it('should return the chat completion stream chunk response', () => { expect( orchestrationStreamChunkResponses.tokenUsageResponse.data - ).toStrictEqual(mockResponses.tokenUsageResponse); + ).toStrictEqual(mockResponses.tokenUsageAndFinishReasonResponse); expect( orchestrationStreamChunkResponses.finishReasonResponse.data - ).toStrictEqual(mockResponses.finishReasonResponse); + ).toStrictEqual(mockResponses.tokenUsageAndFinishReasonResponse); expect( orchestrationStreamChunkResponses.deltaContentResponse.data ).toStrictEqual(mockResponses.deltaContentResponse); diff --git a/packages/orchestration/src/orchestration-utils.temp.ts b/packages/orchestration/src/orchestration-utils.temp.ts new file mode 100644 index 000000000..bcc1a03cd --- /dev/null +++ b/packages/orchestration/src/orchestration-utils.temp.ts @@ -0,0 +1,4 @@ +// create complete config +// test with config + addStreamOptionsToLlmModuleConfig +// test with config + addStreamOptionsToOutputFilteringConfig +// test complete flo with addStreamOptions diff --git a/packages/orchestration/tsconfig.json b/packages/orchestration/tsconfig.json index 2caf7a3cc..0cb2f59d1 100644 --- a/packages/orchestration/tsconfig.json +++ b/packages/orchestration/tsconfig.json @@ -6,7 +6,7 @@ "tsBuildInfoFile": "./dist/.tsbuildinfo", "composite": true }, - "include": ["src/**/*.ts"], + "include": ["src/**/*.ts", "src/orchestration-utils.temp.ts"], "exclude": ["dist/**/*", "test/**/*", "**/*.test.ts", "node_modules/**/*"], "references": [{ "path": "../core" }, { "path": "../ai-api" }] } diff --git a/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-finish-reason.json b/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage-and-finish-reason.json similarity index 100% rename from test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-finish-reason.json rename to test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage-and-finish-reason.json diff --git a/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage.json b/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage.json deleted file mode 100644 index 6da1a7ab8..000000000 --- a/test-util/data/orchestration/orchestration-chat-completion-stream-chunk-response-token-usage.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "request_id": "66172762-8c47-4438-89e7-2689be8f370b", - "module_results": { - "llm": { - "id": "chatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wp", - "object": "chat.completion.chunk", - "created": 1734524005, - "model": "gpt-4o-2024-08-06", - "system_fingerprint": "fp_4e924a4b48", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "content": "rate with SAP's enterprise solutions." - }, - "finish_reason": "stop" - } - ], - "usage": { - "completion_tokens": 271, - "prompt_tokens": 17, - "total_tokens": 288 - } - } - }, - "orchestration_result": { - "id": "chatcmpl-AfnDZfYvuE4SDplaLGF9v0PJjB0wp", - "object": "chat.completion.chunk", - "created": 1734524005, - "model": "gpt-4o-2024-08-06", - "system_fingerprint": "fp_4e924a4b48", - "choices": [ - { - "index": 0, - "delta": { - "role": "assistant", - "content": "rate with SAP's enterprise solutions." - }, - "finish_reason": "stop" - } - ], - "usage": { - "completion_tokens": 271, - "prompt_tokens": 17, - "total_tokens": 288 - } - } -} \ No newline at end of file