Fix structured output dispatch and settings (#14811)

* Fix structured output dispatch and settings fixed #14810 Signed-off-by: Jonas Helming <[email protected]>
eclipse-theia · Feb 3, 2025 · ebb1506 · ebb1506
1 parent 8ebc485
commit ebb1506
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 12 deletions.
diff --git a/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts b/packages/ai-openai/src/browser/openai-frontend-application-contribution.ts
@@ -93,6 +93,7 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio
                 model.apiKey === newModel.apiKey &&
                 model.apiVersion === newModel.apiVersion &&
                 model.supportsDeveloperMessage === newModel.supportsDeveloperMessage &&
+                model.supportsStructuredOutput === newModel.supportsStructuredOutput &&
                 model.enableStreaming === newModel.enableStreaming));
 
         this.manager.removeLanguageModels(...modelsToRemove.map(model => model.id));
@@ -118,6 +119,7 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio
             apiVersion: true,
             supportsDeveloperMessage: !openAIModelsSupportingDeveloperMessages.includes(modelId),
             enableStreaming: !openAIModelsWithDisabledStreaming.includes(modelId),
+            supportsStructuredOutput: !openAIModelsWithoutStructuredOutput.includes(modelId),
             defaultRequestSettings: modelRequestSetting?.requestSettings
         };
     }
@@ -142,6 +144,7 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio
                     apiKey: typeof pref.apiKey === 'string' || pref.apiKey === true ? pref.apiKey : undefined,
                     apiVersion: typeof pref.apiVersion === 'string' || pref.apiVersion === true ? pref.apiVersion : undefined,
                     supportsDeveloperMessage: pref.supportsDeveloperMessage ?? true,
+                    supportsStructuredOutput: pref.supportsStructuredOutput ?? true,
                     enableStreaming: pref.enableStreaming ?? true,
                     defaultRequestSettings: modelRequestSetting?.requestSettings
                 }
@@ -167,3 +170,4 @@ export class OpenAiFrontendApplicationContribution implements FrontendApplicatio
 
 const openAIModelsWithDisabledStreaming = ['o1-preview', 'o1-mini'];
 const openAIModelsSupportingDeveloperMessages = ['o1-preview', 'o1-mini'];
+const openAIModelsWithoutStructuredOutput = ['o1-preview', 'gpt-4-turbo', 'gpt-4', 'gpt-3.5-turbo', 'o1-mini', 'gpt-4o-2024-05-13'];
diff --git a/packages/ai-openai/src/browser/openai-preferences.ts b/packages/ai-openai/src/browser/openai-preferences.ts
@@ -54,6 +54,8 @@ export const OpenAiPreferencesSchema: PreferenceSchema = {
             \n\
             - specify `supportsDeveloperMessage: false` to indicate that the developer role shall not be used.\
             \n\
+            - specify `supportsStructuredOutput: false` to indicate that structured output shall not be used.\
+            \n\
             - specify `enableStreaming: false` to indicate that streaming shall not be used.\
             \n\
             Refer to [our documentation](https://theia-ide.org/docs/user_ai/#openai-compatible-models-eg-via-vllm) for more information.',
@@ -85,6 +87,10 @@ export const OpenAiPreferencesSchema: PreferenceSchema = {
                         type: 'boolean',
                         title: 'Indicates whether the model supports the `developer` role. `true` by default.',
                     },
+                    supportsStructuredOutput: {
+                        type: 'boolean',
+                        title: 'Indicates whether the model supports structured output. `true` by default.',
+                    },
                     enableStreaming: {
                         type: 'boolean',
                         title: 'Indicates whether the streaming API shall be used. `true` by default.',

diff --git a/packages/ai-openai/src/common/openai-language-models-manager.ts b/packages/ai-openai/src/common/openai-language-models-manager.ts
@@ -44,6 +44,10 @@ export interface OpenAiModelDescription {
      * Flag to configure whether the OpenAPI model supports the `developer` role. Default is `true`.
      */
     supportsDeveloperMessage: boolean;
+    /**
+     * Flag to configure whether the OpenAPI model supports structured output. Default is `true`.
+     */
+    supportsStructuredOutput: boolean;
     /**
      * Default request settings for the OpenAI model.
      */

diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts
@@ -50,6 +50,7 @@ export class OpenAiModel implements LanguageModel {
         public apiKey: () => string | undefined,
         public apiVersion: () => string | undefined,
         public supportsDeveloperMessage: boolean,
+        public supportsStructuredOutput: boolean,
         public url: string | undefined,
         public defaultRequestSettings?: { [key: string]: unknown }
     ) { }
@@ -66,13 +67,14 @@ export class OpenAiModel implements LanguageModel {
         const settings = this.getSettings(request);
         const openai = this.initializeOpenAi();
 
+        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput) {
+            return this.handleStructuredOutputRequest(openai, request);
+        }
+
         if (this.isNonStreamingModel(this.model) || (typeof settings.stream === 'boolean' && !settings.stream)) {
             return this.handleNonStreamingRequest(openai, request);
         }
 
-        if (request.response_format?.type === 'json_schema' && this.supportsStructuredOutput()) {
-            return this.handleStructuredOutputRequest(openai, request);
-        }
         if (cancellationToken?.isCancellationRequested) {
             return { text: '' };
         }
@@ -198,15 +200,6 @@ export class OpenAiModel implements LanguageModel {
         return !this.enableStreaming;
     }
 
-    protected supportsStructuredOutput(): boolean {
-        // see https://platform.openai.com/docs/models/gpt-4o
-        return [
-            'gpt-4o',
-            'gpt-4o-2024-08-06',
-            'gpt-4o-mini'
-        ].includes(this.model);
-    }
-
     protected async handleStructuredOutputRequest(openai: OpenAI, request: LanguageModelRequest): Promise<LanguageModelParsedResponse> {
         const settings = this.getSettings(request);
         // TODO implement tool support for structured output (parse() seems to require different tool format)

diff --git a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts
@@ -71,6 +71,7 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
                 model.apiKey = apiKeyProvider;
                 model.apiVersion = apiVersionProvider;
                 model.supportsDeveloperMessage = modelDescription.supportsDeveloperMessage;
+                model.supportsStructuredOutput = modelDescription.supportsStructuredOutput;
                 model.defaultRequestSettings = modelDescription.defaultRequestSettings;
             } else {
                 this.languageModelRegistry.addLanguageModels([
@@ -81,6 +82,7 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana
                         apiKeyProvider,
                         apiVersionProvider,
                         modelDescription.supportsDeveloperMessage,
+                        modelDescription.supportsStructuredOutput,
                         modelDescription.url,
                         modelDescription.defaultRequestSettings
                     )