From 32b82b9cb378bfb6a38a87f9ee04071ba936a47c Mon Sep 17 00:00:00 2001 From: lloydzhou Date: Thu, 11 Jul 2024 00:48:58 +0800 Subject: [PATCH 1/4] change build messages for qwen in client --- app/api/alibaba/[...path]/route.ts | 35 ++++++++------------------- app/client/platforms/alibaba.ts | 39 ++++++++++++++++++++---------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/app/api/alibaba/[...path]/route.ts b/app/api/alibaba/[...path]/route.ts index b2c42ac78079..a8b2209ceaac 100644 --- a/app/api/alibaba/[...path]/route.ts +++ b/app/api/alibaba/[...path]/route.ts @@ -91,34 +91,14 @@ async function request(req: NextRequest) { ); const fetchUrl = `${baseUrl}${path}`; - - const clonedBody = await req.text(); - - const { messages, model, stream, top_p, ...rest } = JSON.parse( - clonedBody, - ) as RequestPayload; - - const requestBody = { - model, - input: { - messages, - }, - parameters: { - ...rest, - top_p: top_p === 1 ? 0.99 : top_p, // qwen top_p is should be < 1 - result_format: "message", - incremental_output: true, - }, - }; - const fetchOptions: RequestInit = { headers: { "Content-Type": "application/json", Authorization: req.headers.get("Authorization") ?? "", - "X-DashScope-SSE": stream ? "enable" : "disable", + "X-DashScope-SSE": req.headers.get("X-DashScope-SSE") ?? "disable", }, method: req.method, - body: JSON.stringify(requestBody), + body: req.body, redirect: "manual", // @ts-ignore duplex: "half", @@ -128,18 +108,23 @@ async function request(req: NextRequest) { // #1815 try to refuse some request to some models if (serverConfig.customModels && req.body) { try { + const clonedBody = await req.text(); + fetchOptions.body = clonedBody; + + const jsonBody = JSON.parse(clonedBody) as { model?: string }; + // not undefined and is false if ( isModelAvailableInServer( serverConfig.customModels, - model as string, - ServiceProvider.Alibaba as string, + jsonBody?.model as string, + ServiceProvider.ByteDance as string, ) ) { return NextResponse.json( { error: true, - message: `you are not allowed to use ${model} model`, + message: `you are not allowed to use ${jsonBody?.model} model`, }, { status: 403, diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 72126d7287aa..9c21a745aa50 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -32,19 +32,25 @@ export interface OpenAIListModelResponse { }>; } -interface RequestPayload { +interface RequestInput { messages: { role: "system" | "user" | "assistant"; content: string | MultimodalContent[]; }[]; - stream?: boolean; - model: string; +} +interface RequestParam { + result_format: string; + incremental_output?: boolean; temperature: number; - presence_penalty: number; - frequency_penalty: number; + repetition_penalty: number; top_p: number; max_tokens?: number; } +interface RequestPayload { + model: string; + input: RequestInput; + parameters: RequestParam; +} export class QwenApi implements LLMApi { path(path: string): string { @@ -91,17 +97,21 @@ export class QwenApi implements LLMApi { }, }; + const shouldStream = !!options.config.stream; const requestPayload: RequestPayload = { - messages, - stream: options.config.stream, model: modelConfig.model, - temperature: modelConfig.temperature, - presence_penalty: modelConfig.presence_penalty, - frequency_penalty: modelConfig.frequency_penalty, - top_p: modelConfig.top_p, + input: { + messages, + }, + parameters: { + result_format: "message", + incremental_output: shouldStream, + temperature: modelConfig.temperature, + // max_tokens: modelConfig.max_tokens, + top_p: modelConfig.top_p === 1 ? 0.99 : modelConfig.top_p, // qwen top_p is should be < 1 + }, }; - const shouldStream = !!options.config.stream; const controller = new AbortController(); options.onController?.(controller); @@ -111,7 +121,10 @@ export class QwenApi implements LLMApi { method: "POST", body: JSON.stringify(requestPayload), signal: controller.signal, - headers: getHeaders(), + headers: { + ...getHeaders(), + "X-DashScope-SSE": shouldStream ? "enable" : "disable", + }, }; // make a fetch request From 2299a4156db2f90112afc34b313d3d6941b20649 Mon Sep 17 00:00:00 2001 From: lloydzhou Date: Thu, 11 Jul 2024 00:50:58 +0800 Subject: [PATCH 2/4] change build messages for qwen in client --- app/client/platforms/alibaba.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 9c21a745aa50..723ba774b8e4 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -42,7 +42,7 @@ interface RequestParam { result_format: string; incremental_output?: boolean; temperature: number; - repetition_penalty: number; + repetition_penalty?: number; top_p: number; max_tokens?: number; } From fec36eb298b7e729ecaa2883f93fdb9e83a62087 Mon Sep 17 00:00:00 2001 From: lloydzhou Date: Thu, 11 Jul 2024 10:22:30 +0800 Subject: [PATCH 3/4] hotfix --- app/api/alibaba/[...path]/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/alibaba/[...path]/route.ts b/app/api/alibaba/[...path]/route.ts index a8b2209ceaac..c97ce5934732 100644 --- a/app/api/alibaba/[...path]/route.ts +++ b/app/api/alibaba/[...path]/route.ts @@ -118,7 +118,7 @@ async function request(req: NextRequest) { isModelAvailableInServer( serverConfig.customModels, jsonBody?.model as string, - ServiceProvider.ByteDance as string, + ServiceProvider.Alibaba as string, ) ) { return NextResponse.json( From 5e7254e8dc434fb57d5190e46cd7c97d81b3318e Mon Sep 17 00:00:00 2001 From: lloydzhou Date: Thu, 11 Jul 2024 14:46:12 +0800 Subject: [PATCH 4/4] hotfix: doubao display name --- app/utils/model.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/utils/model.ts b/app/utils/model.ts index a3a014877a9b..8f6a1a6c7863 100644 --- a/app/utils/model.ts +++ b/app/utils/model.ts @@ -61,7 +61,7 @@ export function collectModelTable( modelTable[fullName]["available"] = available; // swap name and displayName for bytedance if (providerName === "bytedance") { - [name, displayName] = [displayName, name]; + [name, displayName] = [displayName, modelName]; modelTable[fullName]["name"] = name; } if (displayName) {