fix: 修复 anthropic 默认 max_tokens 过大的错误

rxliuli · Aug 27, 2024 · e16adf4 · e16adf4
1 parent a77c0e6
commit e16adf4
Show file tree

Hide file tree

Showing 6 changed files with 16 additions and 11 deletions.
diff --git a/src/llm/anthropic.ts b/src/llm/anthropic.ts
@@ -109,6 +109,13 @@ async function convertMessages(
   )
 }
 
+function getModelMaxTokens(model: string): number {
+  if (model.startsWith('claude-3-')) {
+    return 4096
+  }
+  return 8192
+}
+
 export function anthropicBase(
   createClient: () => AnthropicTypes,
 ): Omit<IAnthropicVertex, 'requiredEnv' | 'supportModels'> {
@@ -135,7 +142,7 @@ export function anthropicBase(
             ).includes(it.role),
           ),
         ),
-        max_tokens: req.max_tokens ?? 8192,
+        max_tokens: req.max_tokens ?? getModelMaxTokens(req.model),
         temperature: req.temperature!,
         metadata: {
           user_id: req.user,

diff --git a/src/llm/deepseek.ts b/src/llm/deepseek.ts
@@ -2,7 +2,7 @@ import { IChat } from './base'
 import { openaiBase } from './openai'
 
 export function deepseek(env: Record<string, string>): IChat {
-  const r = openaiBase(env, {
+  const r = openaiBase({
     apiKey: env.DEEPSEEK_API_KEY,
     baseURL: 'https://api.deepseek.com',
   })

diff --git a/src/llm/groq.ts b/src/llm/groq.ts
@@ -2,7 +2,7 @@ import { IChat } from './base'
 import { openaiBase } from './openai'
 
 export function groq(env: Record<string, string>): IChat {
-  const r = openaiBase(env, {
+  const r = openaiBase({
     apiKey: env.GROQ_API_KEY,
     baseURL: 'https://api.groq.com/openai/v1',
   })

diff --git a/src/llm/lingyiwanwu.ts b/src/llm/lingyiwanwu.ts
@@ -1,7 +1,7 @@
 import { openaiBase } from './openai'
 
 export function lingyiwanwu(env: Record<string, string>) {
-  const r = openaiBase(env, {
+  const r = openaiBase({
     baseURL: 'https://api.lingyiwanwu.com/v1',
     apiKey: env.LINGYIWANWU_API_KEY,
   })

diff --git a/src/llm/moonshot.ts b/src/llm/moonshot.ts
@@ -1,7 +1,7 @@
 import { openaiBase } from './openai'
 
 export function moonshot(env: Record<string, string>) {
-  const r = openaiBase(env, {
+  const r = openaiBase({
     baseURL: 'https://api.moonshot.cn/v1',
     apiKey: env.MOONSHOT_API_KEY,
   })

diff --git a/src/llm/openai.ts b/src/llm/openai.ts
@@ -1,11 +1,7 @@
 import OpenAI, { ClientOptions } from 'openai'
 import { IChat } from './base'
 
-export function openaiBase(
-  env: Record<string, string>,
-  options?: ClientOptions,
-): IChat {
-  const client = new OpenAI(options)
+export function openaiBase(options?: ClientOptions): IChat {
   return {
     name: 'openai',
     supportModels: [
@@ -43,9 +39,11 @@ export function openaiBase(
     ],
     requiredEnv: ['OPENAI_API_KEY'],
     invoke(req) {
+      const client = new OpenAI(options)
       return client.chat.completions.create({ ...req, stream: false })
     },
     async *stream(req, signal) {
+      const client = new OpenAI(options)
       const stream = await client.chat.completions.create(
         { ...req, stream: true },
         { signal },
@@ -58,7 +56,7 @@ export function openaiBase(
 }
 
 export function openai(env: Record<string, string>): IChat {
-  return openaiBase(env, {
+  return openaiBase({
     apiKey: env.OPENAI_API_KEY,
   })
 }