diff --git a/README.md b/README.md index 143a93d..70869f3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -Provides the same agent OpenAI API interface for different LLM models and supports deployment to any Edge Runtime environment. +Provides the same proxy OpenAI API interface for different LLM models, and supports deployment to any Edge Runtime environment. Supported models @@ -18,25 +18,25 @@ Supported models Environment variables -- `API_KEY`: proxy API Key, which must be set when calling the proxy API +- `API_KEY`: Proxy API Key, required when calling the proxy API -- OpenAI: Support OpenAI models, such as `gpt-4o-mini` +- OpenAI: Supports OpenAI models, e.g. `gpt-4o-mini` - `OPENAI_API_KEY`: OpenAI API Key -- VertexAI Anthropic: Support Anthropic models on Google Vertex AI, such as `claude-3-5-sonnet@20240620` +- VertexAI Anthropic: Supports Anthropic models on Google Vertex AI, e.g. `claude-3-5-sonnet@20240620` - `VERTEX_ANTROPIC_GOOGLE_SA_CLIENT_EMAIL`: Google Cloud Service Account Email - `VERTEX_ANTROPIC_GOOGLE_SA_PRIVATE_KEY`: Google Cloud Service Account Private Key - `VERTEX_ANTROPIC_REGION`: Google Vertex AI Anthropic Region - `VERTEX_ANTROPIC_PROJECTID`: Google Vertex AI Anthropic Project ID -- Anthropic: Support Anthropic models, such as `claude-3-5-sonnet-20240620` +- Anthropic: Supports Anthropic models, e.g. `claude-3-5-sonnet-20240620` - `ANTROPIC_API_KEY`: Anthropic API Key -- Google Gemini: Support Google Gemini models, such as `gemini-1.5-flash` +- Google Gemini: Supports Google Gemini models, e.g. `gemini-1.5-flash` - `GOOGLE_GEN_AI_API_KEY`: Google Gemini API Key ## Usage -Once deployed successfully, different models can be called through OpenAI’s API interface. +Once deployed successfully, you can call different models through OpenAI's API interface. -For example, calling OpenAI’s API interface: +For example, calling OpenAI's API interface: ```bash curl http://localhost:8787/v1/chat/completions \ @@ -53,7 +53,7 @@ curl http://localhost:8787/v1/chat/completions \ }' ``` -Or call Anthropic’s API interface: +Or calling Anthropic's API interface: ```bash curl http://localhost:8787/v1/chat/completions \ @@ -70,7 +70,7 @@ curl http://localhost:8787/v1/chat/completions \ }' ``` -And can be used in OpenAI’s official SDK, for example: +And it can be used in OpenAI's official SDK, for example: ```ts const openai = new OpenAI({ diff --git a/package.json b/package.json index b13e6d6..a032015 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,8 @@ "type": "module", "scripts": { "dev": "wrangler dev src/index.ts", - "deploy": "wrangler deploy --minify src/index.ts" + "deploy": "wrangler deploy --minify src/index.ts", + "test": "vitest run" }, "dependencies": { "@anthropic-ai/sdk": "^0.27.0", diff --git a/src/__tests__/index.test.ts b/src/__tests__/index.test.ts index 1316e21..755e4b3 100644 --- a/src/__tests__/index.test.ts +++ b/src/__tests__/index.test.ts @@ -1,11 +1,12 @@ import { beforeAll, beforeEach, describe, expect, it } from 'vitest' import app from '..' import OpenAI from 'openai' -import { omit } from 'lodash-es' +import { omit, pick } from 'lodash-es' import { AnthropicVertex } from '@anthropic-ai/vertex-sdk' import { GoogleAuth } from 'google-auth-library' import { TextBlock } from '@anthropic-ai/sdk/resources/messages.mjs' import { anthropic } from '../llm/anthropic' +import { testClient } from 'hono/testing' const MOCK_ENV = { API_KEY: import.meta.env.VITE_API_KEY, @@ -196,7 +197,13 @@ describe('stream', () => { describe('list models', async () => { it('no api key', async () => { const r = (await ( - await app.request('/v1/models', {}, {}) + await app.request( + '/v1/models', + { + headers: { Authorization: 'Bearer ' + import.meta.env.VITE_API_KEY }, + }, + pick(MOCK_ENV, 'API_KEY'), + ) ).json()) as OpenAI.Models.ModelsPage expect(r.data).empty }) @@ -204,18 +211,15 @@ describe('list models', async () => { const r = (await ( await app.request( '/v1/models', - {}, - { ANTROPIC_API_KEY: import.meta.env.VITE_ANTROPIC_API_KEY }, + { + headers: { + Authorization: 'Bearer ' + import.meta.env.VITE_API_KEY, + }, + }, + pick(MOCK_ENV, 'API_KEY', 'ANTROPIC_API_KEY'), ) ).json()) as OpenAI.Models.ModelsPage - expect(r.data).deep.eq( - anthropic({} as any).supportModels.map((it) => ({ - id: it, - object: 'model', - owned_by: 'system', - created: 0, - })), - ) + expect(r.data.map((it) => it.id)).deep.eq(anthropic(MOCK_ENV).supportModels) }) it('openai api key', async () => { const client = new OpenAI({ @@ -225,6 +229,6 @@ describe('list models', async () => { }, }) const r = await client.models.list() - console.log(r) + expect(r.data).not.empty }) }) diff --git a/src/claude/__tests__/authenticate.test.ts b/src/claude/__tests__/authenticate.test.ts index b242d39..649c39c 100644 --- a/src/claude/__tests__/authenticate.test.ts +++ b/src/claude/__tests__/authenticate.test.ts @@ -4,16 +4,16 @@ import { it, expect } from 'vitest' it('should authenticate', async () => { const token = await authenticate({ - clientEmail: process.env.VITE_GOOGLE_SA_CLIENT_EMAIL!, - privateKey: process.env.VITE_GOOGLE_SA_PRIVATE_KEY!, + clientEmail: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_CLIENT_EMAIL!, + privateKey: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_PRIVATE_KEY!, }) expect(token).toBeDefined() }) it('should call anthropic vertex', async () => { const token = await authenticate({ - clientEmail: process.env.VITE_GOOGLE_SA_CLIENT_EMAIL!, - privateKey: process.env.VITE_GOOGLE_SA_PRIVATE_KEY!, + clientEmail: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_CLIENT_EMAIL!, + privateKey: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_PRIVATE_KEY!, }) const client = new AnthropicVertexWeb({ accessToken: token.access_token, @@ -42,8 +42,8 @@ function buildUrl({ it('should authenticate on edge runtime', async () => { const token = await authenticate({ - clientEmail: process.env.VITE_GOOGLE_SA_CLIENT_EMAIL!, - privateKey: process.env.VITE_GOOGLE_SA_PRIVATE_KEY!, + clientEmail: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_CLIENT_EMAIL!, + privateKey: import.meta.env.VITE_VERTEX_ANTROPIC_GOOGLE_SA_PRIVATE_KEY!, }) expect(token).toBeDefined() const url = buildUrl({ diff --git a/src/claude/__tests__/AnthropicVertex.test.ts b/src/claude/__tests__/web.test.ts similarity index 96% rename from src/claude/__tests__/AnthropicVertex.test.ts rename to src/claude/__tests__/web.test.ts index 184fa2e..0679307 100644 --- a/src/claude/__tests__/AnthropicVertex.test.ts +++ b/src/claude/__tests__/web.test.ts @@ -1,6 +1,9 @@ import { expect, it } from 'vitest' import { AnthropicVertexWeb } from '../web' -import { AnthropicVertex, AnthropicVertex as OriginAnthropicVertex } from '@anthropic-ai/vertex-sdk' +import { + AnthropicVertex, + AnthropicVertex as OriginAnthropicVertex, +} from '@anthropic-ai/vertex-sdk' import { authenticate } from '../authenticate' import { omit } from 'lodash-es' import { ChatAnthropic } from 'langchain-anthropic-edge' @@ -53,7 +56,7 @@ it('should call Custom Client and Origin Client', async () => { expect(r1).not.undefined expect(r2).not.undefined expect(omit(r1, 'id')).deep.equal(omit(r2, 'id')) -}, 10_000) +}, 20_000) it('Call Anthropic Vertex on nodejs', async () => { const client = new AnthropicVertex({ diff --git a/src/claude/authenticate.ts b/src/claude/authenticate.ts index c20d903..e0e12f2 100644 --- a/src/claude/authenticate.ts +++ b/src/claude/authenticate.ts @@ -62,6 +62,9 @@ export async function authenticate(options: { clientEmail: string privateKey: string }): Promise { + if (!options.clientEmail || !options.privateKey) { + throw new Error('clientEmail and privateKey are required') + } if (token === null) { token = await createToken(options) } else if (token.expires_at < Math.floor(Date.now() / 1000)) { diff --git a/src/llm/anthropic.ts b/src/llm/anthropic.ts index 246a8a7..b2014b9 100644 --- a/src/llm/anthropic.ts +++ b/src/llm/anthropic.ts @@ -149,7 +149,6 @@ export function anthropicBase( signal.onabort = () => stream.controller.abort() const chunks: AnthropicTypes.Messages.RawMessageStreamEvent[] = [] let start: AnthropicTypes.Messages.Message | undefined - let delta: AnthropicTypes.Messages.MessageDeltaEvent | undefined for await (const it of stream) { chunks.push(it) const fileds = () => ({ @@ -176,28 +175,29 @@ export function anthropicBase( ], } as OpenAI.ChatCompletionChunk } else if (it.type === 'message_delta') { - delta = it - yield { - ...fileds(), - choices: [ - { - index: 0, - delta: {}, - finish_reason: 'stop', + if (req.stream_options?.include_usage) { + yield { + ...fileds(), + choices: [], + usage: { + prompt_tokens: start!.usage.input_tokens, + completion_tokens: it!.usage.output_tokens, + total_tokens: + start!.usage.input_tokens + it!.usage.output_tokens, }, - ], - } as OpenAI.ChatCompletionChunk - } else if (it.type === 'message_stop') { - yield { - ...fileds(), - choices: [], - usage: { - prompt_tokens: start!.usage.input_tokens, - completion_tokens: delta!.usage.output_tokens, - total_tokens: - start!.usage.input_tokens + delta!.usage.output_tokens, - }, - } as OpenAI.ChatCompletionChunk + } as OpenAI.ChatCompletionChunk + } else { + yield { + ...fileds(), + choices: [ + { + index: 0, + delta: {}, + finish_reason: 'stop', + }, + ], + } as OpenAI.ChatCompletionChunk + } } } }, diff --git a/src/llm/google.ts b/src/llm/google.ts index b2ce356..b613e5f 100644 --- a/src/llm/google.ts +++ b/src/llm/google.ts @@ -1,5 +1,5 @@ import { IChat } from './base' -import { GoogleGenerativeAI, Tool } from '@google/generative-ai' +import { GoogleGenerativeAI } from '@google/generative-ai' import GoogleAI from '@google/generative-ai' import OpenAI from 'openai' @@ -128,7 +128,11 @@ export function google(env: Record): IChat { ], } as OpenAI.ChatCompletionChunk } - if (req.stream_options?.include_usage && last) { + if (!last) { + throw new Error('No response from google') + } + + if (req.stream_options?.include_usage) { yield { ...fields(), choices: [], @@ -138,6 +142,17 @@ export function google(env: Record): IChat { total_tokens: last.usageMetadata!.totalTokenCount, }, } as OpenAI.ChatCompletionChunk + } else { + yield { + ...fields(), + choices: [ + { + index: 0, + delta: {}, + finish_reason: 'stop', + }, + ], + } as OpenAI.ChatCompletionChunk } }, }