diff --git a/docs/docs/syntax/client/client.mdx b/docs/docs/syntax/client/client.mdx index ec01aa7a5..c59d0866c 100644 --- a/docs/docs/syntax/client/client.mdx +++ b/docs/docs/syntax/client/client.mdx @@ -34,8 +34,8 @@ BAML ships with the following providers (you can can also write your own!): - `openai` - `azure-openai` - `anthropic` - - `ollama` - `google-ai` + - `ollama` - Composite client providers - `fallback` - `round-robin` @@ -111,6 +111,21 @@ client MyClient { } } ``` +### Google + +Provider names: +- `google-ai` + +Accepts any options as defined by the [Gemini SDK](https://ai.google.dev/gemini-api/docs/get-started/tutorial?lang=rest#configuration). + +```rust +client MyGoogleClient { + provider google-ai + options{ + model "gemini-1.5-pro-001" + } +} +``` ### Ollama @@ -135,6 +150,7 @@ client MyOllamaClient { 1. For Ollama, in your terminal run `ollama serve` 2. In another window, run `ollama run llama2` (or your model), and you should be good to go. +3. If your Ollama port is not 11434, you can specify the endpoint manually. ```rust client MyClient { @@ -143,28 +159,14 @@ client MyClient { model llama2 options { temperature 0 + base_url "http://localhost:" // Default is 11434 } } } ``` -### Google - -Provider names: -- `google-ai` - -Accepts any options as defined by the [Gemini SDK](https://ai.google.dev/gemini-api/docs/api-overview). - -```rust -client MyGoogleClient { - provider google-ai - options{ - model "gemini-1.5-pro-001" - api_key env.GOOGLE_API_KEY - } -} -``` +This is not the Vertex AI Gemini API, but the Google Generative AI Gemini API, which supports the same models but at a different endpoint. ### Fallback diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs index 82dc28cba..d70dc2644 100644 --- a/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs +++ b/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs @@ -267,19 +267,15 @@ impl RequestBuilder for GoogleClient { prompt: either::Either<&String, &Vec>, stream: bool, ) -> reqwest::RequestBuilder { - let mut should_stream = "generateContent?"; + let mut should_stream = "generateContent"; if stream { - should_stream = "streamGenerateContent?alt=sse&"; + should_stream = "streamGenerateContent?alt=sse"; } let baml_original_url = format!( - "https://generativelanguage.googleapis.com/v1/models/{}:{}key={}", + "https://generativelanguage.googleapis.com/v1/models/{}:{}", self.properties.model_id.as_ref().unwrap_or(&"".to_string()), - should_stream, - self.properties - .api_key - .clone() - .unwrap_or_else(|| "".to_string()) + should_stream ); let mut req = self.client.post( @@ -295,6 +291,13 @@ impl RequestBuilder for GoogleClient { } req = req.header("baml-original-url", baml_original_url); + req = req.header( + "x-goog-api-key", + self.properties + .api_key + .clone() + .unwrap_or_else(|| "".to_string()), + ); let mut body = json!(self.properties.properties); let body_obj = body.as_object_mut().unwrap(); diff --git a/engine/baml-runtime/src/types/expression_helper.rs b/engine/baml-runtime/src/types/expression_helper.rs index ad2385273..dd05b724b 100644 --- a/engine/baml-runtime/src/types/expression_helper.rs +++ b/engine/baml-runtime/src/types/expression_helper.rs @@ -32,7 +32,19 @@ pub fn to_value(ctx: &RuntimeContext, expr: &Expression) -> Result s.clone(), - _ => anyhow::bail!("invalid key type"), + Expression::Identifier(internal_baml_core::ir::repr::Identifier::ENV( + key, + )) => match ctx.env.get(key) { + None => anyhow::bail!("unset env variable '{}'", key), + Some(val) => val.to_string(), + }, + Expression::Identifier( + internal_baml_core::ir::repr::Identifier::Local(key), + ) => key.clone(), + Expression::Identifier(internal_baml_core::ir::repr::Identifier::Ref( + key, + )) => key.join("."), + _ => anyhow::bail!("invalid key {:#?}", k), }; let v = to_value(ctx, v)?; Ok((k, v)) diff --git a/engine/baml-runtime/src/types/runtime_context.rs b/engine/baml-runtime/src/types/runtime_context.rs index 1ae1b8a62..7a645b415 100644 --- a/engine/baml-runtime/src/types/runtime_context.rs +++ b/engine/baml-runtime/src/types/runtime_context.rs @@ -44,7 +44,11 @@ impl RuntimeContext { &self, expr: &Expression, ) -> Result { - serde_json::from_value::(super::expression_helper::to_value(self, expr)?).map_err(|e| { + match super::expression_helper::to_value(self, expr) { + Ok(v) => serde_json::from_value(v).map_err(|e| e.into()), + Err(e) => Err(e), + } + .map_err(|e| { anyhow::anyhow!( "Failed to resolve expression {:?} with error: {:?}", expr, diff --git a/engine/baml-schema-wasm/src/runtime_wasm/mod.rs b/engine/baml-schema-wasm/src/runtime_wasm/mod.rs index 834afeb8b..c4e91212e 100644 --- a/engine/baml-schema-wasm/src/runtime_wasm/mod.rs +++ b/engine/baml-schema-wasm/src/runtime_wasm/mod.rs @@ -921,7 +921,7 @@ impl WasmFunction { .render_prompt(&self.name, &ctx, ¶ms, None) .as_ref() .map(|(p, scope)| (p, scope).into()) - .map_err(|e| wasm_bindgen::JsError::new(&e.to_string())) + .map_err(|e| wasm_bindgen::JsError::new(format!("{e:?}").as_str())) } #[wasm_bindgen] diff --git a/typescript/fiddle-frontend/package.json b/typescript/fiddle-frontend/package.json index 9496d803d..7fe47ce1d 100644 --- a/typescript/fiddle-frontend/package.json +++ b/typescript/fiddle-frontend/package.json @@ -3,7 +3,7 @@ "version": "0.1.0", "private": true, "scripts": { - "dev": "infisical run --env=test -- next dev", + "dev": "next dev", "build": "next build", "start": "next start", "lint": "next lint" diff --git a/typescript/fiddle-frontend/public/_examples/clients.baml b/typescript/fiddle-frontend/public/_examples/clients.baml index 13233644a..803c87f11 100644 --- a/typescript/fiddle-frontend/public/_examples/clients.baml +++ b/typescript/fiddle-frontend/public/_examples/clients.baml @@ -1,4 +1,6 @@ -// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, and Ollama as providers but are expanding to many more. +// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, Gemini, and Ollama as providers but are expanding to many more. + +// We also support any other provider that follows the OpenAI API specification, such as HuggingFace. // For this playground, we have setup a few clients for you to use already with some free credits. @@ -36,4 +38,12 @@ client Claude { max_tokens 1000 } +} + +client Gemini { + provider google-ai + options{ + model "gemini-1.5-pro-001" + api_key env.GOOGLE_API_KEY + } } \ No newline at end of file diff --git a/typescript/fiddle-frontend/public/_examples/intro/chat-roles/baml_src/clients.baml b/typescript/fiddle-frontend/public/_examples/intro/chat-roles/baml_src/clients.baml index 13233644a..0bddbd521 100644 --- a/typescript/fiddle-frontend/public/_examples/intro/chat-roles/baml_src/clients.baml +++ b/typescript/fiddle-frontend/public/_examples/intro/chat-roles/baml_src/clients.baml @@ -1,4 +1,6 @@ -// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, and Ollama as providers but are expanding to many more. +// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, Gemini, and Ollama as providers but are expanding to many more. + +// We also support any other provider that follows the OpenAI API specification, such as HuggingFace. // For this playground, we have setup a few clients for you to use already with some free credits. @@ -36,4 +38,13 @@ client Claude { max_tokens 1000 } -} \ No newline at end of file +} + +client Gemini { + provider google-ai + options{ + model "gemini-1.5-pro-001" + api_key env.GOOGLE_API_KEY + } +} + diff --git a/typescript/fiddle-frontend/public/_examples/intro/classify-message/baml_src/clients.baml b/typescript/fiddle-frontend/public/_examples/intro/classify-message/baml_src/clients.baml index 13233644a..0bddbd521 100644 --- a/typescript/fiddle-frontend/public/_examples/intro/classify-message/baml_src/clients.baml +++ b/typescript/fiddle-frontend/public/_examples/intro/classify-message/baml_src/clients.baml @@ -1,4 +1,6 @@ -// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, and Ollama as providers but are expanding to many more. +// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, Gemini, and Ollama as providers but are expanding to many more. + +// We also support any other provider that follows the OpenAI API specification, such as HuggingFace. // For this playground, we have setup a few clients for you to use already with some free credits. @@ -36,4 +38,13 @@ client Claude { max_tokens 1000 } -} \ No newline at end of file +} + +client Gemini { + provider google-ai + options{ + model "gemini-1.5-pro-001" + api_key env.GOOGLE_API_KEY + } +} + diff --git a/typescript/fiddle-frontend/public/_examples/intro/extract-resume/baml_src/clients.baml b/typescript/fiddle-frontend/public/_examples/intro/extract-resume/baml_src/clients.baml index 13233644a..0bddbd521 100644 --- a/typescript/fiddle-frontend/public/_examples/intro/extract-resume/baml_src/clients.baml +++ b/typescript/fiddle-frontend/public/_examples/intro/extract-resume/baml_src/clients.baml @@ -1,4 +1,6 @@ -// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, and Ollama as providers but are expanding to many more. +// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, Gemini, and Ollama as providers but are expanding to many more. + +// We also support any other provider that follows the OpenAI API specification, such as HuggingFace. // For this playground, we have setup a few clients for you to use already with some free credits. @@ -36,4 +38,13 @@ client Claude { max_tokens 1000 } -} \ No newline at end of file +} + +client Gemini { + provider google-ai + options{ + model "gemini-1.5-pro-001" + api_key env.GOOGLE_API_KEY + } +} + diff --git a/typescript/fiddle-frontend/public/_examples/intro/images/baml_src/clients.baml b/typescript/fiddle-frontend/public/_examples/intro/images/baml_src/clients.baml index 13233644a..96654aa0c 100644 --- a/typescript/fiddle-frontend/public/_examples/intro/images/baml_src/clients.baml +++ b/typescript/fiddle-frontend/public/_examples/intro/images/baml_src/clients.baml @@ -1,4 +1,6 @@ -// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, and Ollama as providers but are expanding to many more. +// These are LLM clients you can use in your functions. We currently support Anthropic, OpenAI / Azure, Gemini, and Ollama as providers but are expanding to many more. + +// We also support any other provider that follows the OpenAI API specification, such as HuggingFace. // For this playground, we have setup a few clients for you to use already with some free credits. @@ -36,4 +38,12 @@ client Claude { max_tokens 1000 } -} \ No newline at end of file +} + +client Gemini { + provider google-ai + options{ + model "gemini-1.5-pro-001" + api_key env.GOOGLE_API_KEY + } +} diff --git a/typescript/fiddle-proxy/.gitignore b/typescript/fiddle-proxy/.gitignore index b512c09d4..1dcef2d9f 100644 --- a/typescript/fiddle-proxy/.gitignore +++ b/typescript/fiddle-proxy/.gitignore @@ -1 +1,2 @@ -node_modules \ No newline at end of file +node_modules +.env \ No newline at end of file diff --git a/typescript/fiddle-proxy/package-lock.json b/typescript/fiddle-proxy/package-lock.json index 486b3e150..f1493d93b 100644 --- a/typescript/fiddle-proxy/package-lock.json +++ b/typescript/fiddle-proxy/package-lock.json @@ -10,11 +10,13 @@ "license": "ISC", "dependencies": { "cors": "^2.8.5", + "dotenv": "16.4.5", "express": "^4.19.2", "http-proxy-middleware": "^3.0.0" }, "devDependencies": { - "@flydotio/node-demo": "^0.2.1" + "@flydotio/node-demo": "^0.2.1", + "cors": "^2.8.5" } }, "node_modules/@alloc/quick-lru": { @@ -547,6 +549,7 @@ "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "dev": true, "dependencies": { "object-assign": "^4", "vary": "^1" @@ -643,6 +646,18 @@ "integrity": "sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==", "dev": true }, + "node_modules/dotenv": { + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -1501,6 +1516,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, "engines": { "node": ">=0.10.0" } diff --git a/typescript/fiddle-proxy/package.json b/typescript/fiddle-proxy/package.json index 798dd511b..ef2dbb370 100644 --- a/typescript/fiddle-proxy/package.json +++ b/typescript/fiddle-proxy/package.json @@ -11,11 +11,14 @@ "author": "", "license": "ISC", "devDependencies": { - "@flydotio/node-demo": "^0.2.1" + "@flydotio/node-demo": "^0.2.1", + "cors": "^2.8.5" + }, "dependencies": { "cors": "^2.8.5", "express": "^4.19.2", + "dotenv": "16.4.5", "http-proxy-middleware": "^3.0.0" } } \ No newline at end of file diff --git a/typescript/fiddle-proxy/server.js b/typescript/fiddle-proxy/server.js index 5b72e893d..2df200cec 100644 --- a/typescript/fiddle-proxy/server.js +++ b/typescript/fiddle-proxy/server.js @@ -1,12 +1,20 @@ const cors = require('cors') const { createProxyMiddleware } = require('http-proxy-middleware') const app = require('express')() +require('dotenv').config() app.use(cors()) app.use( createProxyMiddleware({ changeOrigin: true, + pathRewrite: (path, req) => { + // Ensure the URL does not end with a slash + if (path.endsWith('/')) { + return path.slice(0, -1) + } + return path + }, router: (req) => { // Extract the original target URL from the custom header const originalUrl = req.headers['baml-original-url'] @@ -32,6 +40,12 @@ app.use( } proxyReq.setHeader('x-api-key', process.env.ANTHROPIC_API_KEY) } + if (req.headers['baml-original-url'].includes('gemini')) { + if (process.env.GOOGLE_API_KEY === undefined) { + throw new Error('GOOGLE_API_KEY is missing') + } + proxyReq.setHeader('x-goog-api-key', process.env.GOOGLE_API_KEY) + } }, proxyRes: (proxyRes, req, res) => { proxyRes.headers['Access-Control-Allow-Origin'] = '*'