The maximum number of tokens to allow in the generated response. The default value is the maximum allowed value for the model that you are using. For more information, see Inference parameters for foundation models.
+ max_tokens: ::std::option::Option,
+ /// The likelihood of the model selecting higher-probability options while generating a response. A lower value makes the model more likely to choose higher-probability options, while a higher value makes the model more likely to choose lower-probability options.
+ /// The default value is the default value for the model that you are using. For more information, see Inference parameters for foundation models.
+ temperature: ::std::option::Option,
+ /// The percentage of most-likely candidates that the model considers for the next token. For example, if you choose a value of 0.8 for topP, the model selects from the top 80% of the probability distribution of tokens that could be next in the sequence.
+ /// The default value is the default value for the model that you are using. For more information, see Inference parameters for foundation models.
+ top_p: ::std::option::Option,
+ /// A list of stop sequences. A stop sequence is a sequence of characters that causes the model to stop generating the response.
+ stop_sequences: ::std::option::Option<::std::vec::Vec<::std::string::String>>,
+}
+
+impl Into for InferenceConfiguration {
+ fn into(self) -> bedrock::types::InferenceConfiguration {
+ bedrock::types::InferenceConfiguration::builder()
+ .set_max_tokens(self.max_tokens)
+ .set_temperature(self.temperature)
+ .set_top_p(self.top_p)
+ .set_stop_sequences(self.stop_sequences)
+ .build()
+ }
+}
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/aws/wasm.rs b/engine/baml-runtime/src/internal/llm_client/primitive/aws/wasm.rs
new file mode 100644
index 000000000..76ace6b35
--- /dev/null
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/aws/wasm.rs
@@ -0,0 +1,148 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use aws_config::ConfigLoader;
+use aws_smithy_async::{
+ rt::sleep::{AsyncSleep, Sleep},
+ time::TimeSource,
+};
+use aws_smithy_runtime_api::client::result::{ConnectorError, SdkError};
+use aws_smithy_runtime_api::http::{self, Request};
+use aws_smithy_runtime_api::{
+ client::{
+ http::{
+ HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings,
+ SharedHttpConnector,
+ },
+ orchestrator::HttpRequest,
+ runtime_components::RuntimeComponents,
+ },
+ shared::IntoShared,
+};
+use aws_smithy_types::body::SdkBody;
+
+use aws_config::{BehaviorVersion, SdkConfig};
+use core::pin::Pin;
+use core::task::{Context, Poll};
+use futures::Stream;
+use pin_project_lite::pin_project;
+use std::sync::Arc;
+use std::time::SystemTime;
+
+pub fn load_aws_config() -> ConfigLoader {
+ aws_config::defaults(BehaviorVersion::latest())
+ .sleep_impl(BrowserSleep)
+ .time_source(BrowserTime)
+ .http_client(BrowserHttp2::new())
+}
+
+#[derive(Debug)]
+struct BrowserTime;
+impl TimeSource for BrowserTime {
+ fn now(&self) -> SystemTime {
+ let offset = web_time::SystemTime::now()
+ .duration_since(web_time::UNIX_EPOCH)
+ .unwrap();
+ std::time::UNIX_EPOCH + offset
+ }
+}
+
+#[derive(Debug, Clone)]
+struct BrowserSleep;
+impl AsyncSleep for BrowserSleep {
+ fn sleep(&self, duration: std::time::Duration) -> Sleep {
+ Sleep::new(futures_timer::Delay::new(duration))
+ }
+}
+
+pin_project! {
+ struct StreamWrapper {
+ #[pin]
+ resp: S,
+ }
+}
+
+// These are lies, but JsFuture is only !Send because of web workers, so this is
+// safe in the web panel: https://github.com/rustwasm/wasm-bindgen/issues/2833
+unsafe impl Send for StreamWrapper {}
+unsafe impl Sync for StreamWrapper {}
+
+impl>> http_body::Body for StreamWrapper {
+ type Data = bytes::Bytes;
+ type Error = reqwest::Error;
+
+ fn poll_frame(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ ) -> Poll, Self::Error>>> {
+ let resp = self.project().resp;
+
+ let Poll::Ready(chunk) = resp.poll_next(cx) else {
+ return Poll::Pending;
+ };
+ Poll::Ready(match chunk {
+ Some(Ok(chunk_bytes)) => Some(Ok(http_body::Frame::data(chunk_bytes))),
+ Some(Err(e)) => Some(Err(e)),
+ None => None,
+ })
+ }
+}
+
+#[derive(Debug, Clone)]
+struct BrowserHttp2 {
+ client: Arc,
+}
+
+impl BrowserHttp2 {
+ pub fn new() -> Self {
+ Self {
+ client: Arc::new(reqwest::Client::new()),
+ }
+ }
+
+ async fn send3(&self, smithy_req: Request) -> Result, ConnectorError> {
+ let method = match reqwest::Method::from_bytes(smithy_req.method().as_bytes()) {
+ Ok(method) => method,
+ Err(e) => return Err(ConnectorError::user(Box::new(e))),
+ };
+ let mut req = self.client.request(method, smithy_req.uri());
+
+ for (k, v) in smithy_req.headers() {
+ req = req.header(k, v);
+ }
+
+ if let Some(body) = smithy_req.body().bytes() {
+ req = req.body(Vec::from(body));
+ }
+
+ match req.send().await {
+ Ok(resp) => Ok(http::Response::new(
+ resp.status().into(),
+ SdkBody::from_body_1_x(StreamWrapper {
+ resp: resp.bytes_stream(),
+ }),
+ )),
+ Err(e) => Err(ConnectorError::other(Box::new(e), None)),
+ }
+ }
+}
+
+impl HttpConnector for BrowserHttp2 {
+ fn call(&self, req: HttpRequest) -> HttpConnectorFuture {
+ let clone = self.clone();
+
+ HttpConnectorFuture::new(
+ async move { send_wrapper::SendWrapper::new(clone.send3(req)).await },
+ )
+ }
+}
+
+impl HttpClient for BrowserHttp2 {
+ fn http_connector(
+ &self,
+ _settings: &HttpConnectorSettings,
+ _components: &RuntimeComponents,
+ ) -> SharedHttpConnector {
+ self.clone().into_shared()
+ }
+}
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs
index 92e346cc4..17c3bc028 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/google/google_client.rs
@@ -16,18 +16,18 @@ use crate::{
request::create_client,
};
use anyhow::{Context, Result};
-use baml_types::BamlImage;
+use baml_types::BamlMedia;
use eventsource_stream::Eventsource;
use futures::StreamExt;
use internal_baml_core::ir::ClientWalker;
use internal_baml_jinja::{ChatMessagePart, RenderContext_Client, RenderedChatMessage};
-use reqwest::Response;
use serde_json::json;
use std::collections::HashMap;
struct PostRequestProperities {
default_role: String,
api_key: Option,
headers: HashMap,
+ base_url: String,
proxy_url: Option,
model_id: Option,
properties: HashMap,
@@ -61,6 +61,11 @@ fn resolve_properties(
.and_then(|v| v.as_str().map(|s| s.to_string()))
.or_else(|| Some("gemini-1.5-flash".to_string()));
+ let base_url = properties
+ .remove("base_url")
+ .and_then(|v| v.as_str().map(|s| s.to_string()))
+ .unwrap_or_else(|| "https://generativelanguage.googleapis.com/v1".to_string());
+
let headers = properties.remove("headers").map(|v| {
if let Some(v) = v.as_object() {
v.iter()
@@ -89,6 +94,7 @@ fn resolve_properties(
api_key,
headers,
properties,
+ base_url,
model_id,
proxy_url: ctx.env.get("BOUNDARY_PROXY_URL").map(|s| s.to_string()),
})
@@ -127,7 +133,7 @@ impl SseResponseTrait for GoogleClient {
Ok(Box::pin(
resp.bytes_stream()
.eventsource()
- .inspect(|event| log::info!("Received event: {:?}", event))
+ .inspect(|event| log::trace!("Received event: {:?}", event))
.take_while(|event| {
std::future::ready(event.as_ref().is_ok_and(|e| e.data != "data: \n"))
})
@@ -142,7 +148,7 @@ impl SseResponseTrait for GoogleClient {
start_time: system_start,
latency: instant_start.elapsed(),
model: model_id,
- invocation_params: params.clone(),
+ request_options: params.clone(),
metadata: LLMCompleteResponseMetadata {
baml_is_complete: false,
finish_reason: None,
@@ -171,7 +177,7 @@ impl SseResponseTrait for GoogleClient {
prompt.clone(),
),
start_time: system_start,
- invocation_params: params.clone(),
+ request_options: params.clone(),
latency: instant_start.elapsed(),
message: format!("Failed to parse event: {:#?}", e),
code: ErrorCode::Other(2),
@@ -205,7 +211,7 @@ impl SseResponseTrait for GoogleClient {
impl WithStreamChat for GoogleClient {
async fn stream_chat(
&self,
- ctx: &RuntimeContext,
+ _ctx: &RuntimeContext,
prompt: &Vec,
) -> StreamResponse {
//incomplete, streaming response object is returned
@@ -221,17 +227,21 @@ impl WithStreamChat for GoogleClient {
impl GoogleClient {
pub fn new(client: &ClientWalker, ctx: &RuntimeContext) -> Result {
let properties = super::super::resolve_properties_walker(client, ctx)?;
+ let properties = resolve_properties(properties, ctx)?;
+ let default_role = properties.default_role.clone();
Ok(Self {
name: client.name().into(),
- properties: resolve_properties(properties, ctx)?,
+ properties,
context: RenderContext_Client {
name: client.name().into(),
provider: client.elem().provider.clone(),
+ default_role,
},
features: ModelFeatures {
chat: true,
completion: false,
anthropic_system_constraints: false,
+ resolve_media_urls: true,
},
retry_policy: client
.elem()
@@ -243,24 +253,29 @@ impl GoogleClient {
}
pub fn dynamic_new(client: &ClientProperty, ctx: &RuntimeContext) -> Result {
+ let properties = resolve_properties(
+ client
+ .options
+ .iter()
+ .map(|(k, v)| Ok((k.clone(), json!(v))))
+ .collect::>>()?,
+ ctx,
+ )?;
+ let default_role = properties.default_role.clone();
+
Ok(Self {
name: client.name.clone(),
- properties: resolve_properties(
- client
- .options
- .iter()
- .map(|(k, v)| Ok((k.clone(), json!(v))))
- .collect::>>()?,
- ctx,
- )?,
+ properties,
context: RenderContext_Client {
name: client.name.clone(),
provider: client.provider.clone(),
+ default_role,
},
features: ModelFeatures {
chat: true,
completion: false,
anthropic_system_constraints: false,
+ resolve_media_urls: true,
},
retry_policy: client.retry_policy.clone(),
client: create_client()?,
@@ -273,18 +288,19 @@ impl RequestBuilder for GoogleClient {
&self.client
}
- fn build_request(
+ async fn build_request(
&self,
prompt: either::Either<&String, &Vec>,
stream: bool,
- ) -> reqwest::RequestBuilder {
+ ) -> Result {
let mut should_stream = "generateContent";
if stream {
should_stream = "streamGenerateContent?alt=sse";
}
let baml_original_url = format!(
- "https://generativelanguage.googleapis.com/v1/models/{}:{}",
+ "{}/models/{}:{}",
+ self.properties.base_url,
self.properties.model_id.as_ref().unwrap_or(&"".to_string()),
should_stream
);
@@ -301,7 +317,8 @@ impl RequestBuilder for GoogleClient {
req = req.header(key, value);
}
- req = req.header("baml-original-url", baml_original_url);
+ req = req.header("baml-original-url", baml_original_url.clone());
+ req = req.header("baml-render-url", baml_original_url);
req = req.header(
"x-goog-api-key",
self.properties
@@ -312,20 +329,18 @@ impl RequestBuilder for GoogleClient {
let mut body = json!(self.properties.properties);
let body_obj = body.as_object_mut().unwrap();
-
match prompt {
either::Either::Left(prompt) => {
body_obj.extend(convert_completion_prompt_to_body(prompt))
}
either::Either::Right(messages) => {
- body_obj.extend(convert_chat_prompt_to_body(messages))
+ body_obj.extend(convert_chat_prompt_to_body(messages));
}
}
- req.json(&body)
+ Ok(req.json(&body))
}
-
- fn invocation_params(&self) -> &HashMap {
+ fn request_options(&self) -> &HashMap {
&self.properties.properties
}
}
@@ -354,7 +369,7 @@ impl WithChat for GoogleClient {
model: None,
prompt: internal_baml_jinja::RenderedPrompt::Chat(prompt.clone()),
start_time: system_now,
- invocation_params: self.properties.properties.clone(),
+ request_options: self.properties.properties.clone(),
latency: instant_now.elapsed(),
message: format!(
"Expected exactly one content block, got {}",
@@ -370,7 +385,7 @@ impl WithChat for GoogleClient {
content: response.candidates[0].content.parts[0].text.clone(),
start_time: system_now,
latency: instant_now.elapsed(),
- invocation_params: self.properties.properties.clone(),
+ request_options: self.properties.properties.clone(),
model: self
.properties
.properties
@@ -387,14 +402,13 @@ impl WithChat for GoogleClient {
.finish_reason
.as_ref()
.map(|r| serde_json::to_string(r).unwrap_or("".into())),
- prompt_tokens: Some(response.usage_metadata.prompt_token_count),
- output_tokens: Some(response.usage_metadata.candidates_token_count),
- total_tokens: Some(response.usage_metadata.total_token_count),
+ prompt_tokens: response.usage_metadata.prompt_token_count,
+ output_tokens: response.usage_metadata.candidates_token_count,
+ total_tokens: response.usage_metadata.total_token_count,
},
})
}
}
-
//simple, Map with key "prompt" and value of the prompt string
fn convert_completion_prompt_to_body(prompt: &String) -> HashMap {
let mut map = HashMap::new();
@@ -437,20 +451,20 @@ fn convert_message_parts_to_content(parts: &Vec) -> serde_json:
ChatMessagePart::Text(text) => json!({
"text": text
}),
- ChatMessagePart::Image(image) => match image {
- BamlImage::Base64(image) => json!({
- "inlineDATA": {
- "mimeType": image.media_type,
- "data": image.base64
- }
- }),
- BamlImage::Url(image) => json!({
- "fileData": {
- "type": "url",
- "url": image.url
- }
- }),
- },
+ ChatMessagePart::Image(image) => convert_media_to_content(image, "image"),
+ ChatMessagePart::Audio(audio) => convert_media_to_content(audio, "audio"),
})
.collect()
}
+
+fn convert_media_to_content(media: &BamlMedia, media_type: &str) -> serde_json::Value {
+ match media {
+ BamlMedia::Base64(_, data) => json!({
+ "inlineData": {
+ "mimeType": format!("{}", data.media_type),
+ "data": data.base64
+ }
+ }),
+ _ => panic!("Unsupported media type"),
+ }
+}
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/google/types.rs b/engine/baml-runtime/src/internal/llm_client/primitive/google/types.rs
index 85c59db95..c3a5c983c 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/google/types.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/google/types.rs
@@ -364,9 +364,9 @@ pub struct SearchEntryPoint {
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct UsageMetaData {
- pub prompt_token_count: u64,
- pub candidates_token_count: u64,
- pub total_token_count: u64,
+ pub prompt_token_count: Option,
+ pub candidates_token_count: Option,
+ pub total_token_count: Option,
}
#[cfg(test)]
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs b/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs
index e15d35b86..17f5de130 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/mod.rs
@@ -10,7 +10,8 @@ use crate::{
};
use self::{
- anthropic::AnthropicClient, google::GoogleClient, openai::OpenAIClient, request::RequestBuilder,
+ anthropic::AnthropicClient, aws::AwsClient, google::GoogleClient, openai::OpenAIClient,
+ request::RequestBuilder,
};
use super::{
@@ -18,22 +19,59 @@ use super::{
ExecutionScope, IterOrchestrator, OrchestrationScope, OrchestrationState, OrchestratorNode,
OrchestratorNodeIterator,
},
- retry_policy::CallablePolicy,
- traits::{WithClient, WithPrompt, WithRetryPolicy, WithSingleCallable, WithStreamable},
+ traits::{
+ WithClient, WithPrompt, WithRenderRawCurl, WithRetryPolicy, WithSingleCallable,
+ WithStreamable,
+ },
LLMResponse,
};
mod anthropic;
+mod aws;
mod google;
mod openai;
pub(super) mod request;
+// use crate::internal::llm_client::traits::ambassador_impl_WithRenderRawCurl;
+// use crate::internal::llm_client::traits::ambassador_impl_WithRetryPolicy;
+use enum_dispatch::enum_dispatch;
+
+#[enum_dispatch(WithRetryPolicy)]
+pub enum LLMPrimitive2 {
+ OpenAIClient,
+ AnthropicClient,
+ GoogleClient,
+ AwsClient,
+}
+
+// #[derive(Delegate)]
+// #[delegate(WithRetryPolicy, WithRenderRawCurl)]
pub enum LLMPrimitiveProvider {
OpenAI(OpenAIClient),
Anthropic(AnthropicClient),
Google(GoogleClient),
+ Aws(aws::AwsClient),
}
+// impl WithRetryPolicy for LLMPrimitiveProvider {
+// fn retry_policy_name(&self) -> Option<&str> {
+// match self {
+// LLMPrimitiveProvider::OpenAI(client) => {
+// LLMPrimitive2::OpenAIClient(client).retry_policy_name()
+// }
+// LLMPrimitiveProvider::Anthropic(client) => {
+// LLMPrimitive2::AnthropicClient(client).retry_policy_name()
+// }
+// LLMPrimitiveProvider::Google(client) => {
+// LLMPrimitive2::GoogleClient(client).retry_policy_name()
+// }
+// LLMPrimitiveProvider::Aws(client) => {
+// LLMPrimitive2::AwsClient(client).retry_policy_name()
+// }
+// }
+// }
+// }
+
macro_rules! match_llm_provider {
// Define the variants inside the macro
($self:expr, $method:ident, async $(, $args:tt)*) => {
@@ -41,6 +79,7 @@ macro_rules! match_llm_provider {
LLMPrimitiveProvider::OpenAI(client) => client.$method($($args),*).await,
LLMPrimitiveProvider::Anthropic(client) => client.$method($($args),*).await,
LLMPrimitiveProvider::Google(client) => client.$method($($args),*).await,
+ LLMPrimitiveProvider::Aws(client) => client.$method($($args),*).await,
}
};
@@ -49,6 +88,7 @@ macro_rules! match_llm_provider {
LLMPrimitiveProvider::OpenAI(client) => client.$method($($args),*),
LLMPrimitiveProvider::Anthropic(client) => client.$method($($args),*),
LLMPrimitiveProvider::Google(client) => client.$method($($args),*),
+ LLMPrimitiveProvider::Aws(client) => client.$method($($args),*),
}
};
}
@@ -107,6 +147,7 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for LLMPrimitiveProvider {
OpenAIClient::new_ollama(client, ctx).map(LLMPrimitiveProvider::OpenAI)
}
"google-ai" => GoogleClient::new(client, ctx).map(LLMPrimitiveProvider::Google),
+ "aws-bedrock" => aws::AwsClient::new(client, ctx).map(LLMPrimitiveProvider::Aws),
other => {
let options = [
"openai",
@@ -116,6 +157,7 @@ impl TryFrom<(&ClientWalker<'_>, &RuntimeContext)> for LLMPrimitiveProvider {
"azure-openai",
"fallback",
"round-robin",
+ "aws-bedrock",
];
anyhow::bail!(
"Unsupported provider: {}. Available ones are: {}",
@@ -139,9 +181,14 @@ impl<'ir> WithPrompt<'ir> for LLMPrimitiveProvider {
}
}
-impl WithRetryPolicy for LLMPrimitiveProvider {
- fn retry_policy_name(&self) -> Option<&str> {
- match_llm_provider!(self, retry_policy_name)
+impl WithRenderRawCurl for LLMPrimitiveProvider {
+ async fn render_raw_curl(
+ &self,
+ ctx: &RuntimeContext,
+ prompt: &Vec,
+ stream: bool,
+ ) -> Result {
+ match_llm_provider!(self, render_raw_curl, async, ctx, prompt, stream)
}
}
@@ -186,6 +233,7 @@ impl std::fmt::Display for LLMPrimitiveProvider {
LLMPrimitiveProvider::OpenAI(_) => write!(f, "OpenAI"),
LLMPrimitiveProvider::Anthropic(_) => write!(f, "Anthropic"),
LLMPrimitiveProvider::Google(_) => write!(f, "Google"),
+ LLMPrimitiveProvider::Aws(_) => write!(f, "AWS"),
}
}
}
@@ -194,23 +242,9 @@ impl LLMPrimitiveProvider {
pub fn name(&self) -> &str {
&match_llm_provider!(self, context).name
}
-}
-
-impl RequestBuilder for LLMPrimitiveProvider {
- fn http_client(&self) -> &reqwest::Client {
- match_llm_provider!(self, http_client)
- }
- fn invocation_params(&self) -> &std::collections::HashMap {
- match_llm_provider!(self, invocation_params)
- }
-
- fn build_request(
- &self,
- prompt: either::Either<&String, &Vec>,
- stream: bool,
- ) -> reqwest::RequestBuilder {
- match_llm_provider!(self, build_request, prompt, stream)
+ pub fn request_options(&self) -> &std::collections::HashMap {
+ match_llm_provider!(self, request_options)
}
}
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/mod.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/mod.rs
index 6bbd188bb..0c046b927 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/mod.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/mod.rs
@@ -1,5 +1,6 @@
mod openai_client;
mod properties;
+#[allow(dead_code)]
mod types;
pub use openai_client::OpenAIClient;
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs
index b1ff59ed9..1c69630d2 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/openai_client.rs
@@ -1,7 +1,7 @@
use std::collections::HashMap;
-use anyhow::{Context, Result};
-use baml_types::BamlImage;
+use anyhow::Result;
+use baml_types::{BamlMedia, BamlMediaType};
use internal_baml_core::ir::ClientWalker;
use internal_baml_jinja::{ChatMessagePart, RenderContext_Client, RenderedChatMessage};
@@ -11,9 +11,7 @@ use crate::client_builder::ClientProperty;
use crate::internal::llm_client::primitive::request::{
make_parsed_request, make_request, RequestBuilder,
};
-use crate::internal::llm_client::traits::{
- SseResponseTrait, StreamResponse, WithCompletion, WithStreamChat,
-};
+use crate::internal::llm_client::traits::{SseResponseTrait, StreamResponse, WithStreamChat};
use crate::internal::llm_client::{
traits::{WithChat, WithClient, WithNoCompletion, WithRetryPolicy},
LLMResponse, ModelFeatures,
@@ -81,7 +79,7 @@ impl WithNoCompletion for OpenAIClient {}
// prompt: internal_baml_jinja::RenderedPrompt::Completion(prompt.clone()),
// start_time: system_start,
// latency: instant_start.elapsed(),
-// invocation_params: self.properties.properties.clone(),
+// request_options: self.properties.properties.clone(),
// message: format!(
// "Expected exactly one choices block, got {}",
// response.choices.len()
@@ -99,7 +97,7 @@ impl WithNoCompletion for OpenAIClient {}
// start_time: system_start,
// latency: instant_start.elapsed(),
// model: response.model,
-// invocation_params: self.properties.properties.clone(),
+// request_options: self.properties.properties.clone(),
// metadata: LLMCompleteResponseMetadata {
// baml_is_complete: match response.choices.get(0) {
// Some(c) => match c.finish_reason {
@@ -151,7 +149,7 @@ impl WithChat for OpenAIClient {
prompt: internal_baml_jinja::RenderedPrompt::Chat(prompt.clone()),
start_time: system_start,
latency: instant_start.elapsed(),
- invocation_params: self.properties.properties.clone(),
+ request_options: self.properties.properties.clone(),
message: format!(
"Expected exactly one choices block, got {}",
response.choices.len()
@@ -174,7 +172,7 @@ impl WithChat for OpenAIClient {
start_time: system_start,
latency: instant_start.elapsed(),
model: response.model,
- invocation_params: self.properties.properties.clone(),
+ request_options: self.properties.properties.clone(),
metadata: LLMCompleteResponseMetadata {
baml_is_complete: match response.choices.get(0) {
Some(c) => match c.finish_reason {
@@ -213,11 +211,11 @@ impl RequestBuilder for OpenAIClient {
&self.client
}
- fn build_request(
+ async fn build_request(
&self,
prompt: either::Either<&String, &Vec>,
stream: bool,
- ) -> reqwest::RequestBuilder {
+ ) -> Result {
let mut req = self.client.post(if prompt.is_left() {
format!(
"{}/completions",
@@ -250,6 +248,10 @@ impl RequestBuilder for OpenAIClient {
}
req = req.header("baml-original-url", self.properties.base_url.as_str());
+ req = req.header(
+ "baml-render-url",
+ format!("{}/chat/completions", self.properties.base_url),
+ );
let mut body = json!(self.properties.properties);
let body_obj = body.as_object_mut().unwrap();
match prompt {
@@ -273,13 +275,19 @@ impl RequestBuilder for OpenAIClient {
}
if stream {
- body_obj.insert("stream".into(), true.into());
+ body_obj.insert("stream".into(), json!(true));
+ body_obj.insert(
+ "stream_options".into(),
+ json!({
+ "include_usage": true,
+ }),
+ );
}
- req.json(&body)
+ Ok(req.json(&body))
}
- fn invocation_params(&self) -> &HashMap {
+ fn request_options(&self) -> &HashMap {
&self.properties.properties
}
}
@@ -315,7 +323,7 @@ impl SseResponseTrait for OpenAIClient {
start_time: system_start,
latency: instant_start.elapsed(),
model: "".to_string(),
- invocation_params: params.clone(),
+ request_options: params.clone(),
metadata: LLMCompleteResponseMetadata {
baml_is_complete: false,
finish_reason: None,
@@ -344,7 +352,7 @@ impl SseResponseTrait for OpenAIClient {
prompt.clone(),
),
start_time: system_start,
- invocation_params: params.clone(),
+ request_options: params.clone(),
latency: instant_start.elapsed(),
message: format!("Failed to parse event: {:#?}", e),
code: ErrorCode::Other(2),
@@ -367,6 +375,11 @@ impl SseResponseTrait for OpenAIClient {
}
}
inner.latency = instant_start.elapsed();
+ if let Some(usage) = event.usage.as_ref() {
+ inner.metadata.prompt_tokens = Some(usage.prompt_tokens);
+ inner.metadata.output_tokens = Some(usage.completion_tokens);
+ inner.metadata.total_tokens = Some(usage.total_tokens);
+ }
std::future::ready(Some(LLMResponse::Success(inner.clone())))
},
@@ -378,7 +391,7 @@ impl SseResponseTrait for OpenAIClient {
impl WithStreamChat for OpenAIClient {
async fn stream_chat(
&self,
- ctx: &RuntimeContext,
+ _ctx: &RuntimeContext,
prompt: &Vec,
) -> StreamResponse {
let (resp, system_start, instant_start) =
@@ -411,15 +424,18 @@ macro_rules! make_openai_client {
($client:ident, $properties:ident) => {
Ok(Self {
name: $client.name().into(),
- properties: $properties,
+
context: RenderContext_Client {
name: $client.name().into(),
provider: $client.elem().provider.clone(),
+ default_role: $properties.default_role.clone(),
},
+ properties: $properties,
features: ModelFeatures {
chat: true,
completion: false,
anthropic_system_constraints: false,
+ resolve_media_urls: false,
},
retry_policy: $client
.elem()
@@ -506,17 +522,20 @@ fn convert_message_parts_to_content(parts: &Vec) -> serde_json:
.map(|part| match part {
ChatMessagePart::Text(text) => json!({"type": "text", "text": text}),
ChatMessagePart::Image(image) => match image {
- BamlImage::Url(image) => {
+ BamlMedia::Url(BamlMediaType::Image, image) => {
json!({"type": "image_url", "image_url": json!({
"url": image.url
})})
}
- BamlImage::Base64(image) => {
+ BamlMedia::Base64(BamlMediaType::Image, image) => {
json!({"type": "image_url", "image_url": json!({
- "base64": image.base64
+ "url" : format!("data:{};base64,{}", image.media_type, image.base64)
})})
}
+ _ => json!({}), // return an empty JSON object or any other default value
},
+ // OpenAI does not yet support audio
+ _ => json!({}), // return an empty JSON object or any other default value
})
.collect();
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/azure.rs b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/azure.rs
index 26d310349..e01af2b37 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/azure.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/openai/properties/azure.rs
@@ -83,6 +83,10 @@ pub fn resolve_properties(
}
};
+ properties
+ .entry("max_tokens".into())
+ .or_insert_with(|| 4096.into());
+
Ok(PostRequestProperities {
default_role,
base_url,
diff --git a/engine/baml-runtime/src/internal/llm_client/primitive/request.rs b/engine/baml-runtime/src/internal/llm_client/primitive/request.rs
index 3d480552b..bc31c9f53 100644
--- a/engine/baml-runtime/src/internal/llm_client/primitive/request.rs
+++ b/engine/baml-runtime/src/internal/llm_client/primitive/request.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, fmt::format};
+use std::collections::HashMap;
use anyhow::{Context, Result};
use internal_baml_jinja::RenderedChatMessage;
@@ -8,13 +8,14 @@ use serde::de::DeserializeOwned;
use crate::internal::llm_client::{traits::WithClient, ErrorCode, LLMErrorResponse, LLMResponse};
pub trait RequestBuilder {
- fn build_request(
+ #[allow(async_fn_in_trait)]
+ async fn build_request(
&self,
prompt: either::Either<&String, &Vec>,
stream: bool,
- ) -> reqwest::RequestBuilder;
+ ) -> Result;
- fn invocation_params(&self) -> &HashMap;
+ fn request_options(&self) -> &HashMap;
fn http_client(&self) -> &reqwest::Client;
}
@@ -36,7 +37,11 @@ pub async fn make_request(
let (system_now, instant_now) = (web_time::SystemTime::now(), web_time::Instant::now());
log::debug!("Making request using client {}", client.context().name);
- let req = match client.build_request(prompt, stream).build() {
+ let req = match client
+ .build_request(prompt, stream)
+ .await
+ .context("Failed to build request")
+ {
Ok(req) => req,
Err(e) => {
return Err(LLMResponse::LLMFailure(LLMErrorResponse {
@@ -44,14 +49,32 @@ pub async fn make_request(
model: None,
prompt: to_prompt(prompt),
start_time: system_now,
- invocation_params: client.invocation_params().clone(),
+ request_options: client.request_options().clone(),
latency: instant_now.elapsed(),
- message: e.to_string(),
+ message: format!("{:?}", e),
code: ErrorCode::Other(2),
}));
}
};
+ let req = match req.build() {
+ Ok(req) => req,
+ Err(e) => {
+ return Err(LLMResponse::LLMFailure(LLMErrorResponse {
+ client: client.context().name.to_string(),
+ model: None,
+ prompt: to_prompt(prompt),
+ start_time: system_now,
+ request_options: client.request_options().clone(),
+ latency: instant_now.elapsed(),
+ message: format!("{:?}", e),
+ code: ErrorCode::Other(2),
+ }));
+ }
+ };
+
+ log::debug!("built request: {:?}", req);
+
let response = match client.http_client().execute(req).await {
Ok(response) => response,
Err(e) => {
@@ -60,9 +83,9 @@ pub async fn make_request(
model: None,
prompt: to_prompt(prompt),
start_time: system_now,
- invocation_params: client.invocation_params().clone(),
+ request_options: client.request_options().clone(),
latency: instant_now.elapsed(),
- message: e.to_string(),
+ message: format!("{:?}", e),
code: ErrorCode::Other(2),
}));
}
@@ -75,7 +98,7 @@ pub async fn make_request(
model: None,
prompt: to_prompt(prompt),
start_time: system_now,
- invocation_params: client.invocation_params().clone(),
+ request_options: client.request_options().clone(),
latency: instant_now.elapsed(),
message: format!(
"Request failed: {}",
@@ -94,9 +117,26 @@ pub async fn make_parsed_request(
stream: bool,
) -> Result<(T, web_time::SystemTime, web_time::Instant), LLMResponse> {
let (response, system_now, instant_now) = make_request(client, prompt, stream).await?;
- match response.json::().await.context(format!(
- "Failed to parse into a response accepted by {}",
- std::any::type_name::()
+ let j = match response.json::().await {
+ Ok(response) => response,
+ Err(e) => {
+ return Err(LLMResponse::LLMFailure(LLMErrorResponse {
+ client: client.context().name.to_string(),
+ model: None,
+ prompt: to_prompt(prompt),
+ start_time: system_now,
+ request_options: client.request_options().clone(),
+ latency: instant_now.elapsed(),
+ message: e.to_string(),
+ code: ErrorCode::Other(2),
+ }))
+ }
+ };
+
+ match T::deserialize(&j).context(format!(
+ "Failed to parse into a response accepted by {}: {}",
+ std::any::type_name::(),
+ j
)) {
Ok(response) => Ok((response, system_now, instant_now)),
Err(e) => Err(LLMResponse::LLMFailure(LLMErrorResponse {
@@ -104,9 +144,9 @@ pub async fn make_parsed_request(
model: None,
prompt: to_prompt(prompt),
start_time: system_now,
- invocation_params: client.invocation_params().clone(),
+ request_options: client.request_options().clone(),
latency: instant_now.elapsed(),
- message: e.to_string(),
+ message: format!("{:?}", e),
code: ErrorCode::Other(2),
})),
}
diff --git a/engine/baml-runtime/src/internal/llm_client/traits/chat.rs b/engine/baml-runtime/src/internal/llm_client/traits/chat.rs
index 523834d4d..5ce186551 100644
--- a/engine/baml-runtime/src/internal/llm_client/traits/chat.rs
+++ b/engine/baml-runtime/src/internal/llm_client/traits/chat.rs
@@ -2,11 +2,11 @@ use anyhow::Result;
use internal_baml_jinja::{ChatOptions, RenderedChatMessage};
use crate::{
- internal::llm_client::{LLMErrorResponse, LLMResponse},
+ internal::llm_client::{LLMResponse},
RuntimeContext,
};
-use super::{SseResponseTrait, StreamResponse};
+use super::{StreamResponse};
pub trait WithChat: Sync + Send {
fn chat_options(&self, ctx: &RuntimeContext) -> Result;
diff --git a/engine/baml-runtime/src/internal/llm_client/traits/mod.rs b/engine/baml-runtime/src/internal/llm_client/traits/mod.rs
index d06e26fe3..cb07775ef 100644
--- a/engine/baml-runtime/src/internal/llm_client/traits/mod.rs
+++ b/engine/baml-runtime/src/internal/llm_client/traits/mod.rs
@@ -1,22 +1,35 @@
use std::pin::Pin;
use anyhow::Result;
+
mod chat;
mod completion;
+pub use self::{
+ chat::{WithChat, WithStreamChat},
+ completion::{WithCompletion, WithNoCompletion, WithStreamCompletion},
+};
+use super::{primitive::request::RequestBuilder, LLMResponse, ModelFeatures};
-use baml_types::BamlValue;
+use crate::{internal::prompt_renderer::PromptRenderer, RuntimeContext};
+use baml_types::{BamlMedia, BamlMediaType, BamlValue, MediaBase64};
+use base64::{prelude::BASE64_STANDARD, Engine};
+use futures::stream::{StreamExt, TryStreamExt};
+use infer;
use internal_baml_core::ir::repr::IntermediateRepr;
+use internal_baml_jinja::{ChatMessagePart, RenderedChatMessage};
use internal_baml_jinja::{RenderContext_Client, RenderedPrompt};
-use crate::{internal::prompt_renderer::PromptRenderer, RuntimeContext};
+use reqwest::Url;
-pub use self::{
- chat::{WithChat, WithStreamChat},
- completion::{WithCompletion, WithNoCompletion, WithStreamCompletion},
-};
+use shell_escape::escape;
+use std::borrow::Cow;
-use super::{retry_policy::CallablePolicy, LLMResponse, ModelFeatures};
+use std::str::FromStr; // Add this line at the top of your file // Add this line at the top of your file
+// #[enum_dispatch]
+
+// #[delegatable_trait]
+// #[enum_dispatch]
pub trait WithRetryPolicy {
fn retry_policy_name(&self) -> Option<&str>;
}
@@ -26,6 +39,14 @@ pub trait WithSingleCallable {
async fn single_call(&self, ctx: &RuntimeContext, prompt: &RenderedPrompt) -> LLMResponse;
}
+pub trait WithCurl {
+ #[allow(async_fn_in_trait)]
+ async fn curl_call(
+ &self,
+ ctx: &RuntimeContext,
+ prompt: &RenderedPrompt,
+ ) -> Result, LLMResponse>;
+}
pub trait WithClient {
fn context(&self) -> &RenderContext_Client;
@@ -42,12 +63,127 @@ pub trait WithPrompt<'ir> {
) -> Result;
}
+// #[delegatable_trait]
+// #[enum_dispatch]
+pub trait WithRenderRawCurl {
+ #[allow(async_fn_in_trait)]
+ async fn render_raw_curl(
+ &self,
+ ctx: &RuntimeContext,
+ prompt: &Vec,
+ stream: bool,
+ ) -> Result;
+}
+
impl WithSingleCallable for T
where
T: WithClient + WithChat + WithCompletion,
{
#[allow(async_fn_in_trait)]
async fn single_call(&self, ctx: &RuntimeContext, prompt: &RenderedPrompt) -> LLMResponse {
+ if self.model_features().resolve_media_urls {
+ if let RenderedPrompt::Chat(ref chat) = prompt {
+ let messages_result = futures::stream::iter(chat.iter().map(|p| {
+ let new_parts = p
+ .parts
+ .iter()
+ .map(|part| async move {
+ match part {
+ ChatMessagePart::Image(BamlMedia::Url(_, media_url))
+ | ChatMessagePart::Audio(BamlMedia::Url(_, media_url)) => {
+ let (base64, mime_type) = if media_url.url.starts_with("data:") {
+ let parts: Vec<&str> =
+ media_url.url.splitn(2, ',').collect();
+ let base64 = parts.get(1).unwrap().to_string();
+ let prefix = parts.get(0).unwrap();
+ let mime_type =
+ prefix.splitn(2, ':').next().unwrap().to_string()
+ .split('/').last().unwrap().to_string();
+
+ (base64, mime_type)
+ } else {
+ let client = reqwest::Client::new();
+ let response = match client.get(&media_url.url)
+ // NB(sam): this would workaround CORS issues, but https://github.com/seanmonstar/reqwest/issues/1401
+ //.fetch_mode_no_cors()
+ .send().await
+ {
+ Ok(response) => response,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ format!("Failed to fetch image due to CORS issue: {e:?}")
+ ))
+ } // replace with your error conversion logic
+ };
+ let bytes = match response.bytes().await {
+ Ok(bytes) => bytes,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ e.to_string(),
+ ));
+ } // replace with your error conversion logic
+ };
+ let base64 = BASE64_STANDARD.encode(&bytes);
+ let inferred_type = infer::get(&bytes);
+ let mime_type = inferred_type.map_or_else(
+ || "application/octet-stream".into(),
+ |t| t.extension().into(),
+ );
+ (base64, mime_type)
+ };
+
+ Ok(if matches!(part, ChatMessagePart::Image(_)) {
+ ChatMessagePart::Image(BamlMedia::Base64(
+ BamlMediaType::Image,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("image/{}", mime_type),
+ },
+ ))
+ } else {
+ ChatMessagePart::Audio(BamlMedia::Base64(
+ BamlMediaType::Audio,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("audio/{}", mime_type),
+ },
+ ))
+ })
+ }
+ _ => Ok(part.clone()),
+ }
+ })
+ .collect::>();
+ async move {
+ let new_parts = futures::stream::iter(new_parts)
+ .then(|f| f)
+ .collect::>()
+ .await;
+
+ let new_parts = new_parts.into_iter().collect::, _>>()?;
+
+ Ok::<_, anyhow::Error>(RenderedChatMessage {
+ role: p.role.clone(),
+ parts: new_parts,
+ })
+ }
+ }))
+ .then(|f| f)
+ .collect::>()
+ .await
+ .into_iter()
+ .collect::, _>>();
+
+ let messages = match messages_result {
+ Ok(messages) => messages,
+ Err(e) => {
+ return LLMResponse::OtherFailure(format!("Error occurred: {}", e));
+ }
+ };
+ return self.chat(ctx, &messages).await;
+ }
+ }
+
match prompt {
RenderedPrompt::Chat(p) => self.chat(ctx, p).await,
RenderedPrompt::Completion(p) => self.completion(ctx, p).await,
@@ -55,6 +191,155 @@ where
}
}
+impl WithCurl for T
+where
+ T: WithClient + WithChat + WithCompletion,
+{
+ #[allow(async_fn_in_trait)]
+ async fn curl_call(
+ &self,
+ ctx: &RuntimeContext,
+ prompt: &RenderedPrompt,
+ ) -> Result, LLMResponse> {
+ if self.model_features().resolve_media_urls {
+ if let RenderedPrompt::Chat(ref chat) = prompt {
+ let messages_result = futures::stream::iter(chat.iter().map(|p| {
+ let new_parts = p
+ .parts
+ .iter()
+ .map(|part| async move {
+ match part {
+ ChatMessagePart::Image(BamlMedia::Url(_, media_url))
+ | ChatMessagePart::Audio(BamlMedia::Url(_, media_url)) => {
+ let mut base64 = "".to_string();
+ let mut mime_type = "".to_string();
+ if media_url.url.starts_with("data:") {
+ let parts: Vec<&str> =
+ media_url.url.splitn(2, ',').collect();
+ base64 = parts.get(1).unwrap().to_string();
+ let prefix = parts.get(0).unwrap();
+ mime_type =
+ prefix.splitn(2, ':').next().unwrap().to_string();
+ mime_type =
+ mime_type.split('/').last().unwrap().to_string();
+ } else {
+ let client = reqwest::Client::new();
+ let response = match client.get(&media_url.url).send().await
+ {
+ Ok(response) => response,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ "Failed to fetch image due to CORS issue"
+ .to_string(),
+ ))
+ } // replace with your error conversion logic
+ };
+ let bytes = match response.bytes().await {
+ Ok(bytes) => bytes,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ e.to_string(),
+ ))
+ } // replace with your error conversion logic
+ };
+ base64 = BASE64_STANDARD.encode(&bytes);
+ let inferred_type = infer::get(&bytes);
+ mime_type = inferred_type.map_or_else(
+ || "application/octet-stream".into(),
+ |t| t.extension().into(),
+ );
+ }
+
+ Ok(if matches!(part, ChatMessagePart::Image(_)) {
+ ChatMessagePart::Image(BamlMedia::Base64(
+ BamlMediaType::Image,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("image/{}", mime_type),
+ },
+ ))
+ } else {
+ ChatMessagePart::Audio(BamlMedia::Base64(
+ BamlMediaType::Audio,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("audio/{}", mime_type),
+ },
+ ))
+ })
+ }
+ _ => Ok(part.clone()),
+ }
+ })
+ .collect::>();
+ async move {
+ let new_parts = futures::stream::iter(new_parts)
+ .then(|f| f)
+ .collect::>()
+ .await;
+
+ let new_parts = new_parts.into_iter().collect::, _>>()?;
+
+ Ok::<_, anyhow::Error>(RenderedChatMessage {
+ role: p.role.clone(),
+ parts: new_parts,
+ })
+ }
+ }))
+ .then(|f| f)
+ .collect::>()
+ .await
+ .into_iter()
+ .collect::, _>>();
+
+ let messages = match messages_result {
+ Ok(messages) => messages,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(format!("Error occurred: {}", e)));
+ }
+ };
+ return Ok(messages);
+ }
+ }
+
+ match prompt {
+ RenderedPrompt::Chat(p) => Ok(p.clone()),
+ RenderedPrompt::Completion(p) => Err(LLMResponse::OtherFailure(
+ "Completion prompts are not supported by this provider".to_string(),
+ )),
+ }
+ }
+}
+
+fn escape_single_quotes(s: &str) -> String {
+ escape(Cow::Borrowed(s)).to_string()
+}
+
+fn to_curl_command(
+ url: &str,
+ method: &str,
+ headers: &reqwest::header::HeaderMap,
+ body: Vec,
+) -> String {
+ let mut curl_command = format!("curl -X {} '{}'", method, url);
+
+ for (key, value) in headers.iter() {
+ let header = format!(" -H \"{}: {}\"", key.as_str(), value.to_str().unwrap());
+ curl_command.push_str(&header);
+ }
+
+ let body_json = String::from_utf8_lossy(&body).to_string(); // Convert body to string
+ let pretty_body_json = match serde_json::from_str::(&body_json) {
+ Ok(json_value) => serde_json::to_string_pretty(&json_value).unwrap_or(body_json),
+ Err(_) => body_json,
+ };
+ let fully_escaped_body_json = escape_single_quotes(&pretty_body_json);
+ let body_part = format!(" -d {}", fully_escaped_body_json);
+ curl_command.push_str(&body_part);
+
+ curl_command
+}
+
impl<'ir, T> WithPrompt<'ir> for T
where
T: WithClient + WithChat + WithCompletion,
@@ -69,7 +354,6 @@ where
let features = self.model_features();
let prompt = renderer.render_prompt(ir, ctx, params, self.context())?;
- log::debug!("WithPrompt.render_prompt => {:#?}", prompt);
let mut prompt = match (features.completion, features.chat) {
(true, false) => {
@@ -108,6 +392,53 @@ where
}
}
+impl WithRenderRawCurl for T
+where
+ T: WithClient + WithChat + WithCompletion + RequestBuilder,
+{
+ async fn render_raw_curl(
+ &self,
+ ctx: &RuntimeContext,
+ prompt: &Vec,
+ stream: bool,
+ ) -> Result {
+ let rendered_prompt = RenderedPrompt::Chat(prompt.clone());
+
+ let chat_messages = self.curl_call(ctx, &rendered_prompt).await?;
+ let request_builder = self
+ .build_request(either::Right(&chat_messages), stream)
+ .await?;
+ let mut request = request_builder.build()?;
+ let url_header_value = {
+ let headers = request.headers_mut();
+ let url_header_value = headers
+ .get("baml-render-url")
+ .ok_or(anyhow::anyhow!("Missing header 'baml-render-url'"))?;
+ url_header_value.to_owned()
+ };
+
+ let url_str = url_header_value
+ .to_str()
+ .map_err(|_| anyhow::anyhow!("Invalid header 'baml-render-url'"))?;
+ let new_url = Url::from_str(url_str)?;
+ *request.url_mut() = new_url;
+
+ {
+ let headers = request.headers_mut();
+ headers.remove("baml-original-url");
+ headers.remove("baml-render-url");
+ }
+
+ let body = request
+ .body()
+ .map(|b| b.as_bytes().unwrap_or_default().to_vec())
+ .unwrap_or_default(); // Add this line to handle the Option
+ let request_str = to_curl_command(url_str, "POST", request.headers(), body);
+
+ Ok(request_str)
+ }
+}
+
// Stream related
pub trait SseResponseTrait {
fn response_stream(
@@ -134,10 +465,116 @@ pub trait WithStreamable {
impl WithStreamable for T
where
- T: WithStreamChat + WithStreamCompletion,
+ T: WithClient + WithStreamChat + WithStreamCompletion,
{
#[allow(async_fn_in_trait)]
async fn stream(&self, ctx: &RuntimeContext, prompt: &RenderedPrompt) -> StreamResponse {
+ if self.model_features().resolve_media_urls {
+ if let RenderedPrompt::Chat(ref chat) = prompt {
+ let messages = futures::stream::iter(chat.iter().map(|p| {
+ let new_parts = p
+ .parts
+ .iter()
+ .map(|part| async move {
+ match part {
+ ChatMessagePart::Image(BamlMedia::Url(_, media_url))
+ | ChatMessagePart::Audio(BamlMedia::Url(_, media_url)) => {
+ let mut base64 = "".to_string();
+ let mut mime_type = "".to_string();
+ if media_url.url.starts_with("data:") {
+ let parts: Vec<&str> =
+ media_url.url.splitn(2, ',').collect();
+ base64 = parts.get(1).unwrap().to_string();
+ let prefix = parts.get(0).unwrap();
+ mime_type = prefix
+ .splitn(2, ':')
+ .last()
+ .unwrap() // Get the part after "data:"
+ .split('/')
+ .last()
+ .unwrap() // Get the part after "image/"
+ .split(';')
+ .next()
+ .unwrap() // Get the part before ";base64"
+ .to_string();
+ } else {
+ let client = reqwest::Client::new();
+ let response = match client
+ .get(&media_url.url)
+ // NB(sam): this would workaround CORS issues, but https://github.com/seanmonstar/reqwest/issues/1401
+ //.fetch_mode_no_cors()
+ .send()
+ .await
+ {
+ Ok(response) => response,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ "Failed to fetch image due to CORS issue"
+ .to_string(),
+ ))
+ } // replace with your error conversion logic
+ };
+ let bytes = match response.bytes().await {
+ Ok(bytes) => bytes,
+ Err(e) => {
+ return Err(LLMResponse::OtherFailure(
+ e.to_string(),
+ ))
+ } // replace with your error conversion logic
+ };
+ base64 = BASE64_STANDARD.encode(&bytes);
+ let inferred_type = infer::get(&bytes);
+ mime_type = inferred_type.map_or_else(
+ || "application/octet-stream".into(),
+ |t| t.extension().into(),
+ );
+ }
+ Ok(if matches!(part, ChatMessagePart::Image(_)) {
+ ChatMessagePart::Image(BamlMedia::Base64(
+ BamlMediaType::Image,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("image/{}", mime_type),
+ },
+ ))
+ } else {
+ ChatMessagePart::Audio(BamlMedia::Base64(
+ BamlMediaType::Audio,
+ MediaBase64 {
+ base64: base64,
+ media_type: format!("audio/{}", mime_type),
+ },
+ ))
+ })
+ }
+ _ => Ok(part.clone()),
+ }
+ })
+ .collect::>();
+ async move {
+ let new_parts = futures::stream::iter(new_parts)
+ .then(|f| f)
+ .collect::>()
+ .await;
+
+ let new_parts = new_parts.into_iter().collect::, _>>()?;
+
+ Ok(RenderedChatMessage {
+ role: p.role.clone(),
+ parts: new_parts,
+ })
+ }
+ }))
+ .then(|f| f)
+ .collect::>()
+ .await
+ .into_iter()
+ .collect::, _>>()?;
+
+ return self.stream_chat(ctx, &messages).await;
+ }
+ }
+
match prompt {
RenderedPrompt::Chat(p) => self.stream_chat(ctx, p).await,
RenderedPrompt::Completion(p) => self.stream_completion(ctx, p).await,
diff --git a/engine/baml-runtime/src/internal/prompt_renderer/mod.rs b/engine/baml-runtime/src/internal/prompt_renderer/mod.rs
index a2978daa8..96ff49e2d 100644
--- a/engine/baml-runtime/src/internal/prompt_renderer/mod.rs
+++ b/engine/baml-runtime/src/internal/prompt_renderer/mod.rs
@@ -71,6 +71,7 @@ impl PromptRenderer {
client_ctx: &RenderContext_Client,
) -> Result {
let func = ir.find_function(&self.function_name)?;
+
let Some(func_v2) = func.as_v2() else {
error_unsupported!(
"function",
diff --git a/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs b/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs
index 447a1adfe..8b54c33ae 100644
--- a/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs
+++ b/engine/baml-runtime/src/internal/prompt_renderer/render_output_format.rs
@@ -1,4 +1,4 @@
-use std::collections::{HashMap, HashSet};
+use std::collections::{HashSet};
use anyhow::Result;
use baml_types::BamlValue;
diff --git a/engine/baml-runtime/src/lib.rs b/engine/baml-runtime/src/lib.rs
index a8a149823..940b22a56 100644
--- a/engine/baml-runtime/src/lib.rs
+++ b/engine/baml-runtime/src/lib.rs
@@ -33,6 +33,7 @@ use baml_types::BamlValue;
use client_builder::ClientBuilder;
use indexmap::IndexMap;
use internal_baml_core::configuration::GeneratorOutputType;
+use on_log_event::LogEventCallbackSync;
use runtime::InternalBamlRuntime;
#[cfg(not(target_arch = "wasm32"))]
@@ -51,6 +52,9 @@ pub use internal_baml_jinja::{ChatMessagePart, RenderedPrompt};
#[cfg(feature = "internal")]
pub use runtime_interface::InternalRuntimeInterface;
+#[cfg(feature = "internal")]
+pub use internal_baml_core as internal_core;
+
#[cfg(not(feature = "internal"))]
pub(crate) use internal_baml_jinja::{ChatMessagePart, RenderedPrompt};
#[cfg(not(feature = "internal"))]
@@ -82,7 +86,7 @@ impl BamlRuntime {
.collect();
Ok(BamlRuntime {
inner: InternalBamlRuntime::from_directory(path)?,
- tracer: BamlTracer::new(None, env_vars.into_iter()).into(),
+ tracer: BamlTracer::new(None, env_vars.into_iter())?.into(),
env_vars: copy,
})
}
@@ -98,7 +102,7 @@ impl BamlRuntime {
.collect();
Ok(BamlRuntime {
inner: InternalBamlRuntime::from_file_content(root_path, files)?,
- tracer: BamlTracer::new(None, env_vars.into_iter()).into(),
+ tracer: BamlTracer::new(None, env_vars.into_iter())?.into(),
env_vars: copy,
})
}
@@ -341,4 +345,14 @@ impl ExperimentalTracingInterface for BamlRuntime {
fn flush(&self) -> Result<()> {
self.tracer.flush()
}
+
+ fn drain_stats(&self) -> InnerTraceStats {
+ self.tracer.drain_stats()
+ }
+
+ #[cfg(not(target_arch = "wasm32"))]
+ fn set_log_event_callback(&self, log_event_callback: LogEventCallbackSync) -> Result<()> {
+ self.tracer.set_log_event_callback(log_event_callback);
+ Ok(())
+ }
}
diff --git a/engine/baml-runtime/src/request/mod.rs b/engine/baml-runtime/src/request/mod.rs
index dfbd4c49f..3e1d80087 100644
--- a/engine/baml-runtime/src/request/mod.rs
+++ b/engine/baml-runtime/src/request/mod.rs
@@ -1,15 +1,35 @@
use anyhow::{Context, Result};
-use instant::Duration;
+use web_time::Duration;
-pub(crate) fn create_client() -> Result {
+fn builder() -> reqwest::ClientBuilder {
cfg_if::cfg_if! {
if #[cfg(target_arch = "wasm32")] {
- Ok(reqwest::Client::new())
+ reqwest::Client::builder()
} else {
reqwest::Client::builder()
+ // NB: we can NOT set a total request timeout here: our users
+ // regularly have requests that take multiple minutes, due to how
+ // long LLMs take
+ .connect_timeout(Duration::from_secs(10))
.http2_keep_alive_interval(Some(Duration::from_secs(10)))
- .build()
- .context("Failed to create reqwest client")
}
}
}
+
+pub(crate) fn create_client() -> Result {
+ builder().build().context("Failed to create reqwest client")
+}
+
+pub(crate) fn create_tracing_client() -> Result {
+ cfg_if::cfg_if! {
+ if #[cfg(target_arch = "wasm32")] {
+ let cb = builder();
+ } else {
+ let cb =builder()
+ // Wait up to 30s to send traces to the backend
+ .read_timeout(Duration::from_secs(30));
+ }
+ }
+
+ cb.build().context("Failed to create reqwest client")
+}
diff --git a/engine/baml-runtime/src/request/no_wasm.rs b/engine/baml-runtime/src/request/no_wasm.rs
deleted file mode 100644
index 93be95806..000000000
--- a/engine/baml-runtime/src/request/no_wasm.rs
+++ /dev/null
@@ -1,59 +0,0 @@
-use std::collections::HashMap;
-
-use anyhow::Error;
-
-#[derive(Debug)]
-pub enum NoWasmRequestError {
- BuildError(Error),
- FetchError(Error),
- ResponseError(u16, reqwest::Response),
- JsonError(Error),
- SerdeError(Error),
-}
-
-pub async fn response_text(response: reqwest::Response) -> Result
+
+- If nothing works, please file an issue on [Github](https://github.com/BoundaryML/baml/issues), ideally with a screenshot of the error and the steps to reproduce it.
+
+## Common Issues
+### No code lens in BAML files
+
+This can happen in two cases:
+1. You have syntax error in some `.baml` file. You can check the error in the `Problems` tab in VSCode or running the `generate` command in the terminal (See [Generate](/docs/calling-baml/generate-baml-client))
+
+2. BAML extension is broken. Please try the tools above!
+
+### BAML extension is not working
+
+### Tests hanging
+
+We've seen sparse repros of this, but closing the playground and reopening it should fix it.
+
+### Tests failing to run
+
+You can debug the actual network request being made by BAML by opening developer tools:
+
+
diff --git a/docs/docs/get-started/deploying/docker.mdx b/docs/docs/get-started/deploying/docker.mdx
new file mode 100644
index 000000000..362bc2732
--- /dev/null
+++ b/docs/docs/get-started/deploying/docker.mdx
@@ -0,0 +1,30 @@
+
+When you develop with BAML, the BAML VScode extension generates a `baml_client` directory (on every save) with all the generated code you need to use your AI functions in your application.
+
+We recommend you add `baml_client` to your `.gitignore` file to avoid committing generated code to your repository, and re-generate the client code when you build and deploy your application.
+
+You _could_ commit the generated code if you're starting out to not deal with this, just make sure the VSCode extension version matches your baml package dependency version (e.g. `baml-py` for python and `@boundaryml/baml` for TS) so there are no compatibility issues.
+
+To build your client you can use the following command. See also [Generating Clients](/docs/calling-baml/generate-baml-client):
+
+
+
+Or type `BAML Playground` in the VSCode Command Bar (`CMD + Shift + P` or `CTRL + Shift + P`) to open the playground.
+
+
+
+## Setting Env Variables
+
+Click on the `Settings` button in top right of the playground and set the environment variables.
+
+It should have an indicator saying how many unset variables are there.
+
+
+
+The playground should persist the environment variables between closing and opening VSCode.
+
+
+
+You can toggle between seeing the results of all test cases or all test cases for the current function.
+
+
diff --git a/docs/docs/get-started/quickstart/python.mdx b/docs/docs/get-started/quickstart/python.mdx
new file mode 100644
index 000000000..8dee33156
--- /dev/null
+++ b/docs/docs/get-started/quickstart/python.mdx
@@ -0,0 +1,73 @@
+Here's a sample repository:
+https://github.com/BoundaryML/baml-examples/tree/main/python-fastapi-starter
+
+To set up BAML in python do the following:
+
+
*/}
-
-## Overview
-
-Before you call an LLM, ask yourself what kind of input or output youre
-expecting. If you want the LLM to generate text, then you probably want a
-string, but if you're trying to get it to collect user details, you may want it
-to return a complex type like `UserDetails`.
-
-Thinking this way can help you decompose large complex prompts into smaller,
-more measurable functions, and will also help you build more complex workflows
-and agents.
-
-# Extracting a resume from text
-
-The best way to learn BAML is to run an example in our web playground -- [PromptFiddle.com](https://promptfiddle.com).
-
-But at a high-level, BAML is simple to use -- prompts are built using [Jinja syntax](https://jinja.palletsprojects.com/en/3.1.x/) to make working with strings easier. But we extended jinja to add type-support, static analysis of your template variables, and we have a real-time preview of prompts in the BAML VSCode extension no matter how much logic your prompts use.
-
-Here's an example from PromptFiddle:
-
-```rust baml_src/main.baml
-client
-
-This feature is limited for users on the free tier, and available as many times as needed for paid users.
diff --git a/docs/docs/guides/improve_results/fine_tune.mdx b/docs/docs/guides/improve_results/fine_tune.mdx
deleted file mode 100644
index d7c5d3ca5..000000000
--- a/docs/docs/guides/improve_results/fine_tune.mdx
+++ /dev/null
@@ -1,5 +0,0 @@
----
-title: "Fine-tune a model using my production data"
----
-
-Reach out to us on Discord if you want to improve performance, reduce costs or latencies using fine-tuned models! We are working on seamless integrations with fine-tuning platforms.
\ No newline at end of file
diff --git a/docs/docs/guides/overview.mdx b/docs/docs/guides/overview.mdx
deleted file mode 100644
index 1e8b2eccd..000000000
--- a/docs/docs/guides/overview.mdx
+++ /dev/null
@@ -1,50 +0,0 @@
----
-title: "Table of contents"
----
-
-These tutorials assume you've already done the [Learn BAML](/docs/guides/hello_world/level0) tutorials first and have a hang of some of the basics.
-
-Ping us on [Discord](https://discord.gg/BTNBeXGuaS) if you have any questions!
-
+
+See anthropic docs for the latest list of all models. You can pass any model name you wish, we will not check if it exists.
+
-
-
-With baml, you can modify the implementation of a function and keep the application logic that uses the
-function unchanged.
-
-Checkout [PromptFiddle](https://promptfiddle.com) to see various BAML function examples.
\ No newline at end of file
diff --git a/docs/docs/syntax/generator.mdx b/docs/docs/syntax/generator.mdx
deleted file mode 100644
index fa788dc7d..000000000
--- a/docs/docs/syntax/generator.mdx
+++ /dev/null
@@ -1,21 +0,0 @@
----
-title: generator
----
-
-The `generator` configuration needs to be added anywhere in .baml files to generate the `baml_client` in Python or Typescript.
-
-We recommend running **baml init** to have this setup for you with sane defaults.
-
-Here is how you can add a generator block:
-
-```rust
-generator MyGenerator{
- output_type typescript // or python/pydantic, ruby
- output_dir ../
-}
-```
-
-| Property | Description | Options | Default |
-| ------------------- | ------------------------------------------------ | --------------------------------- | ---------------------------------------------- |
-| output_type | The language of the generated client | python/pydantic, ruby, typescript | |
-| output_dir | The directory where we'll output the generated baml_client | | ../ |
diff --git a/docs/docs/syntax/overview.mdx b/docs/docs/syntax/overview.mdx
deleted file mode 100644
index 29f7a8ef9..000000000
--- a/docs/docs/syntax/overview.mdx
+++ /dev/null
@@ -1,68 +0,0 @@
----
-title: BAML Project Structure
----
-
-A BAML project has the following structure:
-
-```bash
-.
-├── baml_client/ # Generated code
-├── baml_src/ # Prompts live here
-│ └── foo.baml
-# The rest of your project (not generated nor used by BAML)
-├── app/
-│ ├── __init__.py
-│ └── main.py
-└── pyproject.toml
-
-```
-
-1. `baml_src/` is the directory where you write your BAML files with the AI
- function declarations, prompts, retry policies, etc. It also contains
- [generator](/syntax/generator) blocks which configure how and where to
- transpile your BAML code.
-
-2. `baml_client/` is the directory where BAML will generate code, and where you'll
- import the generated code from.
-
-
-
-| LLM Output | Desired Type | Baml Output | How |
-| ------------------------------------------------------------------------------------------------------------------------------------------------ | ------------ | --------------- | ------------------------------------------------------------------------------------------ |
-| `great` | Style | `Style.GREAT` | We handle case insensitivity |
-| `"great"` | Style | `Style.GREAT` | We handle extra quotes |
-| `great` | Style[] | `[Style.GREAT]` | Non-array types are automatically wrapped in an array |
-| `{ "feeling": "great" }` | Style | `Style.GREAT` | When looking for a singular value, we can parse dictionaries of 1 keys as singular objects |
-| 
-Note, we can apply the same parsing logic to any type, not just enums. e.g. in the
-case of numbers, we can remove commas and parse the number. This page outlines all
-the rules we use to parse each type.
-
-