Generic multimodal BAML type

docs/docs/syntax/type.mdx

@@ -32,7 +32,7 @@ title: Supported Types
 
 - **Syntax:** `null`
 
-### ✅  Images
+### ✅ Images
 
 You can use an image like this:
 
@@ -47,6 +47,23 @@ function DescribeImage(myImg: image) -> string {
 }
 ```
 
+
+### ✅ Audio
+We support audio for existing models that support it, such as Gemini Pro and Flash. 
+You can use audio like this:
+
+```rust
+function DescribeAudio(myAudio: audio) -> string {
+  client GPT4Turbo
+  prompt #"
+    {{ _.role("user")}}
+    Describe the tone of this song in four words:
+    {{ myAudio }}
+  "#
+}
+```
+
+
 ### ⚠️ bytes
 
 - Not yet supported. Use a `string[]` or `int[]` instead.

engine/baml-lib/baml-core/src/ir/ir_helpers/to_baml_arg.rs

@@ -1,4 +1,4 @@
-use baml_types::{BamlMap, BamlValue, TypeValue};
+use baml_types::{BamlMap, BamlMedia, BamlMediaType, BamlValue, TypeValue};
 
 use crate::ir::{FieldType, IntermediateRepr};
 
@@ -57,18 +57,23 @@ pub fn validate_arg(
             TypeValue::Bool if matches!(value, BamlValue::Bool(_)) => Some(value.clone()),
             TypeValue::Null if matches!(value, BamlValue::Null) => Some(value.clone()),
             TypeValue::Image => match value {
-                BamlValue::Image(v) => Some(BamlValue::Image(v.clone())),
+                BamlValue::Media(v) => Some(BamlValue::Media(v.clone())),
                 BamlValue::Map(kv) => {
                     if let Some(BamlValue::String(s)) = kv.get("url") {
-                        Some(BamlValue::Image(baml_types::BamlImage::url(s.to_string())))
+                        Some(BamlValue::Media(baml_types::BamlMedia::url(
+                            BamlMediaType::Image,
+                            s.to_string(),
+                            None,
+                        )))
                     } else if let (
                         Some(BamlValue::String(s)),
-                        Some(BamlValue::String(media_type)),
+                        Some(BamlValue::String(media_type_str)),
                     ) = (kv.get("base64"), kv.get("media_type"))
                     {
-                        Some(BamlValue::Image(baml_types::BamlImage::base64(
+                        Some(BamlValue::Media(baml_types::BamlMedia::base64(
+                            BamlMediaType::Image,
                             s.to_string(),
-                            media_type.to_string(),
+                            media_type_str.to_string(),
                         )))
                     } else {
                         scope.push_error(format!(
@@ -83,6 +88,38 @@ pub fn validate_arg(
                     None
                 }
             },
+            TypeValue::Audio => match value {
+                BamlValue::Media(v) => Some(BamlValue::Media(v.clone())),
+                BamlValue::Map(kv) => {
+                    if let Some(BamlValue::String(s)) = kv.get("url") {
+                        Some(BamlValue::Media(baml_types::BamlMedia::url(
+                            BamlMediaType::Audio,
+                            s.to_string(),
+                            None,
+                        )))
+                    } else if let (
+                        Some(BamlValue::String(s)),
+                        Some(BamlValue::String(media_type_str)),
+                    ) = (kv.get("base64"), kv.get("media_type"))
+                    {
+                        Some(BamlValue::Media(baml_types::BamlMedia::base64(
+                            BamlMediaType::Audio,
+                            s.to_string(),
+                            media_type_str.to_string(),
+                        )))
+                    } else {
+                        scope.push_error(format!(
+                                "Invalid audio: expected `url` or (`base64` and `media_type`), got `{}`",
+                                value
+                            ));
+                        None
+                    }
+                }
+                _ => {
+                    scope.push_error(format!("Expected type {:?}, got `{}`", t, value));
+                    None
+                }
+            },
             _ => {
                 scope.push_error(format!("Expected type {:?}, got `{}`", t, value));
                 None

engine/baml-lib/baml-core/src/ir/json_schema.rs

@@ -193,6 +193,17 @@ impl<'db> WithJsonSchema for FieldType {
                     "required": ["url"],
 
                 }),
+                TypeValue::Audio => json!({
+                    // anyOf either an object that has a uri, or it has a base64 string
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            // "format": "uri",
+                        }
+                    },
+                    "required": ["url"],
+                }),
             },
             FieldType::List(item) => json!({
                 "type": "array",

engine/baml-lib/baml-core/src/ir/walker.rs

@@ -340,6 +340,10 @@ impl<'a> Walker<'a, &'a Client> {
     pub fn span(&self) -> Option<&crate::Span> {
         self.item.attributes.span.as_ref()
     }
+
+    pub fn options(&self) -> &Vec<(String, Expression)> {
+        &self.elem().options
+    }
 }
 
 impl<'a> Walker<'a, &'a RetryPolicy> {

engine/baml-lib/baml-types/src/baml_value.rs

@@ -2,7 +2,8 @@ use std::{collections::HashSet, fmt};
 
 use serde::{de::Visitor, ser::SerializeStruct, Deserialize, Deserializer};
 
-use crate::{BamlImage, BamlMap};
+use crate::media::BamlMediaType;
+use crate::{BamlMap, BamlMedia};
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum BamlValue {
@@ -12,7 +13,7 @@ pub enum BamlValue {
     Bool(bool),
     Map(BamlMap<String, BamlValue>),
     List(Vec<BamlValue>),
-    Image(BamlImage),
+    Media(BamlMedia),
     Enum(String, String),
     Class(String, BamlMap<String, BamlValue>),
     Null,
@@ -27,19 +28,33 @@ impl serde::Serialize for BamlValue {
             BamlValue::Bool(b) => serializer.serialize_bool(*b),
             BamlValue::Map(m) => m.serialize(serializer),
             BamlValue::List(l) => l.serialize(serializer),
-            BamlValue::Image(i) => {
-                let mut s = serializer.serialize_struct("BamlImage", 2)?;
-                match i {
-                    BamlImage::Url(u) => {
-                        s.serialize_field("url", &u.url)?;
-                    }
-                    BamlImage::Base64(b) => {
-                        s.serialize_field("base64", &b.base64)?;
-                        s.serialize_field("media_type", &b.media_type)?;
-                    }
+            BamlValue::Media(i) => match i {
+                BamlMedia::Url(BamlMediaType::Image, u) => {
+                    let mut s = serializer.serialize_struct("BamlImage", 2)?;
+                    s.serialize_field("url", &u.url)?;
+                    s.end()
                 }
-                s.end()
-            }
+                BamlMedia::Base64(BamlMediaType::Image, b) => {
+                    let mut s = serializer.serialize_struct("BamlImage", 2)?;
+                    s.serialize_field("base64", &b.base64)?;
+                    s.serialize_field("media_type", &b.media_type)?;
+                    s.end()
+                }
+                BamlMedia::Url(BamlMediaType::Audio, u) => {
+                    let mut s = serializer.serialize_struct("BamlAudio", 2)?;
+
+                    s.serialize_field("url", &u.url)?;
+                    s.end()
+                }
+                BamlMedia::Base64(BamlMediaType::Audio, b) => {
+                    let mut s = serializer.serialize_struct("BamlAudio", 2)?;
+
+                    s.serialize_field("base64", &b.base64)?;
+                    s.serialize_field("media_type", &b.media_type)?;
+                    s.end()
+                }
+            },
+
             BamlValue::Enum(_, v) => serializer.serialize_str(v),
             BamlValue::Class(_, m) => m.serialize(serializer),
             BamlValue::Null => serializer.serialize_none(),
@@ -82,7 +97,12 @@ impl BamlValue {
                     format!("list<{}>", value_type)
                 }
             }
-            BamlValue::Image(_) => "image".into(),
+            BamlValue::Media(m) => match m {
+                BamlMedia::Url(BamlMediaType::Image, _) => "image".into(),
+                BamlMedia::Base64(BamlMediaType::Image, _) => "image".into(),
+                BamlMedia::Url(BamlMediaType::Audio, _) => "audio".into(),
+                BamlMedia::Base64(BamlMediaType::Audio, _) => "audio".into(),
+            },
             BamlValue::Enum(e, _) => format!("enum {}", e),
             BamlValue::Class(c, _) => format!("class {}", c),
             BamlValue::Null => "null".into(),

engine/baml-lib/baml-types/src/field_type/mod.rs

@@ -9,6 +9,7 @@ pub enum TypeValue {
     // Char,
     Null,
     Image,
+    Audio,
 }
 
 impl std::fmt::Display for TypeValue {
@@ -20,6 +21,7 @@ impl std::fmt::Display for TypeValue {
             TypeValue::Bool => write!(f, "bool"),
             TypeValue::Null => write!(f, "null"),
             TypeValue::Image => write!(f, "image"),
+            TypeValue::Audio => write!(f, "audio"),
         }
     }
 }

engine/baml-lib/baml-types/src/lib.rs

@@ -1,5 +1,5 @@
-mod image;
 mod map;
+mod media;
 #[cfg(feature = "mini-jinja")]
 mod minijinja;
 
@@ -8,5 +8,5 @@ mod field_type;
 
 pub use baml_value::BamlValue;
 pub use field_type::{FieldType, TypeValue};
-pub use image::{BamlImage, ImageBase64, ImageUrl};
 pub use map::Map as BamlMap;
+pub use media::{BamlMedia, BamlMediaType, MediaBase64, MediaUrl};