feat: make baml-cli log::info by default

engine/baml-runtime/src/cli/mod.rs

@@ -47,17 +47,6 @@ pub struct RuntimeCliDefaults {
 
 impl RuntimeCli {
     pub fn run(&mut self, defaults: RuntimeCliDefaults) -> Result<()> {
-        if let Err(e) = env_logger::Builder::from_env(
-            env_logger::Env::new()
-                .filter_or("BAML_LOG", "info")
-                .write_style("BAML_LOG_STYLE"),
-        )
-        // .format_target(false)
-        .try_init()
-        {
-            eprintln!("Failed to initialize BAML logger: {:#}", e);
-        };
-
         match &mut self.command {
             Commands::Generate(args) => {
                 args.from = BamlRuntime::parse_baml_src_path(&args.from)?;

engine/baml-runtime/src/cli/serve/error.rs

@@ -1,8 +1,11 @@
 use axum::response::{IntoResponse, Response};
 use http::StatusCode;
+use internal_baml_core::ir::scope_diagnostics::ScopeStack;
 use serde::Serialize;
 use serde_json::json;
 
+use crate::{errors::ExposedError, internal::llm_client::LLMResponse};
+
 use super::json_response::Json;
 
 /// The concrete HTTP error type that we return to our users if something goes wrong.
@@ -22,6 +25,47 @@ pub enum BamlError {
     InternalError(String),
 }
 
+impl BamlError {
+    pub(crate) fn from_anyhow(err: anyhow::Error) -> Self {
+        if let Some(er) = err.downcast_ref::<ExposedError>() {
+            match er {
+                ExposedError::ValidationError(_) => Self::ValidationFailure(format!("{:?}", err)),
+            }
+        } else if let Some(er) = err.downcast_ref::<ScopeStack>() {
+            Self::InvalidArgument(format!("{:?}", er))
+        } else if let Some(er) = err.downcast_ref::<LLMResponse>() {
+            match er {
+                LLMResponse::Success(_) => {
+                    Self::InternalError(format!("Unexpected error from BAML: {:?}", err))
+                }
+                LLMResponse::LLMFailure(failed) => match &failed.code {
+                    crate::internal::llm_client::ErrorCode::Other(2) => Self::InternalError(
+                        format!("Something went wrong with the LLM client: {:?}", err),
+                    ),
+                    crate::internal::llm_client::ErrorCode::Other(_)
+                    | crate::internal::llm_client::ErrorCode::InvalidAuthentication
+                    | crate::internal::llm_client::ErrorCode::NotSupported
+                    | crate::internal::llm_client::ErrorCode::RateLimited
+                    | crate::internal::llm_client::ErrorCode::ServerError
+                    | crate::internal::llm_client::ErrorCode::ServiceUnavailable
+                    | crate::internal::llm_client::ErrorCode::UnsupportedResponse(_) => {
+                        Self::ClientError(format!("{:?}", err))
+                    }
+                },
+                LLMResponse::UserFailure(msg) => {
+                    Self::InvalidArgument(format!("Invalid argument: {}", msg))
+                }
+                LLMResponse::InternalFailure(_) => Self::InternalError(format!(
+                    "Something went wrong with the LLM client: {}",
+                    err
+                )),
+            }
+        } else {
+            Self::InternalError(format!("{:?}", err))
+        }
+    }
+}
+
 impl IntoResponse for BamlError {
     fn into_response(self) -> Response {
         (

engine/baml-runtime/src/cli/serve/mod.rs

@@ -403,11 +403,7 @@ Tip: test that the server is up using `curl http://localhost:{}/_debug/ping`
                     BamlError::InternalError(message.clone()).into_response()
                 }
             },
-            Err(e) => BamlError::InternalError(format!(
-                "{:?}",
-                e.context("Failed to produce FunctionResult")
-            ))
-            .into_response(),
+            Err(e) => BamlError::from_anyhow(e).into_response(),
         }
     }
 

engine/baml-runtime/src/main.rs

@@ -2,6 +2,18 @@ use anyhow::Result;
 use baml_runtime::{BamlRuntime, RuntimeCliDefaults};
 
 fn main() -> Result<()> {
+    // We init here and not in run_cli because Python/Node FFI set run_cli on library import, not at argv parse time
+    if let Err(e) = env_logger::Builder::from_env(
+        env_logger::Env::new()
+            .filter_or("BAML_LOG", "info")
+            .write_style("BAML_LOG_STYLE"),
+    )
+    // .format_target(false)
+    .try_init()
+    {
+        eprintln!("Failed to initialize BAML logger: {:#}", e);
+    };
+
     let argv: Vec<String> = std::env::args().collect();
 
     BamlRuntime::run_cli(

engine/baml-runtime/src/runtime/runtime_interface.rs

@@ -396,26 +396,34 @@ impl RuntimeInterface for InternalBamlRuntime {
                 ))
             }
         };
-        let baml_args = match self.ir().check_function_params(
+        let baml_args = self.ir().check_function_params(
             &func,
             &params,
             ArgCoercer {
                 span_path: None,
                 allow_implicit_cast_to_string: false,
             },
-        ) {
-            Ok(args) => args,
-            Err(e) => {
-                return Ok(FunctionResult::new(
-                    OrchestrationScope::default(),
-                    LLMResponse::UserFailure(format!(
-                        "Failed while validating args for {function_name}: {:?}",
-                        e
-                    )),
-                    None,
-                ))
-            }
-        };
+        )?;
+        // let baml_args = match self.ir().check_function_params(
+        //     &func,
+        //     &params,
+        //     ArgCoercer {
+        //         span_path: None,
+        //         allow_implicit_cast_to_string: false,
+        //     },
+        // ) {
+        //     Ok(args) => args,
+        //     Err(e) => {
+        //         return Ok(FunctionResult::new(
+        //             OrchestrationScope::default(),
+        //             LLMResponse::UserFailure(format!(
+        //                 "Failed while validating args for {function_name}: {:?}",
+        //                 e
+        //             )),
+        //             None,
+        //         ))
+        //     }
+        // };
 
         let renderer = PromptRenderer::from_function(&func, self.ir(), &ctx)?;
         let orchestrator = self.orchestration_graph(renderer.client_spec(), &ctx)?;

engine/baml-runtime/tests/harness.rs

@@ -1,7 +1,7 @@
 use assert_cmd::prelude::*;
 
-use anyhow::{anyhow, Result};
-use cfg_if::cfg_if;
+use anyhow::Result;
+use indoc::indoc;
 use std::{any, path::PathBuf, process::Command};
 
 #[derive(Debug, Eq, PartialEq)]
@@ -17,7 +17,11 @@ pub struct Harness {
 impl Harness {
     pub fn new<S: AsRef<str>>(test_name: S) -> Result<Self> {
         if std::env::var("OPENAI_API_KEY").is_err() || std::env::var("ANTHROPIC_API_KEY").is_err() {
-            anyhow::bail!("Integration tests using tests/harness.rs require OPENAI_API_KEY and ANTHROPIC_API_KEY; you can skip these using 'cargo test --lib'");
+            anyhow::bail!(indoc! {"
+                Integration tests using tests/harness.rs require OPENAI_API_KEY and ANTHROPIC_API_KEY.
+
+                You can skip these using 'cargo test --lib', or run these specifically using 'cargo test --test test_cli'.
+            "});
         }
 
         // Run the CLI in /tmp/baml-runtime-test-harness/test_name.

engine/baml-runtime/tests/test_cli.rs

@@ -233,4 +233,72 @@ mod test_cli {
 
         Ok(())
     }
+
+    #[rstest]
+    #[tokio::test]
+    async fn invalid_arg() -> Result<()> {
+        let h = Harness::new(format!("invalid_arg_test"))?;
+
+        const PORT: &str = "2035";
+
+        let run = h.run_cli("init")?.output()?;
+        assert_eq!(run.status.code(), Some(0));
+
+        let mut child = h
+            .run_cli(format!("serve --preview --port {PORT}"))?
+            .spawn()?;
+        defer! { let _ = child.kill(); }
+
+        assert!(
+            reqwest::get(&format!("http://localhost:{PORT}/_debug/ping"))
+                .await?
+                .status()
+                .is_success()
+        );
+
+        let resume = indoc! {"
+      Vaibhav Gupta
+      [email protected]
+
+      Experience:
+      - Founder at BoundaryML
+      - CV Engineer at Google
+      - CV Engineer at Microsoft
+
+      Skills:
+      - Rust
+      - C++
+    "};
+        let resp = reqwest::Client::new()
+            .post(&format!("http://localhost:{PORT}/call/ExtractResume"))
+            .json(&json!({ "resume": resume }))
+            .send()
+            .await?;
+        assert_eq!(resp.status(), StatusCode::OK);
+
+        let resp = reqwest::Client::new()
+            .post(&format!("http://localhost:{PORT}/call/ExtractResume"))
+            .json(&json!({ "not-resume": resume }))
+            .send()
+            .await?;
+        assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+
+        let resp = reqwest::Client::new()
+            .post(&format!("http://localhost:{PORT}/call/ExtractResume"))
+            .json(&json!({ "resume": { "not": "string" } }))
+            .send()
+            .await?;
+        assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+
+        let resp = reqwest::Client::new()
+            .post(&format!(
+                "http://localhost:{PORT}/call/ExtractResumeNonexistent"
+            ))
+            .json(&json!({ "resume": resume }))
+            .send()
+            .await?;
+        assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
+
+        Ok(())
+    }
 }

engine/language_client_python/pyproject.toml

@@ -16,4 +16,4 @@ python-source = "python_src"
 features = ["pyo3/extension-module"]
 
 [project.scripts]
-baml-cli = "baml_py:invoke_runtime_cli"
+baml-cli = "baml_cli:invoke_runtime_cli"

engine/language_client_python/python_src/baml_cli.py

@@ -0,0 +1,10 @@
+import os
+
+if "BAML_LOG" not in os.environ:
+    os.environ["BAML_LOG"] = "info"
+
+
+def invoke_runtime_cli():
+    import baml_py
+
+    baml_py.invoke_runtime_cli()

engine/language_client_typescript/cli.js

@@ -1,6 +1,10 @@
 #!/usr/bin/env node
 
 if (require.main === module) {
+  if (!process.env.BAML_LOG) {
+    process.env.BAML_LOG = 'info'
+  }
+
   const baml = require('./native')
   baml.invoke_runtime_cli(process.argv.slice(1))
 }

integ-tests/baml_src/test-files/functions/input/named-args/single/named-boolean.baml

@@ -1,7 +1,7 @@
 
 
 function TestFnNamedArgsSingleBool(myBool: bool) -> string{
-  client Vertex
+  client GPT35
   prompt #"
     Return this value back to me: {{myBool}}
   "#

integ-tests/python/baml_client/inlinedbaml.py

@@ -33,7 +33,7 @@
     "test-files/dynamic/client-registry.baml": "// Intentionally use a bad key\nclient<llm> BadClient {\n  provider openai\n  options {\n    model \"gpt-3.5-turbo\"\n    api_key \"sk-invalid\"\n  }\n}\n\nfunction ExpectFailure() -> string {\n  client BadClient\n\n  prompt #\"\n    What is the capital of England?\n  \"#\n}\n",
     "test-files/dynamic/dynamic.baml": "class DynamicClassOne {\n  @@dynamic\n}\n\nenum DynEnumOne {\n  @@dynamic\n}\n\nenum DynEnumTwo {\n  @@dynamic\n}\n\nclass SomeClassNestedDynamic {\n  hi string\n  @@dynamic\n\n}\n\nclass DynamicClassTwo {\n  hi string\n  some_class SomeClassNestedDynamic\n  status DynEnumOne\n  @@dynamic\n}\n\nfunction DynamicFunc(input: DynamicClassOne) -> DynamicClassTwo {\n  client GPT35\n  prompt #\"\n    Please extract the schema from \n    {{ input }}\n\n    {{ ctx.output_format }}\n  \"#\n}\n\nclass DynInputOutput {\n  testKey string\n  @@dynamic\n}\n\nfunction DynamicInputOutput(input: DynInputOutput) -> DynInputOutput {\n  client GPT35\n  prompt #\"\n    Here is some input data:\n    ----\n    {{ input }}\n    ----\n\n    Extract the information.\n    {{ ctx.output_format }}\n  \"#\n}\n\nfunction DynamicListInputOutput(input: DynInputOutput[]) -> DynInputOutput[] {\n  client GPT35\n  prompt #\"\n    Here is some input data:\n    ----\n    {{ input }}\n    ----\n\n    Extract the information.\n    {{ ctx.output_format }}\n  \"#\n}\n\n\n\nclass DynamicOutput {\n  @@dynamic\n}\n \nfunction MyFunc(input: string) -> DynamicOutput {\n  client GPT35\n  prompt #\"\n    Given a string, extract info using the schema:\n\n    {{ input}}\n\n    {{ ctx.output_format }}\n  \"#\n}\n\n",
     "test-files/functions/input/named-args/single/named-audio.baml": "function AudioInput(aud: audio) -> string{\n  client Gemini\n  prompt #\"\n    {{ _.role(\"user\") }}\n\n   Does this sound like a roar? Yes or no? One word no other characters.\n    \n    {{ aud }}\n  \"#\n}\n\n\ntest TestURLAudioInput{\n  functions [AudioInput]\n  args {\n    aud{ \n      url https://actions.google.com/sounds/v1/emergency/beeper_emergency_call.ogg\n    }\n  }  \n}\n\n\n",
-    "test-files/functions/input/named-args/single/named-boolean.baml": "\n\nfunction TestFnNamedArgsSingleBool(myBool: bool) -> string{\n  client Vertex\n  prompt #\"\n    Return this value back to me: {{myBool}}\n  \"#\n}\n\ntest TestFnNamedArgsSingleBool {\n  functions [TestFnNamedArgsSingleBool]\n  args {\n    myBool true\n  }\n}",
+    "test-files/functions/input/named-args/single/named-boolean.baml": "\n\nfunction TestFnNamedArgsSingleBool(myBool: bool) -> string{\n  client GPT35\n  prompt #\"\n    Return this value back to me: {{myBool}}\n  \"#\n}\n\ntest TestFnNamedArgsSingleBool {\n  functions [TestFnNamedArgsSingleBool]\n  args {\n    myBool true\n  }\n}",
     "test-files/functions/input/named-args/single/named-class-list.baml": "\n\n\nfunction TestFnNamedArgsSingleStringList(myArg: string[]) -> string{\n  client GPT35\n  prompt #\"\n    Return this value back to me: {{myArg}}\n  \"#\n}\n\ntest TestFnNamedArgsSingleStringList {\n  functions [TestFnNamedArgsSingleStringList]\n  args {\n    myArg [\"hello\", \"world\"]\n  }\n}",
     "test-files/functions/input/named-args/single/named-class.baml": "class NamedArgsSingleClass {\n  key string\n  key_two bool\n  key_three int\n  // TODO: doesn't work with keys with numbers\n  // key2 bool\n  // key3 int\n}\n\nfunction TestFnNamedArgsSingleClass(myArg: NamedArgsSingleClass) -> string {\n  client GPT35\n  prompt #\"\n    Print these values back to me:\n    {{myArg.key}}\n    {{myArg.key_two}}\n    {{myArg.key_three}}\n  \"#\n}\n\ntest TestFnNamedArgsSingleClass {\n  functions [TestFnNamedArgsSingleClass]\n  args {\n    myArg {\n      key \"example\",\n      key_two true,\n      key_three 42\n    }\n  }\n}\n\nfunction TestMulticlassNamedArgs(myArg: NamedArgsSingleClass, myArg2: NamedArgsSingleClass) -> string {\n  client GPT35\n  prompt #\"\n    Print these values back to me:\n    {{myArg.key}}\n    {{myArg.key_two}}\n    {{myArg.key_three}}\n    {{myArg2.key}}\n    {{myArg2.key_two}}\n    {{myArg2.key_three}}\n  \"#\n}",
     "test-files/functions/input/named-args/single/named-enum-list.baml": "enum NamedArgsSingleEnumList {\n  ONE\n  TWO\n}\n\nfunction TestFnNamedArgsSingleEnumList(myArg: NamedArgsSingleEnumList[]) -> string {\n  client GPT35\n  prompt #\"\n    Print these values back to me:\n    {{myArg}}\n  \"#\n}\n\ntest TestFnNamedArgsSingleEnumList {\n  functions [TestFnNamedArgsSingleEnumList]\n  args {\n    myArg [ONE, TWO]\n  }\n}",

integ-tests/python/tests/test_functions.py

@@ -971,6 +971,7 @@ async def test_descriptions():
     assert res.parens == "parens1"
     assert res.other_group == "other"
 
+
 @pytest.mark.asyncio
 async def test_caching():
     story_idea = """
@@ -993,17 +994,26 @@ async def test_caching():
     print("Duration no caching: ", duration)
     print("Duration with caching: ", duration2)
 
-    assert duration2 < duration, "Expected second call to be faster than first by a large margin."
+    assert (
+        duration2 < duration
+    ), "Expected second call to be faster than first by a large margin."
+
 
 @pytest.mark.asyncio
 async def test_arg_exceptions():
-    with pytest.raises(errors.BamlInvalidArgumentError):
-        _ = await b.TestCaching(111) # intentionally passing an int instead of a string
+    with pytest.raises(IndexError):
+        print("this should fail:", [0, 1, 2][5])
 
+    with pytest.raises(errors.BamlInvalidArgumentError):
+        _ = await b.TestCaching(
+            111
+        )  # ldintentionally passing an int instead of a string
 
     with pytest.raises(errors.BamlClientError):
         cr = baml_py.ClientRegistry()
-        cr.add_llm_client("MyClient", "openai", {"model": "gpt-4o-mini", "api_key": "INVALID_KEY"})
+        cr.add_llm_client(
+            "MyClient", "openai", {"model": "gpt-4o-mini", "api_key": "INVALID_KEY"}
+        )
         cr.set_primary("MyClient")
         await b.MyFunc(
             input="My name is Harrison. My hair is black and I'm 6 feet tall.",
@@ -1012,7 +1022,9 @@ async def test_arg_exceptions():
 
     with pytest.raises(errors.BamlClientHttpError):
         cr = baml_py.ClientRegistry()
-        cr.add_llm_client("MyClient", "openai", {"model": "gpt-4o-mini", "api_key": "INVALID_KEY"})
+        cr.add_llm_client(
+            "MyClient", "openai", {"model": "gpt-4o-mini", "api_key": "INVALID_KEY"}
+        )
         cr.set_primary("MyClient")
         await b.MyFunc(
             input="My name is Harrison. My hair is black and I'm 6 feet tall.",
@@ -1022,7 +1034,10 @@ async def test_arg_exceptions():
     with pytest.raises(errors.BamlValidationError):
         await b.DummyOutputFunction("dummy input")
 
+
 @pytest.mark.asyncio
 async def test_map_as_param():
     with pytest.raises(errors.BamlInvalidArgumentError):
-        _ = await b.TestFnNamedArgsSingleMapStringToMap({ "a" : "b"}) # intentionally passing the wrong type
+        _ = await b.TestFnNamedArgsSingleMapStringToMap(
+            {"a": "b"}
+        )  # intentionally passing the wrong type