Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixed the deserializer when it comes to massive markdown #677

Merged
merged 3 commits into from
Jun 13, 2024
Merged

Fixed the deserializer when it comes to massive markdown #677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6b38241
Fixed the deserializer when it comes to massive markdown
hellovai Jun 13, 2024
f4ad677
Patch format
hellovai Jun 13, 2024
b508571
Updating inlined code
hellovai Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions engine/baml-lib/jsonish/src/jsonish/parser/fixing_parser/json_parse_state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,35 +168,43 @@ impl JsonParseState {
if let Some((_, next_c)) = next.peek() {
match next_c {
'\n' => {
log::debug!("Closing due to: newline after comma");
return Some(idx);
}
' ' => {
log::debug!("Testing for comment after space + comma");
// If after the space we have "//" or "/*" or the beginning of a key, we'll close the string
while let Some((_, c)) = next.next() {
match c {
' ' => {}
'\n' => {
return Some(idx);
}
let mut buffer = ",".to_string();
while let Some((_, next_next_c)) = next.next() {
buffer.push(next_next_c);
match next_next_c {
' ' | '\n' => {}
'/' => match next.peek() {
Some((_, '/')) => {
// This is likely a comment
return Some(idx);
}
Some((_, '*')) => {
// This is likely a comment
return Some(idx);
}
_ => {
let _ = self.consume(c);
// let _ = self.consume(c);
}
},
'"' => {
// This is likely a new key
log::debug!("Closing due to: new key after space + comma");
return Some(idx);
}
x => {
let _ = self.consume(x);
break;
}
}
}
for c in buffer.chars() {
let _ = self.consume(c);
}
}
_ => {
let _ = self.consume(c);
Expand Down
2 changes: 1 addition & 1 deletion integ-tests/python/baml_client/inlinedbaml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
"fiddle-examples/extract-names.baml": "function ExtractNames(input: string) -> string[] {\n client GPT4\n prompt #\"\n Extract the names from this INPUT:\n \n INPUT:\n ---\n {{ input }}\n ---\n\n {{ ctx.output_format }}\n\n Response:\n \"#\n}\n",
"fiddle-examples/images/image.baml": "function DescribeImage(img: image) -> string {\n client GPT4Turbo\n prompt #\"\n {{ _.role(\"user\") }}\n\n\n Describe the image below in 5 words:\n {{ img }}\n \"#\n\n}\n\nclass FakeImage {\n url string\n}\n\nclass ClassWithImage {\n myImage image\n param2 string\n fake_image FakeImage\n}\n\n// chat role user present\nfunction DescribeImage2(classWithImage: ClassWithImage, img2: image) -> string {\n client GPT4Turbo\n prompt #\"\n {{ _.role(\"user\") }}\n You should return 2 answers that answer the following commands.\n\n 1. Describe this in 5 words:\n {{ classWithImage.myImage }}\n\n 2. Also tell me what's happening here in one sentence:\n {{ img2 }}\n \"#\n}\n\n// no chat role\nfunction DescribeImage3(classWithImage: ClassWithImage, img2: image) -> string {\n client GPT4Turbo\n prompt #\"\n Describe this in 5 words:\n {{ classWithImage.myImage }}\n\n Tell me also what's happening here in one sentence and relate it to the word {{ classWithImage.param2 }}:\n {{ img2 }}\n \"#\n}\n\n\n// system prompt and chat prompt\nfunction DescribeImage4(classWithImage: ClassWithImage, img2: image) -> string {\n client GPT4Turbo\n prompt #\"\n {{ _.role(\"system\")}}\n\n Describe this in 5 words:\n {{ classWithImage.myImage }}\n\n Tell me also what's happening here in one sentence and relate it to the word {{ classWithImage.param2 }}:\n {{ img2 }}\n \"#\n}",
"fiddle-examples/symbol-tuning.baml": "enum Category3 {\n Refund @alias(\"k1\")\n @description(\"Customer wants to refund a product\")\n\n CancelOrder @alias(\"k2\")\n @description(\"Customer wants to cancel an order\")\n\n TechnicalSupport @alias(\"k3\")\n @description(\"Customer needs help with a technical issue unrelated to account creation or login\")\n\n AccountIssue @alias(\"k4\")\n @description(\"Specifically relates to account-login or account-creation\")\n\n Question @alias(\"k5\")\n @description(\"Customer has a question\")\n}\n\nfunction ClassifyMessage3(input: string) -> Category {\n client GPT4\n\n prompt #\"\n Classify the following INPUT into ONE\n of the following categories:\n\n INPUT: {{ input }}\n\n {{ ctx.output_format }}\n\n Response:\n \"#\n}",
"main.baml": "generator lang_python {\n output_type python/pydantic\n output_dir \"../python\"\n}\n\ngenerator lang_typescript {\n output_type typescript\n output_dir \"../typescript\"\n}\n\n// generator lang_ruby {\n// output_type ruby/sorbet\n// output_dir \"../ruby\"\n// }\n",
"main.baml": "generator lang_python {\n output_type python/pydantic\n output_dir \"../python\"\n}\n\ngenerator lang_typescript {\n output_type typescript\n output_dir \"../typescript\"\n}\n\ngenerator lang_ruby {\n output_type ruby/sorbet\n output_dir \"../ruby\"\n}\n",
"test-files/aliases/classes.baml": "class TestClassAlias {\n key string @alias(\"key-dash\") @description(#\"\n This is a description for key\n af asdf\n \"#)\n key2 string @alias(\"key21\")\n key3 string @alias(\"key with space\")\n key4 string //unaliased\n key5 string @alias(\"key.with.punctuation/123\")\n}\n\nfunction FnTestClassAlias(input: string) -> TestClassAlias {\n client GPT35\n prompt #\"\n {{ctx.output_format}}\n \"#\n}\n\ntest FnTestClassAlias {\n functions [FnTestClassAlias]\n args {\n input \"example input\"\n }\n}\n",
"test-files/aliases/enums.baml": "enum TestEnum {\n A @alias(\"k1\") @description(#\"\n User is angry\n \"#)\n B @alias(\"k22\") @description(#\"\n User is happy\n \"#)\n // tests whether k1 doesnt incorrectly get matched with k11\n C @alias(\"k11\") @description(#\"\n User is sad\n \"#)\n D @alias(\"k44\") @description(\n User is confused\n )\n E @description(\n User is excited\n )\n F @alias(\"k5\") // only alias\n \n G @alias(\"k6\") @description(#\"\n User is bored\n With a long description\n \"#)\n \n @@alias(\"Category\")\n}\n\nfunction FnTestAliasedEnumOutput(input: string) -> TestEnum {\n client GPT35\n prompt #\"\n Classify the user input into the following category\n \n {{ ctx.output_format }}\n\n {{ _.role('user') }}\n {{input}}\n\n {{ _.role('assistant') }}\n Category ID:\n \"#\n}\n\ntest FnTestAliasedEnumOutput {\n functions [FnTestAliasedEnumOutput]\n args {\n input \"mehhhhh\"\n }\n}",
"test-files/comments/comments.baml": "// add some functions, classes, enums etc with comments all over.",
Expand Down
Loading
Loading