From 2b174dd9c552693052aaedc1d1814326165fda77 Mon Sep 17 00:00:00 2001
From: Clint Herron <hanclinto@gmail.com>
Date: Wed, 5 Jun 2024 19:28:13 -0700
Subject: [PATCH 1/7] Adding simple bare-bones test for end-to-end integration
 test for json validation against auto-generated JSON-schema grammars.

---
 Makefile                           |  2 +-
 tests/test-grammar-integration.cpp | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index dddf647cd551d..4ea59c0b4ef29 100644
--- a/Makefile
+++ b/Makefile
@@ -1051,7 +1051,7 @@ tests/test-grammar-parser: tests/test-grammar-parser.cpp ggml.o llama.o grammar-
 	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
 
-tests/test-grammar-integration: tests/test-grammar-integration.cpp ggml.o llama.o grammar-parser.o $(OBJS)
+tests/test-grammar-integration: tests/test-grammar-integration.cpp json-schema-to-grammar.o ggml.o llama.o grammar-parser.o $(OBJS)
 	$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
 	$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
 
diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index 8787fb1ec6987..000bf426de951 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -7,6 +7,7 @@
 #include "ggml.h"
 #include "llama.h"
 #include "grammar-parser.h"
+#include "json-schema-to-grammar.h"
 #include "unicode.h"
 #include <cassert>
 #include <string>
@@ -468,6 +469,31 @@ empty ::= "blah" | )""";
     fprintf(stderr, "  ✅︎ Passed\n");
 }
 
+static void test_json_schema() {
+    // Note that this is similar to the regular grammar tests,
+    //  but we convert each json schema to a grammar before parsing.
+    // Otherwise, this test structure is the same.
+
+    test_grammar(
+        "empty schema",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{}
+            )"""
+            )),
+        // Passing strings
+        {
+            "{}",
+            "{\"foo\": \"bar\"}",
+        },
+        // Failing strings
+        {
+            "",
+        }
+    );
+}
+
 int main() {
     fprintf(stdout, "Running grammar integration tests...\n");
     test_simple_grammar();
@@ -477,6 +503,7 @@ int main() {
     test_failure_missing_root();
     test_failure_missing_reference();
     test_failure_left_recursion();
+    test_json_schema();
     fprintf(stdout, "All tests passed.\n");
     return 0;
 }

From 74985def8016cd77884ea2db7bc408f9dca91e22 Mon Sep 17 00:00:00 2001
From: Clint Herron <hanclinto@gmail.com>
Date: Wed, 5 Jun 2024 22:29:25 -0700
Subject: [PATCH 2/7] Adding additional examples as documented in #7789 . Also
 adding the ability to automatically output improperly failing grammars to
 debug output files so they can more easily be examined in the gbnf-validator
 program.

---
 tests/test-grammar-integration.cpp | 549 ++++++++++++++++++++++++++++-
 1 file changed, 548 insertions(+), 1 deletion(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index 000bf426de951..740828e69f76b 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -86,6 +86,23 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
 
         if (!matched) {
             fprintf(stderr, "❌ (failed to match)\n");
+
+            // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
+            // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
+            FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");
+            if (grammar_file) {
+                fprintf(grammar_file, "%s", grammar_str.c_str());
+                fclose(grammar_file);
+            }
+
+            // DEBUG: Write the test string to test-grammar-integration.string.txt
+            FILE* string_file = fopen("test-grammar-integration.string.txt", "w");
+            if (string_file) {
+                fprintf(string_file, "%s", test_string.c_str());
+                fclose(string_file);
+            }
+
+            fprintf(stderr, "    Analyze in detail by running: `./gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt`\n");
         } else {
             fprintf(stdout, "✅︎\n");
         }
@@ -475,7 +492,7 @@ static void test_json_schema() {
     // Otherwise, this test structure is the same.
 
     test_grammar(
-        "empty schema",
+        "empty schema (object)",
         // Grammar
         json_schema_to_grammar(nlohmann::ordered_json::parse(
             R"""(
@@ -492,6 +509,536 @@ static void test_json_schema() {
             "",
         }
     );
+
+    test_grammar(
+        "exotic formats (list)",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+  "items": [
+    { "format": "date" },
+    { "format": "uuid" },
+    { "format": "time" },
+    { "format": "date-time" }
+  ]
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            // "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
+            // "[]", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
+            R"""(["2012-04-23", "12345678-1234-1234-1234-1234567890ab", "18:25:43.511Z", "2012-04-23T18:25:43.511Z"])""",
+            //R"""(["2012-04-23","12345678-1234-1234-1234-1234567890ab"])""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
+            //R"""({"foo": "bar"})""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
+        },
+        // Failing strings
+        {
+            R"""(["foo", "bar"])""",
+            R"""(["12345678-1234-1234-1234-1234567890ab"])""",
+        }
+    );
+
+    test_grammar(
+        "string",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "string"
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+            "\"bar\"",
+            "\"\"",
+        },
+        // Failing strings
+        {
+            "{}",
+            "\"foo\": \"bar\"",
+        }
+    );
+
+    test_grammar(
+        "string w/ min length 1",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "string",
+    "minLength": 1
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+            "\"bar\"",
+        },
+        // Failing strings
+        {
+            "\"\"",
+            "{}",
+            "\"foo\": \"bar\"",
+        }
+    );
+
+    test_grammar(
+        "string w/ min length 3",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "string",
+    "minLength": 3
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+            "\"bar\"",
+            "\"foobar\"",
+        },
+        // Failing strings
+        {
+            "\"\"",
+            "\"f\"",
+            "\"fo\"",
+        }
+    );
+
+    test_grammar(
+        "string w/ max length",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "string",
+    "maxLength": 3
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+            "\"bar\"",
+            "\"\"",
+            "\"f\"",
+            "\"fo\"",
+        },
+        // Failing strings
+        {
+            "\"foobar\"",
+        }
+    );
+
+    test_grammar(
+        "string w/ min & max length",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "string",
+    "minLength": 1,
+    "maxLength": 4
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+            "\"bar\"",
+            "\"f\"",
+            "\"barf\"",
+        },
+        // Failing strings
+        {
+            "\"\"",
+            "\"barfo\"",
+            "\"foobar\"",
+        }
+    );
+
+    test_grammar(
+        "boolean",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "boolean"
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "true",
+            "false",
+        },
+        // Failing strings
+        {
+            "\"\"",
+            "\"true\"",
+            "True",
+            "FALSE",
+        }
+    );
+
+    test_grammar(
+        "integer",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "type": "integer"
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "0",
+            "12345",
+            "1234567890123456"
+        },
+        // Failing strings
+        {
+            "",
+            "01",
+            "007",
+            "12345678901234567"
+        }
+    );
+
+    test_grammar(
+        "string const",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "const": "foo"
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"foo\"",
+        },
+        // Failing strings
+        {
+            "foo",
+            "\"bar\"",
+        }
+    );
+
+    test_grammar(
+        "non-string const",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "const": true
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "true",
+        },
+        // Failing strings
+        {
+            "",
+            "foo",
+            "\"true\"",
+        }
+    );
+
+    test_grammar(
+        "non-string const",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "enum": ["red", "amber", "green", null, 42, ["foo"]]
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "\"red\"",
+            "null",
+            "42",
+            "[\"foo\"]",
+        },
+        // Failing strings
+        {
+            "",
+            "420",
+            "true",
+            "foo",
+        }
+    );
+
+
+    test_grammar(
+        "min+max items",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "items": {
+        "type": ["number", "integer"]
+    },
+    "minItems": 3,
+    "maxItems": 5
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "[1, 2, 3]",
+            "[1, 2, 3, 4]",
+            "[1, 2, 3, 4, 5]",
+        },
+        // Failing strings
+        {
+            "[1, 2]",
+            "[1, 2, 3, 4, 5, 6]",
+            "1"
+        }
+    );
+
+    // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
+    test_grammar(
+        "object properties",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+  "type": "object",
+  "properties": {
+    "number": { "type": "number" },
+    "street_name": { "type": "string" },
+    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+  }
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+            // "By default, leaving out properties is valid"
+            R"""({ "street_name": "Pennsylvania" })""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
+            // "By extension, even an empty object is valid"
+            R"""({})""",
+            // "By default, providing additional properties is valid"
+            // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
+            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
+            // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
+            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+        },
+        // Failing strings
+        {
+            // Change datatype from number to string
+            R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+            // Reorder properties
+            R"""({ "street_name": "Pennsylvania", "number": 1600 })""",
+            // Reorder properties
+            R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+        }
+    );
+
+
+    // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
+    test_grammar(
+        "object properties, additionalProperties: true",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+  "type": "object",
+  "properties": {
+    "number": { "type": "number" },
+    "street_name": { "type": "string" },
+    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+  },
+  "additionalProperties": true
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            //R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",
+            // "By default, leaving out properties is valid"
+            //R"""({ "street_name": "Pennsylvania" })""",
+            //R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
+            // "By extension, even an empty object is valid"
+            R"""({})""",
+            // "By default, providing additional properties is valid"
+            // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
+            //R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
+            // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
+            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+        },
+        // Failing strings
+        {
+            // Change datatype from number to string
+            R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+            // Reorder properties
+            R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",
+        }
+    );
+
+    // Additional properties: false
+    test_grammar(
+        "required + optional props each in original order",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+  "type": "object",
+  "properties": {
+    "number": { "type": "number" },
+    "street_name": { "type": "string" },
+    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+  },
+  "additionalProperties": false
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            R"""({ "street_name": "Pennsylvania" })""",
+            R"""({ "number": 1600, "street_type":"Avenue"})""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+            // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
+            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+        },
+        // Failing strings
+        {
+            // Reorder properties
+            R"""({ "street_type": "Avenue", "number": 1600 })""",
+            // Add "direction"
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",
+        }
+    );
+
+    test_grammar(
+        "required + optional props each in original order",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+    "properties": {
+        "b": {"type": "string"},
+        "a": {"type": "string"},
+        "d": {"type": "string"},
+        "c": {"type": "string"}
+    },
+    "required": ["a", "b"],
+    "additionalProperties": false
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "{\"b\": \"foo\", \"a\": \"bar\"}",
+            "{\"b\":\"foo\",\"a\":\"bar\",\"d\":\"qux\"}",
+            "{\"b\":\"foo\", \"a\":\"bar\", \"d\":\"qux\", \"c\":\"baz\"}",
+        },
+        // Failing strings
+        {
+            "{\"a\": \"foo\", \"b\": \"bar\"}",
+            "{\"b\": \"bar\"}",
+            "{\"a\": \"foo\", \"c\": \"baz\"}",
+            "{\"a\":\"foo\", \"b\":\"bar\", \"c\":\"baz\", \"d\":\"qux\"}",
+        }
+    );
+
+    // NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties
+    test_grammar(
+        "required props",
+        // Grammar
+        json_schema_to_grammar(nlohmann::ordered_json::parse(
+            R"""(
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://example.com/product.schema.json",
+  "title": "Product",
+  "description": "A product from Acme's catalog",
+  "type": "object",
+  "properties": {
+    "productId": {
+      "description": "The unique identifier for a product",
+      "type": "integer"
+    },
+    "productName": {
+      "description": "Name of the product",
+      "type": "string"
+    },
+    "price": {
+      "description": "The price of the product",
+      "type": "number",
+      "exclusiveMinimum": 0
+    },
+    "tags": {
+      "description": "Tags for the product",
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "minItems": 1,
+      "uniqueItems": true
+    },
+    "dimensions": {
+      "type": "object",
+      "properties": {
+        "length": {
+          "type": "number"
+        },
+        "width": {
+          "type": "number"
+        },
+        "height": {
+          "type": "number"
+        }
+      },
+      "required": [ "length", "width", "height" ]
+    }
+  },
+  "required": [ "productId", "productName", "price" ]
+}
+            )"""
+            )),
+        // Passing strings
+        {
+            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50}",
+            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\"]}",
+            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\"], \"dimensions\": {\"length\": 785, \"width\": 250.5, \"height\": -0.359}}",
+        },
+        // Failing strings
+        {
+            "{}", // Missing all required properties
+            "{\"productName\": \"A green door\", \"price\": 12.50, \"productId\": 1}", // Out of order properties
+            // TODO: The following line should fail, but currently it passes. `exclusiveMinimum` is not supported, as it would likely be too difficult to implement.
+            //  Perhaps special checks for minimum and maximum values of 0 could be added (since that's relatively easy to do with grammars), but anything else would likely be too complex.
+            // "{\"productId\": 1, \"productName\": \"A green door\", \"price\": -12.50}",
+            "{\"productId\": 1, \"productName\": \"A green door\"}", // Missing required property (price)
+            "{\"productName\": \"A green door\", \"price\": 12.50}", // Missing required property (productId)
+            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": []}", // tags is empty, but minItems is 1
+            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"dimensions\": {\"length\": 785, \"width\": 250.5, \"height\": -0.359}, \"tags\": [\"home\", \"green\"]}", // Tags and dimensions are out of order
+            // TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.
+            // "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\", \"home\"]}",
+
+        }
+    );
+
+
 }
 
 int main() {

From acd3c468af375280b331bc9a0ec0731209bf52da Mon Sep 17 00:00:00 2001
From: Clint Herron <hanclinto@gmail.com>
Date: Wed, 5 Jun 2024 22:41:03 -0700
Subject: [PATCH 3/7] Uncommenting formerly commented tests so that they fail
 for others who are attempting to reproduce the bugs.

---
 tests/test-grammar-integration.cpp | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index 740828e69f76b..acb197dc3ea87 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -102,7 +102,7 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
                 fclose(string_file);
             }
 
-            fprintf(stderr, "    Analyze in detail by running: `./gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt`\n");
+            fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command:     ./gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
         } else {
             fprintf(stdout, "✅︎\n");
         }
@@ -837,9 +837,9 @@ static void test_json_schema() {
             R"""({})""",
             // "By default, providing additional properties is valid"
             // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
-            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
             // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
-            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
         },
         // Failing strings
         {
@@ -872,17 +872,20 @@ static void test_json_schema() {
             )),
         // Passing strings
         {
-            //R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",
+            // TODO: Following line should pass and doesn't
+            R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",
             // "By default, leaving out properties is valid"
-            //R"""({ "street_name": "Pennsylvania" })""",
-            //R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
+            // TODO: Following line should pass and doesn't
+            R"""({ "street_name": "Pennsylvania" })""",
+            // TODO: Following line should pass and doesn't
+            R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
             // "By extension, even an empty object is valid"
             R"""({})""",
             // "By default, providing additional properties is valid"
             // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
-            //R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
             // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
-            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
         },
         // Failing strings
         {

From d4a63b0538df3b4fc6afdf131603e4369fc14e5c Mon Sep 17 00:00:00 2001
From: HanClinto <hanclinto@gmail.com>
Date: Wed, 12 Jun 2024 10:42:37 -0700
Subject: [PATCH 4/7] Merging improved schema test methods added by @ochafik in
 #7797

---
 tests/test-grammar-integration.cpp | 194 ++++++++++++-----------------
 1 file changed, 82 insertions(+), 112 deletions(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index acb197dc3ea87..cd00b97256b5f 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -13,6 +13,8 @@
 #include <string>
 #include <vector>
 
+using json = nlohmann::ordered_json;
+
 static llama_grammar* build_grammar(const std::string & grammar_str) {
     auto parsed_grammar = grammar_parser::parse(grammar_str.c_str());
 
@@ -66,8 +68,8 @@ static bool match_string(const std::string & input, llama_grammar* grammar) {
     return false;
 }
 
-static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
-    fprintf(stderr, "⚫ Testing %s. Grammar: %s\n", test_desc.c_str(), grammar_str.c_str());
+static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
+    fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
     fflush(stderr);
 
     auto grammar = build_grammar(grammar_str);
@@ -136,6 +138,12 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
     // Clean up allocated memory
     llama_grammar_free(grammar);
 }
+static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
+    test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);
+}
+static void test_schema(const std::string & test_desc, const std::string & schema_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
+    test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str)), passing_strings, failing_strings);
+}
 
 static void test_simple_grammar() {
     // Test case for a simple grammar
@@ -491,14 +499,12 @@ static void test_json_schema() {
     //  but we convert each json schema to a grammar before parsing.
     // Otherwise, this test structure is the same.
 
-    test_grammar(
+    test_schema(
         "empty schema (object)",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {}
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "{}",
@@ -510,11 +516,10 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "exotic formats (list)",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
   "items": [
     { "format": "date" },
@@ -523,8 +528,7 @@ static void test_json_schema() {
     { "format": "date-time" }
   ]
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             // "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
@@ -540,16 +544,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "string"
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -563,17 +565,15 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string w/ min length 1",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "string",
     "minLength": 1
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -587,17 +587,15 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string w/ min length 3",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "string",
     "minLength": 3
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -612,17 +610,15 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string w/ max length",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "string",
     "maxLength": 3
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -637,18 +633,16 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string w/ min & max length",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "string",
     "minLength": 1,
     "maxLength": 4
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -664,16 +658,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "boolean",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "boolean"
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "true",
@@ -688,16 +680,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "integer",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "type": "integer"
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "0",
@@ -713,16 +703,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "string const",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "const": "foo"
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"foo\"",
@@ -734,16 +722,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "non-string const",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "const": true
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "true",
@@ -756,16 +742,14 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "non-string const",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "enum": ["red", "amber", "green", null, 42, ["foo"]]
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "\"red\"",
@@ -783,11 +767,10 @@ static void test_json_schema() {
     );
 
 
-    test_grammar(
+    test_schema(
         "min+max items",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "items": {
         "type": ["number", "integer"]
@@ -795,8 +778,7 @@ static void test_json_schema() {
     "minItems": 3,
     "maxItems": 5
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "[1, 2, 3]",
@@ -812,11 +794,10 @@ static void test_json_schema() {
     );
 
     // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
-    test_grammar(
+    test_schema(
         "object properties",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
   "type": "object",
   "properties": {
@@ -825,8 +806,7 @@ static void test_json_schema() {
     "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
   }
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
@@ -854,11 +834,10 @@ static void test_json_schema() {
 
 
     // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
-    test_grammar(
+    test_schema(
         "object properties, additionalProperties: true",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
   "type": "object",
   "properties": {
@@ -868,8 +847,7 @@ static void test_json_schema() {
   },
   "additionalProperties": true
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             // TODO: Following line should pass and doesn't
@@ -897,11 +875,10 @@ static void test_json_schema() {
     );
 
     // Additional properties: false
-    test_grammar(
+    test_schema(
         "required + optional props each in original order",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
   "type": "object",
   "properties": {
@@ -911,8 +888,7 @@ static void test_json_schema() {
   },
   "additionalProperties": false
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             R"""({ "street_name": "Pennsylvania" })""",
@@ -931,11 +907,10 @@ static void test_json_schema() {
         }
     );
 
-    test_grammar(
+    test_schema(
         "required + optional props each in original order",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
     "properties": {
         "b": {"type": "string"},
@@ -946,8 +921,7 @@ static void test_json_schema() {
     "required": ["a", "b"],
     "additionalProperties": false
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "{\"b\": \"foo\", \"a\": \"bar\"}",
@@ -964,11 +938,10 @@ static void test_json_schema() {
     );
 
     // NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties
-    test_grammar(
+    test_schema(
         "required props",
-        // Grammar
-        json_schema_to_grammar(nlohmann::ordered_json::parse(
-            R"""(
+        // Schema
+        R"""(
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "$id": "https://example.com/product.schema.json",
@@ -1016,8 +989,7 @@ static void test_json_schema() {
   },
   "required": [ "productId", "productName", "price" ]
 }
-            )"""
-            )),
+        )""",
         // Passing strings
         {
             "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50}",
@@ -1040,8 +1012,6 @@ static void test_json_schema() {
 
         }
     );
-
-
 }
 
 int main() {

From 939b58ae6bca5dbf4e822a97eb7ea9af82148616 Mon Sep 17 00:00:00 2001
From: HanClinto <hanclinto@gmail.com>
Date: Wed, 12 Jun 2024 10:47:01 -0700
Subject: [PATCH 5/7] Adding #define to temporarily remove failing tests so
 that this PR can pass CI, but still be useful for other PRs that want to
 leverage the framework.

---
 tests/test-grammar-integration.cpp | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index cd00b97256b5f..69f082aa681c3 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -15,6 +15,8 @@
 
 using json = nlohmann::ordered_json;
 
+//#define INCLUDE_FAILING_TESTS 1
+
 static llama_grammar* build_grammar(const std::string & grammar_str) {
     auto parsed_grammar = grammar_parser::parse(grammar_str.c_str());
 
@@ -816,10 +818,12 @@ static void test_json_schema() {
             // "By extension, even an empty object is valid"
             R"""({})""",
             // "By default, providing additional properties is valid"
+#ifdef INCLUDE_FAILING_TESTS
             // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
             // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+#endif
         },
         // Failing strings
         {
@@ -850,6 +854,9 @@ static void test_json_schema() {
         )""",
         // Passing strings
         {
+            // "By extension, even an empty object is valid"
+            R"""({})""",
+#ifdef INCLUDE_FAILING_TESTS
             // TODO: Following line should pass and doesn't
             R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",
             // "By default, leaving out properties is valid"
@@ -857,13 +864,12 @@ static void test_json_schema() {
             R"""({ "street_name": "Pennsylvania" })""",
             // TODO: Following line should pass and doesn't
             R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
-            // "By extension, even an empty object is valid"
-            R"""({})""",
             // "By default, providing additional properties is valid"
             // TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
             // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+#endif
         },
         // Failing strings
         {
@@ -895,8 +901,10 @@ static void test_json_schema() {
             R"""({ "number": 1600, "street_type":"Avenue"})""",
             R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
             R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
+#ifdef INCLUDE_FAILING_TESTS
             // TODO: Spaces should be permitted around enum values, but currently they fail to pass.
-            // R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+            R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
+#endif
         },
         // Failing strings
         {

From 3bdf7c30210a3a42d3a5d077b95315e51615f131 Mon Sep 17 00:00:00 2001
From: Clint Herron <hanclinto@gmail.com>
Date: Fri, 21 Jun 2024 22:43:57 -0400
Subject: [PATCH 6/7] Fixing nits from ochafik. Removing escape slashes, adding
 additional failing cases, fixing some other strings.

---
 tests/test-grammar-integration.cpp | 45 ++++++++++++++++--------------
 1 file changed, 24 insertions(+), 21 deletions(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index 69f082aa681c3..b987e07ddeaeb 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -106,7 +106,7 @@ static void test(const std::string & test_desc, const std::string & grammar_str,
                 fclose(string_file);
             }
 
-            fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command:     ./gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
+            fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command:     ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
         } else {
             fprintf(stdout, "✅︎\n");
         }
@@ -510,11 +510,15 @@ static void test_json_schema() {
         // Passing strings
         {
             "{}",
-            "{\"foo\": \"bar\"}",
+            R"""({"foo": "bar"})""",
         },
         // Failing strings
         {
             "",
+            "[]",
+            "null",
+            "\"\"",
+            "true",
         }
     );
 
@@ -932,16 +936,16 @@ static void test_json_schema() {
         )""",
         // Passing strings
         {
-            "{\"b\": \"foo\", \"a\": \"bar\"}",
-            "{\"b\":\"foo\",\"a\":\"bar\",\"d\":\"qux\"}",
-            "{\"b\":\"foo\", \"a\":\"bar\", \"d\":\"qux\", \"c\":\"baz\"}",
+            R"""({"b": "foo", "a": "bar"})""",
+            R"""({"b":"foo","a":"bar","d":"qux"})""",
+            R"""({"b":"foo", "a":"bar", "d":"qux", "c":"baz"})""",
         },
         // Failing strings
         {
-            "{\"a\": \"foo\", \"b\": \"bar\"}",
-            "{\"b\": \"bar\"}",
-            "{\"a\": \"foo\", \"c\": \"baz\"}",
-            "{\"a\":\"foo\", \"b\":\"bar\", \"c\":\"baz\", \"d\":\"qux\"}",
+            R"""({"a": "foo", "b": "bar"})""",
+            R"""({"b": "bar"})""",
+            R"""({"a": "foo", "c": "baz"})""",
+            R"""({"a":"foo", "b":"bar", "c":"baz", "d":"qux"})""",
         }
     );
 
@@ -1000,24 +1004,23 @@ static void test_json_schema() {
         )""",
         // Passing strings
         {
-            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50}",
-            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\"]}",
-            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\"], \"dimensions\": {\"length\": 785, \"width\": 250.5, \"height\": -0.359}}",
+            R"""({"productId": 1, "productName": "A green door", "price": 12.50})""",
+            R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"]})""",
+            R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"], "dimensions": {"length": 785, "width": 250.5, "height": -0.359}})""",
         },
         // Failing strings
         {
-            "{}", // Missing all required properties
-            "{\"productName\": \"A green door\", \"price\": 12.50, \"productId\": 1}", // Out of order properties
+            R"""({})""", // Missing all required properties
+            R"""({"productName": "A green door", "price": 12.50, "productId": 1})""", // Out of order properties
             // TODO: The following line should fail, but currently it passes. `exclusiveMinimum` is not supported, as it would likely be too difficult to implement.
             //  Perhaps special checks for minimum and maximum values of 0 could be added (since that's relatively easy to do with grammars), but anything else would likely be too complex.
-            // "{\"productId\": 1, \"productName\": \"A green door\", \"price\": -12.50}",
-            "{\"productId\": 1, \"productName\": \"A green door\"}", // Missing required property (price)
-            "{\"productName\": \"A green door\", \"price\": 12.50}", // Missing required property (productId)
-            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": []}", // tags is empty, but minItems is 1
-            "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"dimensions\": {\"length\": 785, \"width\": 250.5, \"height\": -0.359}, \"tags\": [\"home\", \"green\"]}", // Tags and dimensions are out of order
+            // R"""({"productId": 1, "productName": "A green door", "price": -12.50})""",
+            R"""({"productId": 1, "productName": "A green door"})""", // Missing required property (price)
+            R"""({"productName": "A green door", "price": 12.50})""", // Missing required property (productId)
+            R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": []})""", // tags is empty, but minItems is 1
+            R"""({"productId": 1, "productName": "A green door", "price": 12.50, "dimensions": {"length": 785, "width": 250.5, "height": -0.359}, "tags": ["home", "green"]})""", // Tags and dimensions are out of order
             // TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.
-            // "{\"productId\": 1, \"productName\": \"A green door\", \"price\": 12.50, \"tags\": [\"home\", \"green\", \"home\"]}",
-
+            // R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green", "home"]})""",
         }
     );
 }

From 5d15dc832eb5b9286e1f8920eac285525f3ed419 Mon Sep 17 00:00:00 2001
From: Clint Herron <hanclinto@gmail.com>
Date: Fri, 21 Jun 2024 23:00:31 -0400
Subject: [PATCH 7/7] Fixing grammar indentation to be consistent throughout
 file.

---
 tests/test-grammar-integration.cpp | 305 +++++++++++++++--------------
 1 file changed, 153 insertions(+), 152 deletions(-)

diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp
index b987e07ddeaeb..96f90c01e0d97 100644
--- a/tests/test-grammar-integration.cpp
+++ b/tests/test-grammar-integration.cpp
@@ -428,10 +428,11 @@ static void test_quantifiers() {
 static void test_failure_missing_root() {
     fprintf(stderr, "⚫ Testing missing root node:\n");
     // Test case for a grammar that is missing a root rule
-    const std::string grammar_str = R"""(rot ::= expr
-expr ::= term ("+" term)*
-term ::= number
-number ::= [0-9]+)""";
+    const std::string grammar_str = R"""(
+        rot ::= expr
+        expr ::= term ("+" term)*
+        term ::= number
+        number ::= [0-9]+)""";
 
     grammar_parser::parse_state parsed_grammar = grammar_parser::parse(grammar_str.c_str());
 
@@ -448,10 +449,10 @@ static void test_failure_missing_reference() {
 
     // Test case for a grammar that is missing a referenced rule
     const std::string grammar_str =
-R"""(root ::= expr
-expr ::= term ("+" term)*
-term ::= numero
-number ::= [0-9]+)""";
+        R"""(root ::= expr
+        expr ::= term ("+" term)*
+        term ::= numero
+        number ::= [0-9]+)""";
 
     fprintf(stderr, "    Expected error:  ");
 
@@ -473,24 +474,24 @@ static void test_failure_left_recursion() {
 
     // Test more complicated left recursion detection
     const std::string medium_str = R"""(
-root ::= asdf
-asdf ::= "a" | asdf "a"
-)""";
+        root ::= asdf
+        asdf ::= "a" | asdf "a"
+        )""";
     assert(test_build_grammar_fails(medium_str));
 
     // Test even more complicated left recursion detection
     const std::string hard_str = R"""(
-root ::= asdf
-asdf ::= "a" | foo "b"
-foo ::= "c" | asdf "d" | "e")""";
+        root ::= asdf
+        asdf ::= "a" | foo "b"
+        foo ::= "c" | asdf "d" | "e")""";
     assert(test_build_grammar_fails(hard_str));
 
     // Test yet even more complicated left recursion detection
     const std::string hardest_str = R"""(
-root ::= asdf
-asdf ::= "a" | foo "b"
-foo ::= "c" | empty asdf "d" | "e"
-empty ::= "blah" | )""";
+        root ::= asdf
+        asdf ::= "a" | foo "b"
+        foo ::= "c" | empty asdf "d" | "e"
+        empty ::= "blah" | )""";
     assert(test_build_grammar_fails(hardest_str));
 
     fprintf(stderr, "  ✅︎ Passed\n");
@@ -505,7 +506,7 @@ static void test_json_schema() {
         "empty schema (object)",
         // Schema
         R"""(
-{}
+            {}
         )""",
         // Passing strings
         {
@@ -526,14 +527,14 @@ static void test_json_schema() {
         "exotic formats (list)",
         // Schema
         R"""(
-{
-  "items": [
-    { "format": "date" },
-    { "format": "uuid" },
-    { "format": "time" },
-    { "format": "date-time" }
-  ]
-}
+            {
+            "items": [
+                { "format": "date" },
+                { "format": "uuid" },
+                { "format": "time" },
+                { "format": "date-time" }
+            ]
+            }
         )""",
         // Passing strings
         {
@@ -554,9 +555,9 @@ static void test_json_schema() {
         "string",
         // Schema
         R"""(
-{
-    "type": "string"
-}
+            {
+                "type": "string"
+            }
         )""",
         // Passing strings
         {
@@ -575,10 +576,10 @@ static void test_json_schema() {
         "string w/ min length 1",
         // Schema
         R"""(
-{
-    "type": "string",
-    "minLength": 1
-}
+            {
+                "type": "string",
+                "minLength": 1
+            }
         )""",
         // Passing strings
         {
@@ -597,10 +598,10 @@ static void test_json_schema() {
         "string w/ min length 3",
         // Schema
         R"""(
-{
-    "type": "string",
-    "minLength": 3
-}
+            {
+                "type": "string",
+                "minLength": 3
+            }
         )""",
         // Passing strings
         {
@@ -620,10 +621,10 @@ static void test_json_schema() {
         "string w/ max length",
         // Schema
         R"""(
-{
-    "type": "string",
-    "maxLength": 3
-}
+            {
+                "type": "string",
+                "maxLength": 3
+            }
         )""",
         // Passing strings
         {
@@ -643,11 +644,11 @@ static void test_json_schema() {
         "string w/ min & max length",
         // Schema
         R"""(
-{
-    "type": "string",
-    "minLength": 1,
-    "maxLength": 4
-}
+            {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 4
+            }
         )""",
         // Passing strings
         {
@@ -668,9 +669,9 @@ static void test_json_schema() {
         "boolean",
         // Schema
         R"""(
-{
-    "type": "boolean"
-}
+            {
+                "type": "boolean"
+            }
         )""",
         // Passing strings
         {
@@ -690,9 +691,9 @@ static void test_json_schema() {
         "integer",
         // Schema
         R"""(
-{
-    "type": "integer"
-}
+            {
+                "type": "integer"
+            }
         )""",
         // Passing strings
         {
@@ -713,9 +714,9 @@ static void test_json_schema() {
         "string const",
         // Schema
         R"""(
-{
-    "const": "foo"
-}
+            {
+                "const": "foo"
+            }
         )""",
         // Passing strings
         {
@@ -732,9 +733,9 @@ static void test_json_schema() {
         "non-string const",
         // Schema
         R"""(
-{
-    "const": true
-}
+            {
+                "const": true
+            }
         )""",
         // Passing strings
         {
@@ -752,9 +753,9 @@ static void test_json_schema() {
         "non-string const",
         // Schema
         R"""(
-{
-    "enum": ["red", "amber", "green", null, 42, ["foo"]]
-}
+            {
+                "enum": ["red", "amber", "green", null, 42, ["foo"]]
+            }
         )""",
         // Passing strings
         {
@@ -777,13 +778,13 @@ static void test_json_schema() {
         "min+max items",
         // Schema
         R"""(
-{
-    "items": {
-        "type": ["number", "integer"]
-    },
-    "minItems": 3,
-    "maxItems": 5
-}
+            {
+                "items": {
+                    "type": ["number", "integer"]
+                },
+                "minItems": 3,
+                "maxItems": 5
+            }
         )""",
         // Passing strings
         {
@@ -804,14 +805,14 @@ static void test_json_schema() {
         "object properties",
         // Schema
         R"""(
-{
-  "type": "object",
-  "properties": {
-    "number": { "type": "number" },
-    "street_name": { "type": "string" },
-    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
-  }
-}
+            {
+            "type": "object",
+            "properties": {
+                "number": { "type": "number" },
+                "street_name": { "type": "string" },
+                "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+            }
+            }
         )""",
         // Passing strings
         {
@@ -846,15 +847,15 @@ static void test_json_schema() {
         "object properties, additionalProperties: true",
         // Schema
         R"""(
-{
-  "type": "object",
-  "properties": {
-    "number": { "type": "number" },
-    "street_name": { "type": "string" },
-    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
-  },
-  "additionalProperties": true
-}
+            {
+            "type": "object",
+            "properties": {
+                "number": { "type": "number" },
+                "street_name": { "type": "string" },
+                "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+            },
+            "additionalProperties": true
+            }
         )""",
         // Passing strings
         {
@@ -889,15 +890,15 @@ static void test_json_schema() {
         "required + optional props each in original order",
         // Schema
         R"""(
-{
-  "type": "object",
-  "properties": {
-    "number": { "type": "number" },
-    "street_name": { "type": "string" },
-    "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
-  },
-  "additionalProperties": false
-}
+            {
+            "type": "object",
+            "properties": {
+                "number": { "type": "number" },
+                "street_name": { "type": "string" },
+                "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
+            },
+            "additionalProperties": false
+            }
         )""",
         // Passing strings
         {
@@ -923,16 +924,16 @@ static void test_json_schema() {
         "required + optional props each in original order",
         // Schema
         R"""(
-{
-    "properties": {
-        "b": {"type": "string"},
-        "a": {"type": "string"},
-        "d": {"type": "string"},
-        "c": {"type": "string"}
-    },
-    "required": ["a", "b"],
-    "additionalProperties": false
-}
+            {
+                "properties": {
+                    "b": {"type": "string"},
+                    "a": {"type": "string"},
+                    "d": {"type": "string"},
+                    "c": {"type": "string"}
+                },
+                "required": ["a", "b"],
+                "additionalProperties": false
+            }
         )""",
         // Passing strings
         {
@@ -954,53 +955,53 @@ static void test_json_schema() {
         "required props",
         // Schema
         R"""(
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://example.com/product.schema.json",
-  "title": "Product",
-  "description": "A product from Acme's catalog",
-  "type": "object",
-  "properties": {
-    "productId": {
-      "description": "The unique identifier for a product",
-      "type": "integer"
-    },
-    "productName": {
-      "description": "Name of the product",
-      "type": "string"
-    },
-    "price": {
-      "description": "The price of the product",
-      "type": "number",
-      "exclusiveMinimum": 0
-    },
-    "tags": {
-      "description": "Tags for the product",
-      "type": "array",
-      "items": {
-        "type": "string"
-      },
-      "minItems": 1,
-      "uniqueItems": true
-    },
-    "dimensions": {
-      "type": "object",
-      "properties": {
-        "length": {
-          "type": "number"
-        },
-        "width": {
-          "type": "number"
-        },
-        "height": {
-          "type": "number"
-        }
-      },
-      "required": [ "length", "width", "height" ]
-    }
-  },
-  "required": [ "productId", "productName", "price" ]
-}
+            {
+            "$schema": "https://json-schema.org/draft/2020-12/schema",
+            "$id": "https://example.com/product.schema.json",
+            "title": "Product",
+            "description": "A product from Acme's catalog",
+            "type": "object",
+            "properties": {
+                "productId": {
+                "description": "The unique identifier for a product",
+                "type": "integer"
+                },
+                "productName": {
+                "description": "Name of the product",
+                "type": "string"
+                },
+                "price": {
+                "description": "The price of the product",
+                "type": "number",
+                "exclusiveMinimum": 0
+                },
+                "tags": {
+                "description": "Tags for the product",
+                "type": "array",
+                "items": {
+                    "type": "string"
+                },
+                "minItems": 1,
+                "uniqueItems": true
+                },
+                "dimensions": {
+                "type": "object",
+                "properties": {
+                    "length": {
+                    "type": "number"
+                    },
+                    "width": {
+                    "type": "number"
+                    },
+                    "height": {
+                    "type": "number"
+                    }
+                },
+                "required": [ "length", "width", "height" ]
+                }
+            },
+            "required": [ "productId", "productName", "price" ]
+            }
         )""",
         // Passing strings
         {