From 7834a73a1734867f9b74e05f6175bfa698aee6e5 Mon Sep 17 00:00:00 2001
From: Jhen-Jie Hong <developer@jhen.me>
Date: Thu, 25 Apr 2024 13:44:57 +0800
Subject: [PATCH] feat(ts): sync grammar converter (#48)

* feat(ts): sync grammar converter

* fix(ts): type issues

* feat(ts): expose options

* feat(ts): sync schema converter

* feat(ts): handle resolve refs in convertJsonSchemaToGrammar
---
 docs/API/README.md                            |  22 +-
 docs/API/classes/LlamaContext.md              |  26 +-
 docs/API/classes/SchemaGrammarConverter.md    | 216 ++++-
 .../__snapshots__/grammar.test.ts.snap        |  64 +-
 src/__tests__/grammar.test.ts                 |   2 +
 src/grammar.ts                                | 883 ++++++++++++++++--
 6 files changed, 1065 insertions(+), 148 deletions(-)

diff --git a/docs/API/README.md b/docs/API/README.md
index b62bbf82..52e9ee5b 100644
--- a/docs/API/README.md
+++ b/docs/API/README.md
@@ -43,7 +43,7 @@ llama.rn
 
 #### Defined in
 
-[index.ts:43](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L43)
+[index.ts:43](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L43)
 
 ___
 
@@ -53,7 +53,7 @@ ___
 
 #### Defined in
 
-[index.ts:41](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L41)
+[index.ts:41](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L41)
 
 ___
 
@@ -63,7 +63,7 @@ ___
 
 #### Defined in
 
-[index.ts:39](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L39)
+[index.ts:39](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L39)
 
 ___
 
@@ -80,29 +80,31 @@ ___
 
 #### Defined in
 
-[index.ts:29](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L29)
+[index.ts:29](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L29)
 
 ## Functions
 
 ### convertJsonSchemaToGrammar
 
-▸ **convertJsonSchemaToGrammar**(`«destructured»`): `string`
+▸ **convertJsonSchemaToGrammar**(`«destructured»`): `string` \| `Promise`<`string`\>
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
 | `«destructured»` | `Object` |
+| › `allowFetch?` | `boolean` |
+| › `dotall?` | `boolean` |
 | › `propOrder?` | `PropOrder` |
 | › `schema` | `any` |
 
 #### Returns
 
-`string`
+`string` \| `Promise`<`string`\>
 
 #### Defined in
 
-[grammar.ts:134](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L134)
+[grammar.ts:824](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L824)
 
 ___
 
@@ -122,7 +124,7 @@ ___
 
 #### Defined in
 
-[index.ts:160](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L160)
+[index.ts:160](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L160)
 
 ___
 
@@ -136,7 +138,7 @@ ___
 
 #### Defined in
 
-[index.ts:176](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L176)
+[index.ts:176](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L176)
 
 ___
 
@@ -156,4 +158,4 @@ ___
 
 #### Defined in
 
-[index.ts:156](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L156)
+[index.ts:156](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L156)
diff --git a/docs/API/classes/LlamaContext.md b/docs/API/classes/LlamaContext.md
index 0b6572aa..d65fbe21 100644
--- a/docs/API/classes/LlamaContext.md
+++ b/docs/API/classes/LlamaContext.md
@@ -40,7 +40,7 @@
 
 #### Defined in
 
-[index.ts:60](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L60)
+[index.ts:60](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L60)
 
 ## Properties
 
@@ -50,7 +50,7 @@
 
 #### Defined in
 
-[index.ts:56](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L56)
+[index.ts:56](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L56)
 
 ___
 
@@ -60,7 +60,7 @@ ___
 
 #### Defined in
 
-[index.ts:54](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L54)
+[index.ts:54](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L54)
 
 ___
 
@@ -70,7 +70,7 @@ ___
 
 #### Defined in
 
-[index.ts:58](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L58)
+[index.ts:58](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L58)
 
 ## Methods
 
@@ -93,7 +93,7 @@ ___
 
 #### Defined in
 
-[index.ts:129](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L129)
+[index.ts:129](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L129)
 
 ___
 
@@ -114,7 +114,7 @@ ___
 
 #### Defined in
 
-[index.ts:84](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L84)
+[index.ts:84](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L84)
 
 ___
 
@@ -134,7 +134,7 @@ ___
 
 #### Defined in
 
-[index.ts:121](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L121)
+[index.ts:121](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L121)
 
 ___
 
@@ -154,7 +154,7 @@ ___
 
 #### Defined in
 
-[index.ts:125](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L125)
+[index.ts:125](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L125)
 
 ___
 
@@ -176,7 +176,7 @@ Load cached prompt & completion state from a file.
 
 #### Defined in
 
-[index.ts:73](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L73)
+[index.ts:73](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L73)
 
 ___
 
@@ -190,7 +190,7 @@ ___
 
 #### Defined in
 
-[index.ts:151](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L151)
+[index.ts:151](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L151)
 
 ___
 
@@ -214,7 +214,7 @@ Save current cached prompt & completion state to a file.
 
 #### Defined in
 
-[index.ts:80](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L80)
+[index.ts:80](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L80)
 
 ___
 
@@ -228,7 +228,7 @@ ___
 
 #### Defined in
 
-[index.ts:113](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L113)
+[index.ts:113](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L113)
 
 ___
 
@@ -248,4 +248,4 @@ ___
 
 #### Defined in
 
-[index.ts:117](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/index.ts#L117)
+[index.ts:117](https://github.com/mybigday/llama.rn/blob/17714d4/src/index.ts#L117)
diff --git a/docs/API/classes/SchemaGrammarConverter.md b/docs/API/classes/SchemaGrammarConverter.md
index 51c7cec3..14f33c6c 100644
--- a/docs/API/classes/SchemaGrammarConverter.md
+++ b/docs/API/classes/SchemaGrammarConverter.md
@@ -10,56 +10,138 @@
 
 ### Properties
 
+- [\_allowFetch](SchemaGrammarConverter.md#_allowfetch)
+- [\_dotall](SchemaGrammarConverter.md#_dotall)
 - [\_propOrder](SchemaGrammarConverter.md#_proporder)
+- [\_refs](SchemaGrammarConverter.md#_refs)
+- [\_refsBeingResolved](SchemaGrammarConverter.md#_refsbeingresolved)
 - [\_rules](SchemaGrammarConverter.md#_rules)
 
 ### Methods
 
-- [addRule](SchemaGrammarConverter.md#addrule)
+- [\_addPrimitive](SchemaGrammarConverter.md#_addprimitive)
+- [\_addRule](SchemaGrammarConverter.md#_addrule)
+- [\_buildObjectRule](SchemaGrammarConverter.md#_buildobjectrule)
+- [\_generateUnionRule](SchemaGrammarConverter.md#_generateunionrule)
+- [\_resolveRef](SchemaGrammarConverter.md#_resolveref)
+- [\_visitPattern](SchemaGrammarConverter.md#_visitpattern)
 - [formatGrammar](SchemaGrammarConverter.md#formatgrammar)
+- [resolveRefs](SchemaGrammarConverter.md#resolverefs)
 - [visit](SchemaGrammarConverter.md#visit)
 
 ## Constructors
 
 ### constructor
 
-• **new SchemaGrammarConverter**(`propOrder?`)
+• **new SchemaGrammarConverter**(`options`)
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
-| `propOrder?` | `PropOrder` |
+| `options` | `Object` |
+| `options.allow_fetch?` | `boolean` |
+| `options.dotall?` | `boolean` |
+| `options.prop_order?` | `PropOrder` |
 
 #### Defined in
 
-[grammar.ts:39](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L39)
+[grammar.ts:211](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L211)
 
 ## Properties
 
+### \_allowFetch
+
+• `Private` **\_allowFetch**: `boolean`
+
+#### Defined in
+
+[grammar.ts:201](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L201)
+
+___
+
+### \_dotall
+
+• `Private` **\_dotall**: `boolean`
+
+#### Defined in
+
+[grammar.ts:203](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L203)
+
+___
+
 ### \_propOrder
 
 • `Private` **\_propOrder**: `PropOrder`
 
 #### Defined in
 
-[grammar.ts:35](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L35)
+[grammar.ts:199](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L199)
+
+___
+
+### \_refs
+
+• `Private` **\_refs**: `Object`
+
+#### Index signature
+
+▪ [key: `string`]: `any`
+
+#### Defined in
+
+[grammar.ts:207](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L207)
+
+___
+
+### \_refsBeingResolved
+
+• `Private` **\_refsBeingResolved**: `Set`<`string`\>
+
+#### Defined in
+
+[grammar.ts:209](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L209)
 
 ___
 
 ### \_rules
 
-• `Private` **\_rules**: `Map`<`string`, `string`\>
+• `Private` **\_rules**: `Object`
+
+#### Index signature
+
+▪ [key: `string`]: `string`
 
 #### Defined in
 
-[grammar.ts:37](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L37)
+[grammar.ts:205](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L205)
 
 ## Methods
 
-### addRule
+### \_addPrimitive
 
-▸ `Private` **addRule**(`name`, `rule`): `string`
+▸ **_addPrimitive**(`name`, `rule`): `string`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `rule` | `undefined` \| `BuiltinRule` |
+
+#### Returns
+
+`string`
+
+#### Defined in
+
+[grammar.ts:693](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L693)
+
+___
+
+### \_addRule
+
+▸ **_addRule**(`name`, `rule`): `string`
 
 #### Parameters
 
@@ -74,7 +156,92 @@ ___
 
 #### Defined in
 
-[grammar.ts:45](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L45)
+[grammar.ts:224](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L224)
+
+___
+
+### \_buildObjectRule
+
+▸ **_buildObjectRule**(`properties`, `required`, `name`, `additionalProperties`): `string`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `properties` | `any`[] |
+| `required` | `Set`<`string`\> |
+| `name` | `string` |
+| `additionalProperties` | `any` |
+
+#### Returns
+
+`string`
+
+#### Defined in
+
+[grammar.ts:710](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L710)
+
+___
+
+### \_generateUnionRule
+
+▸ **_generateUnionRule**(`name`, `altSchemas`): `string`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `name` | `string` |
+| `altSchemas` | `any`[] |
+
+#### Returns
+
+`string`
+
+#### Defined in
+
+[grammar.ts:312](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L312)
+
+___
+
+### \_resolveRef
+
+▸ **_resolveRef**(`ref`): `string`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `ref` | `string` |
+
+#### Returns
+
+`string`
+
+#### Defined in
+
+[grammar.ts:518](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L518)
+
+___
+
+### \_visitPattern
+
+▸ **_visitPattern**(`pattern`, `name`): `string`
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `pattern` | `string` |
+| `name` | `string` |
+
+#### Returns
+
+`string`
+
+#### Defined in
+
+[grammar.ts:323](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L323)
 
 ___
 
@@ -88,20 +255,41 @@ ___
 
 #### Defined in
 
-[grammar.ts:125](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L125)
+[grammar.ts:813](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L813)
+
+___
+
+### resolveRefs
+
+▸ **resolveRefs**(`schema`, `url`): `Promise`<`any`\>
+
+#### Parameters
+
+| Name | Type |
+| :------ | :------ |
+| `schema` | `any` |
+| `url` | `string` |
+
+#### Returns
+
+`Promise`<`any`\>
+
+#### Defined in
+
+[grammar.ts:247](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L247)
 
 ___
 
 ### visit
 
-▸ **visit**(`schema`, `name?`): `string`
+▸ **visit**(`schema`, `name`): `string`
 
 #### Parameters
 
 | Name | Type |
 | :------ | :------ |
 | `schema` | `any` |
-| `name?` | `string` |
+| `name` | `string` |
 
 #### Returns
 
@@ -109,4 +297,4 @@ ___
 
 #### Defined in
 
-[grammar.ts:65](https://github.com/mybigday/llama.rn/blob/e3e9f86/src/grammar.ts#L65)
+[grammar.ts:529](https://github.com/mybigday/llama.rn/blob/17714d4/src/grammar.ts#L529)
diff --git a/src/__tests__/__snapshots__/grammar.test.ts.snap b/src/__tests__/__snapshots__/grammar.test.ts.snap
index e8343ba8..b0c22208 100644
--- a/src/__tests__/__snapshots__/grammar.test.ts.snap
+++ b/src/__tests__/__snapshots__/grammar.test.ts.snap
@@ -1,33 +1,49 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`with prop order 1`] = `
-"space ::= \\" \\"?
-0-function ::= \\"\\\\\\"create_event\\\\\\"\\"
-string ::=  \\"\\\\\\"\\" (
-        [^\\"\\\\\\\\] |
-        \\"\\\\\\\\\\" ([\\"\\\\\\\\/bfnrt] | \\"u\\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
-      )* \\"\\\\\\"\\" space
-0-arguments ::= \\"{\\" space \\"\\\\\\"date\\\\\\"\\" space \\":\\" space string \\",\\" space \\"\\\\\\"time\\\\\\"\\" space \\":\\" space string \\",\\" space \\"\\\\\\"title\\\\\\"\\" space \\":\\" space string \\"}\\" space
-0 ::= \\"{\\" space \\"\\\\\\"function\\\\\\"\\" space \\":\\" space 0-function \\",\\" space \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space 0-arguments \\"}\\" space
-1-function ::= \\"\\\\\\"image_search\\\\\\"\\"
-1-arguments ::= \\"{\\" space \\"\\\\\\"query\\\\\\"\\" space \\":\\" space string \\"}\\" space
-1 ::= \\"{\\" space \\"\\\\\\"function\\\\\\"\\" space \\":\\" space 1-function \\",\\" space \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space 1-arguments \\"}\\" space
-root ::= 0 | 1
+"alternative-0 ::= \\"{\\" space alternative-0-arguments-kv \\",\\" space alternative-0-function-kv \\"}\\" space
+alternative-0-arguments ::= \\"{\\" space  (alternative-0-arguments-title-kv alternative-0-arguments-title-rest | alternative-0-arguments-date-kv alternative-0-arguments-date-rest | alternative-0-arguments-time-kv )? \\"}\\" space
+alternative-0-arguments-date-kv ::= \\"\\\\\\"date\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-date-rest ::= ( \\",\\" space alternative-0-arguments-time-kv )?
+alternative-0-arguments-kv ::= \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space alternative-0-arguments
+alternative-0-arguments-time-kv ::= \\"\\\\\\"time\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-title-kv ::= \\"\\\\\\"title\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-title-rest ::= ( \\",\\" space alternative-0-arguments-date-kv )? alternative-0-arguments-date-rest
+alternative-0-function ::= \\"\\\\\\"create_event\\\\\\"\\"
+alternative-0-function-kv ::= \\"\\\\\\"function\\\\\\"\\" space \\":\\" space alternative-0-function
+alternative-1 ::= \\"{\\" space alternative-1-arguments-kv \\",\\" space alternative-1-function-kv \\"}\\" space
+alternative-1-arguments ::= \\"{\\" space  (alternative-1-arguments-query-kv )? \\"}\\" space
+alternative-1-arguments-kv ::= \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space alternative-1-arguments
+alternative-1-arguments-query-kv ::= \\"\\\\\\"query\\\\\\"\\" space \\":\\" space string
+alternative-1-function ::= \\"\\\\\\"image_search\\\\\\"\\"
+alternative-1-function-kv ::= \\"\\\\\\"function\\\\\\"\\" space \\":\\" space alternative-1-function
+char ::= [^\\"\\\\\\\\] | \\"\\\\\\\\\\" ([\\"\\\\\\\\/bfnrt] | \\"u\\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
+root ::= alternative-0 | alternative-1
+space ::= \\" \\"?
+string ::= \\"\\\\\\"\\" char* \\"\\\\\\"\\" space
 "
 `;
 
 exports[`without prop order 1`] = `
-"space ::= \\" \\"?
-string ::=  \\"\\\\\\"\\" (
-        [^\\"\\\\\\\\] |
-        \\"\\\\\\\\\\" ([\\"\\\\\\\\/bfnrt] | \\"u\\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
-      )* \\"\\\\\\"\\" space
-0-arguments ::= \\"{\\" space \\"\\\\\\"date\\\\\\"\\" space \\":\\" space string \\",\\" space \\"\\\\\\"time\\\\\\"\\" space \\":\\" space string \\",\\" space \\"\\\\\\"title\\\\\\"\\" space \\":\\" space string \\"}\\" space
-0-function ::= \\"\\\\\\"create_event\\\\\\"\\"
-0 ::= \\"{\\" space \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space 0-arguments \\",\\" space \\"\\\\\\"function\\\\\\"\\" space \\":\\" space 0-function \\"}\\" space
-1-arguments ::= \\"{\\" space \\"\\\\\\"query\\\\\\"\\" space \\":\\" space string \\"}\\" space
-1-function ::= \\"\\\\\\"image_search\\\\\\"\\"
-1 ::= \\"{\\" space \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space 1-arguments \\",\\" space \\"\\\\\\"function\\\\\\"\\" space \\":\\" space 1-function \\"}\\" space
-root ::= 0 | 1
+"alternative-0 ::= \\"{\\" space alternative-0-function-kv \\",\\" space alternative-0-arguments-kv \\"}\\" space
+alternative-0-arguments ::= \\"{\\" space  (alternative-0-arguments-title-kv alternative-0-arguments-title-rest | alternative-0-arguments-date-kv alternative-0-arguments-date-rest | alternative-0-arguments-time-kv )? \\"}\\" space
+alternative-0-arguments-date-kv ::= \\"\\\\\\"date\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-date-rest ::= ( \\",\\" space alternative-0-arguments-time-kv )?
+alternative-0-arguments-kv ::= \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space alternative-0-arguments
+alternative-0-arguments-time-kv ::= \\"\\\\\\"time\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-title-kv ::= \\"\\\\\\"title\\\\\\"\\" space \\":\\" space string
+alternative-0-arguments-title-rest ::= ( \\",\\" space alternative-0-arguments-date-kv )? alternative-0-arguments-date-rest
+alternative-0-function ::= \\"\\\\\\"create_event\\\\\\"\\"
+alternative-0-function-kv ::= \\"\\\\\\"function\\\\\\"\\" space \\":\\" space alternative-0-function
+alternative-1 ::= \\"{\\" space alternative-1-function-kv \\",\\" space alternative-1-arguments-kv \\"}\\" space
+alternative-1-arguments ::= \\"{\\" space  (alternative-1-arguments-query-kv )? \\"}\\" space
+alternative-1-arguments-kv ::= \\"\\\\\\"arguments\\\\\\"\\" space \\":\\" space alternative-1-arguments
+alternative-1-arguments-query-kv ::= \\"\\\\\\"query\\\\\\"\\" space \\":\\" space string
+alternative-1-function ::= \\"\\\\\\"image_search\\\\\\"\\"
+alternative-1-function-kv ::= \\"\\\\\\"function\\\\\\"\\" space \\":\\" space alternative-1-function
+char ::= [^\\"\\\\\\\\] | \\"\\\\\\\\\\" ([\\"\\\\\\\\/bfnrt] | \\"u\\" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
+root ::= alternative-0 | alternative-1
+space ::= \\" \\"?
+string ::= \\"\\\\\\"\\" char* \\"\\\\\\"\\" space
 "
 `;
diff --git a/src/__tests__/grammar.test.ts b/src/__tests__/grammar.test.ts
index 6b1093fe..a731bf52 100644
--- a/src/__tests__/grammar.test.ts
+++ b/src/__tests__/grammar.test.ts
@@ -15,6 +15,7 @@ const schema = {
           },
         },
       },
+      required: ['function', 'arguments'],
     },
     {
       type: 'object',
@@ -27,6 +28,7 @@ const schema = {
           },
         },
       },
+      required: ['function', 'arguments'],
     },
   ],
 }
diff --git a/src/grammar.ts b/src/grammar.ts
index ea75ce21..6a845cf5 100644
--- a/src/grammar.ts
+++ b/src/grammar.ts
@@ -1,140 +1,849 @@
+/* eslint-disable no-restricted-syntax */
+/* eslint-disable no-underscore-dangle */
 const SPACE_RULE = '" "?'
 
-const PRIMITIVE_RULES: { [key: string]: string } = {
-  boolean: '("true" | "false") space',
-  number:
-    '("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? space',
-  integer: '("-"? ([0-9] | [1-9] [0-9]*)) space',
-  string: ` "\\"" (
-        [^"\\\\] |
-        "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])
-      )* "\\"" space`,
-  null: '"null" space',
+function buildRepetition(
+  itemRule: string,
+  minItems: number,
+  maxItems: number | undefined,
+  opts: {
+    separatorRule?: string
+    itemRuleIsLiteral?: boolean
+  } = {},
+) {
+  const separatorRule = opts.separatorRule ?? ''
+  const itemRuleIsLiteral = opts.itemRuleIsLiteral ?? false
+
+  if (separatorRule === '') {
+    if (minItems === 0 && maxItems === 1) {
+      return `${itemRule}?`
+    } else if (minItems === 1 && maxItems === undefined) {
+      return `${itemRule}+`
+    }
+  }
+
+  let result = ''
+  if (minItems > 0) {
+    if (itemRuleIsLiteral && separatorRule === '') {
+      result = `"${itemRule.slice(1, -1).repeat(minItems)}"`
+    } else {
+      result = Array.from({ length: minItems }, () => itemRule).join(
+        separatorRule !== '' ? ` ${separatorRule} ` : ' ',
+      )
+    }
+  }
+
+  const optRepetitions = (upToN: number, prefixWithSep = false): string => {
+    const content =
+      separatorRule !== '' && prefixWithSep
+        ? `${separatorRule} ${itemRule}`
+        : itemRule
+    if (upToN === 0) {
+      return ''
+    } else if (upToN === 1) {
+      return `(${content})?`
+    } else if (separatorRule !== '' && !prefixWithSep) {
+      return `(${content} ${optRepetitions(upToN - 1, true)})?`
+    } else {
+      return (
+        Array.from({ length: upToN }, () => `(${content}`)
+          .join(' ')
+          .trim() + Array.from({ length: upToN }, () => ')?').join('')
+      )
+    }
+  }
+
+  if (minItems > 0 && maxItems !== minItems) {
+    result += ' '
+  }
+
+  if (maxItems !== undefined) {
+    result += optRepetitions(maxItems - minItems, minItems > 0)
+  } else {
+    const itemOperator = `(${
+      separatorRule !== '' ? `${separatorRule} ` : ''
+    }${itemRule})`
+
+    if (minItems === 0 && separatorRule !== '') {
+      result = `(${itemRule} ${itemOperator}*)?`
+    } else {
+      result += `${itemOperator}*`
+    }
+  }
+
+  return result
+}
+
+class BuiltinRule {
+  content: string
+
+  deps: string[]
+
+  constructor(content: string, deps: string[]) {
+    this.content = content
+    this.deps = deps || []
+  }
+}
+
+const UP_TO_15_DIGITS = buildRepetition('[0-9]', 0, 15)
+
+const PRIMITIVE_RULES: { [key: string]: BuiltinRule } = {
+  boolean: new BuiltinRule('("true" | "false") space', []),
+  'decimal-part': new BuiltinRule(`[0-9] ${UP_TO_15_DIGITS}`, []),
+  'integral-part': new BuiltinRule(`[0-9] | [1-9] ${UP_TO_15_DIGITS}`, []),
+  number: new BuiltinRule(
+    '("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space',
+    ['integral-part', 'decimal-part'],
+  ),
+  integer: new BuiltinRule('("-"? integral-part) space', ['integral-part']),
+  value: new BuiltinRule('object | array | string | number | boolean | null', [
+    'object',
+    'array',
+    'string',
+    'number',
+    'boolean',
+    'null',
+  ]),
+  object: new BuiltinRule(
+    '"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space',
+    ['string', 'value'],
+  ),
+  array: new BuiltinRule('"[" space ( value ("," space value)* )? "]" space', [
+    'value',
+  ]),
+  uuid: new BuiltinRule(
+    `"\\"" ${[8, 4, 4, 4, 12]
+      .map((n) => [...new Array(n)].map((_) => '[0-9a-fA-F]').join(''))
+      .join(' "-" ')} "\\"" space`,
+    [],
+  ),
+  char: new BuiltinRule(
+    `[^"\\\\] | "\\\\" (["\\\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])`,
+    [],
+  ),
+  string: new BuiltinRule(`"\\"" char* "\\"" space`, ['char']),
+  null: new BuiltinRule('"null" space', []),
+}
+
+// TODO: support "uri", "email" string formats
+const STRING_FORMAT_RULES: { [key: string]: BuiltinRule } = {
+  date: new BuiltinRule(
+    '[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )',
+    [],
+  ),
+  time: new BuiltinRule(
+    '([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )',
+    [],
+  ),
+  'date-time': new BuiltinRule('date "T" time', ['date', 'time']),
+  'date-string': new BuiltinRule('"\\"" date "\\"" space', ['date']),
+  'time-string': new BuiltinRule('"\\"" time "\\"" space', ['time']),
+  'date-time-string': new BuiltinRule('"\\"" date-time "\\"" space', [
+    'date-time',
+  ]),
+}
+
+const RESERVED_NAMES = {
+  root: true,
+  ...PRIMITIVE_RULES,
+  ...STRING_FORMAT_RULES,
 }
 
 const INVALID_RULE_CHARS_RE = /[^\dA-Za-z-]+/g
 const GRAMMAR_LITERAL_ESCAPE_RE = /[\n\r"]/g
-const GRAMMAR_LITERAL_ESCAPES = { '\r': '\\r', '\n': '\\n', '"': '\\"' } as {
-  [key: string]: string
+const GRAMMAR_LITERAL_ESCAPES: any = {
+  '\r': '\\r',
+  '\n': '\\n',
+  '"': '\\"',
+  '-': '\\-',
+  ']': '\\]',
 }
 
+const NON_LITERAL_SET = new Set('|.()[]{}*+?')
+const ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS = new Set('[]()|{}*+?')
+
 const formatLiteral = (literal: string): string => {
-  const escaped = JSON.stringify(literal).replace(
+  const escaped = literal.replace(
     GRAMMAR_LITERAL_ESCAPE_RE,
     (m) => GRAMMAR_LITERAL_ESCAPES[m] || '',
   )
   return `"${escaped}"`
 }
 
+const generateConstantRule = (value: any): string =>
+  formatLiteral(JSON.stringify(value))
+
 interface PropOrder {
   [key: string]: number
 }
 
-// JSON schema to grammar converter (Ref: https://github.com/ggerganov/llama.cpp/blob/master/examples/json-schema-to-grammar.py)
+// Helper function to group elements by a key function
+function* groupBy(iterable: Iterable<any>, keyFn: (x: any) => any) {
+  let lastKey = null
+  let group = []
+  for (const element of iterable) {
+    const key = keyFn(element)
+    if (lastKey !== null && key !== lastKey) {
+      yield [lastKey, group]
+      group = []
+    }
+    group.push(element)
+    lastKey = key
+  }
+  if (group.length > 0) {
+    yield [lastKey, group]
+  }
+}
+
 export class SchemaGrammarConverter {
   private _propOrder: PropOrder
 
-  private _rules: Map<string, string>
+  private _allowFetch: boolean
+
+  private _dotall: boolean
 
-  constructor(propOrder?: PropOrder) {
-    this._propOrder = propOrder || {}
-    this._rules = new Map<string, string>();
-    this._rules.set('space', SPACE_RULE)
+  private _rules: { [key: string]: string }
+
+  private _refs: { [key: string]: any }
+
+  private _refsBeingResolved: Set<string>
+
+  constructor(options: {
+    prop_order?: PropOrder
+    allow_fetch?: boolean
+    dotall?: boolean
+  }) {
+    this._propOrder = options.prop_order || {}
+    this._allowFetch = options.allow_fetch || false
+    this._dotall = options.dotall || false
+    this._rules = { space: SPACE_RULE }
+    this._refs = {}
+    this._refsBeingResolved = new Set()
   }
 
-  private addRule(name: string, rule: string): string {
-    const escName = name.replace(INVALID_RULE_CHARS_RE, '-');
-    let key = escName;
+  _addRule(name: string, rule: string): string {
+    const escName = name.replace(INVALID_RULE_CHARS_RE, '-')
+    let key = escName
 
-    if (this._rules.has(escName)) {
-      if (this._rules.get(escName) === rule) {
-        return key;
+    if (escName in this._rules) {
+      if (this._rules[escName] === rule) {
+        return key
       }
 
-      let i = 0;
-      while (this._rules.has(`${escName}${i}`)) {
-        i += 1;
+      let i = 0
+      while (
+        `${escName}${i}` in this._rules &&
+        this._rules[`${escName}${i}`] !== rule
+      ) {
+        i += 1
       }
-      key = `${escName}${i}`;
+      key = `${escName}${i}`
     }
 
-    this._rules.set(key, rule);
-    return key;
+    this._rules[key] = rule
+    return key
   }
 
-  public visit(schema: any, name?: string): string {
-    const schemaType = schema.type
-    const ruleName = name || 'root'
+  async resolveRefs(schema: any, url: string): Promise<any> {
+    const visit: any = async (n: any) => {
+      if (Array.isArray(n)) {
+        return Promise.all(n.map(visit))
+      } else if (typeof n === 'object' && n !== null) {
+        let ref = n.$ref
+        let target
+        if (ref !== undefined && !this._refs[ref]) {
+          if (ref.startsWith('https://')) {
+            if (!this._allowFetch) {
+              throw new Error(
+                'Fetching remote schemas is not allowed (use --allow-fetch for force)',
+              )
+            }
 
-    if (schema.oneOf || schema.anyOf) {
-      const rule = (schema.oneOf || schema.anyOf)
-        .map((altSchema: any, i: number) =>
-          this.visit(altSchema, `${name}${name ? '-' : ''}${i}`),
-        )
-        .join(' | ')
+            const fragSplit = ref.split('#')
+            const baseUrl = fragSplit[0]
+
+            target = this._refs[baseUrl]
+            if (!target) {
+              target = await this.resolveRefs(
+                await fetch(ref).then((res) => res.json()),
+                baseUrl,
+              )
+              this._refs[baseUrl] = target
+            }
+
+            if (
+              fragSplit.length === 1 ||
+              fragSplit[fragSplit.length - 1] === ''
+            ) {
+              return target
+            }
+          } else if (ref.startsWith('#/')) {
+            target = schema
+            ref = `${url}${ref}`
+            n.$ref = ref
+          } else {
+            throw new Error(`Unsupported ref ${ref}`)
+          }
+
+          const selectors = ref.split('#')[1].split('/').slice(1)
+          for (const sel of selectors) {
+            if (!target || !(sel in target)) {
+              throw new Error(
+                `Error resolving ref ${ref}: ${sel} not in ${JSON.stringify(
+                  target,
+                )}`,
+              )
+            }
+            target = target[sel]
+          }
+
+          this._refs[ref] = target
+        } else {
+          await Promise.all(Object.values(n).map(visit))
+        }
+      }
+
+      return n
+    }
+
+    return visit(schema)
+  }
+
+  _generateUnionRule(name: string, altSchemas: any[]): string {
+    return altSchemas
+      .map((altSchema, i) =>
+        this.visit(
+          altSchema,
+          `${name ?? ''}${name ? '-' : 'alternative-'}${i}`,
+        ),
+      )
+      .join(' | ')
+  }
+
+  _visitPattern(pattern: string, name: string): string {
+    if (!pattern.startsWith('^') || !pattern.endsWith('$')) {
+      throw new Error('Pattern must start with "^" and end with "$"')
+    }
+    pattern = pattern.slice(1, -1)
+    const subRuleIds: { [key: string]: string } = {}
+
+    let i = 0
+    const { length } = pattern
+
+    const getDot = () => {
+      let rule
+      if (this._dotall) {
+        rule = '[\\U00000000-\\U0010FFFF]'
+      } else {
+        // Accept any character... except \n and \r line break chars (\x0A and \xOD)
+        rule = '[^\\x0A\\x0D]'
+      }
+      return this._addRule('dot', rule)
+    }
+
+    const toRule = ([s, isLiteral]: [string, boolean]) =>
+      isLiteral ? `"${s}"` : s
+
+    const transform = () => {
+      const start = i
+      // For each component of this sequence, store its string representation and whether it's a literal.
+      // We only need a flat structure here to apply repetition operators to the last item, and
+      // to merge literals at the and (we're parsing grouped ( sequences ) recursively and don't treat '|' specially
+      // (GBNF's syntax is luckily very close to regular expressions!)
+      const seq: Array<[string, boolean]> = []
+
+      const joinSeq = () => {
+        const ret = []
+        for (const [isLiteral, g] of groupBy(seq, (x) => x[1])) {
+          if (isLiteral) {
+            ret.push([[...g].map((x) => x[0]).join(''), true])
+          } else {
+            ret.push(...g)
+          }
+        }
+        if (ret.length === 1) {
+          return ret[0]
+        }
+        return [ret.map((x) => toRule(x)).join(' '), false]
+      }
+
+      while (i < length) {
+        const c = pattern[i]
+        if (c === '.') {
+          seq.push([getDot(), false])
+          i += 1
+        } else if (c === '(') {
+          i += 1
+          if (i < length) {
+            if (pattern[i] === '?') {
+              throw new Error(
+                `Unsupported pattern syntax "${pattern[i]}" at index ${i} of /${pattern}/`,
+              )
+            }
+          }
+          seq.push([`(${toRule(transform())})`, false])
+        } else if (c === ')') {
+          i += 1
+          if (start <= 0 || pattern[start - 1] !== '(') {
+            throw new Error(
+              `Unbalanced parentheses; start = ${start}, i = ${i}, pattern = ${pattern}`,
+            )
+          }
+          return joinSeq()
+        } else if (c === '[') {
+          let squareBrackets = c
+          i += 1
+          while (i < length && pattern[i] !== ']') {
+            if (pattern[i] === '\\') {
+              squareBrackets += pattern.slice(i, i + 2)
+              i += 2
+            } else {
+              squareBrackets += pattern[i]
+              i += 1
+            }
+          }
+          if (i >= length) {
+            throw new Error(
+              `Unbalanced square brackets; start = ${start}, i = ${i}, pattern = ${pattern}`,
+            )
+          }
+          squareBrackets += ']'
+          i += 1
+          seq.push([squareBrackets, false])
+        } else if (c === '|') {
+          seq.push(['|', false])
+          i += 1
+        } else if (c === '*' || c === '+' || c === '?') {
+          seq[seq.length - 1] = [
+            toRule(seq[seq.length - 1] || ['', false]) + c,
+            false,
+          ]
+          i += 1
+        } else if (c === '{') {
+          let curlyBrackets = c
+          i += 1
+          while (i < length && pattern[i] !== '}') {
+            curlyBrackets += pattern[i]
+            i += 1
+          }
+          if (i >= length) {
+            throw new Error(
+              `Unbalanced curly brackets; start = ${start}, i = ${i}, pattern = ${pattern}`,
+            )
+          }
+          curlyBrackets += '}'
+          i += 1
+          const nums = curlyBrackets
+            .slice(1, -1)
+            .split(',')
+            .map((s) => s.trim())
+          let minTimes: number
+          let maxTimes: number | undefined
+          if (nums.length === 1) {
+            minTimes = parseInt(nums[0] as string, 10)
+            maxTimes = minTimes
+          } else {
+            if (nums.length !== 2) {
+              throw new Error(`Invalid quantifier ${curlyBrackets}`)
+            }
+            minTimes = nums[0] ? parseInt(nums[0], 10) : 0
+            maxTimes = nums[1] ? parseInt(nums[1], 10) : Infinity
+          }
+
+          let [sub] = seq[seq.length - 1] || ['', false]
+          const [, subIsLiteral] = seq[seq.length - 1] || ['', false]
+
+          if (!subIsLiteral) {
+            let id = subRuleIds[sub]
+            if (id === undefined) {
+              id = this._addRule(
+                `${name}-${Object.keys(subRuleIds).length + 1}`,
+                sub,
+              )
+              subRuleIds[sub] = id
+            }
+            sub = id
+          }
+
+          seq[seq.length - 1] = [
+            buildRepetition(
+              subIsLiteral ? `"${sub}"` : sub,
+              minTimes,
+              maxTimes,
+              { itemRuleIsLiteral: subIsLiteral },
+            ),
+            false,
+          ]
+        } else {
+          let literal = ''
+          while (i < length) {
+            if (pattern[i] === '\\' && i < length - 1) {
+              const next = pattern[i + 1]
+              if (ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALS.has(next || '')) {
+                i += 1
+                literal += pattern[i]
+                i += 1
+              } else {
+                literal += pattern.slice(i, i + 2)
+                i += 2
+              }
+            } else if (pattern[i] === '"') {
+              literal += '\\"'
+              i += 1
+            } else if (
+              !NON_LITERAL_SET.has(pattern[i] || '') &&
+              (i === length - 1 ||
+                literal === '' ||
+                pattern[i + 1] === '.' ||
+                !NON_LITERAL_SET.has(pattern[i + 1] || ''))
+            ) {
+              literal += pattern[i]
+              i += 1
+            } else {
+              break
+            }
+          }
+          if (literal !== '') {
+            seq.push([literal, true])
+          }
+        }
+      }
+
+      return joinSeq()
+    }
 
-      return this.addRule(ruleName, rule)
+    return this._addRule(name, `"\\"" ${toRule(transform())} "\\"" space`)
+  }
+
+  _resolveRef(ref: string): string {
+    let refName = ref.split('/').pop() || ''
+    if (!(refName in this._rules) && !this._refsBeingResolved.has(ref)) {
+      this._refsBeingResolved.add(ref)
+      const resolved = this._refs[ref]
+      refName = this.visit(resolved, refName)
+      this._refsBeingResolved.delete(ref)
+    }
+    return refName
+  }
+
+  visit(schema: any, name: string): string {
+    const schemaType = schema.type
+    const schemaFormat = schema.format
+    const isRoot = name in RESERVED_NAMES ? `${name}-` : name == ''
+    const ruleName = isRoot ? 'root' : name
+
+    const ref = schema.$ref
+    if (ref !== undefined) {
+      return this._addRule(ruleName, this._resolveRef(ref))
+    } else if (schema.oneOf || schema.anyOf) {
+      return this._addRule(
+        ruleName,
+        this._generateUnionRule(name, schema.oneOf || schema.anyOf),
+      )
+    } else if (Array.isArray(schemaType)) {
+      return this._addRule(
+        ruleName,
+        this._generateUnionRule(
+          name,
+          schemaType.map((t) => ({ type: t })),
+        ),
+      )
     } else if ('const' in schema) {
-      return this.addRule(ruleName, formatLiteral(schema.const))
+      return this._addRule(ruleName, generateConstantRule(schema.const))
     } else if ('enum' in schema) {
-      const rule = schema.enum.map((v: string) => formatLiteral(v)).join(' | ')
-      return this.addRule(ruleName, rule)
-    } else if (schemaType === 'object' && 'properties' in schema) {
-      // TODO: `required` keyword (from python implementation)
-      const propOrder = this._propOrder
-      const propPairs = Object.entries(schema.properties).sort((a, b) => {
-        // sort by position in prop_order (if specified) then by key
-        const orderA = propOrder[a[0]] ?? Infinity
-        const orderB = propOrder[b[0]] ?? Infinity
-        return orderA - orderB || a[0].localeCompare(b[0])
-      })
+      const rule = schema.enum
+        .map((v: any) => generateConstantRule(v))
+        .join(' | ')
+      return this._addRule(ruleName, rule)
+    } else if (
+      (schemaType === undefined || schemaType === 'object') &&
+      ('properties' in schema ||
+        ('additionalProperties' in schema &&
+          schema.additionalProperties !== true))
+    ) {
+      const required: Set<string> = new Set(schema.required || [])
+      const properties = Object.entries(schema.properties ?? {})
+      return this._addRule(
+        ruleName,
+        this._buildObjectRule(
+          properties,
+          required,
+          name,
+          schema.additionalProperties,
+        ),
+      )
+    } else if (
+      (schemaType === undefined || schemaType === 'object') &&
+      'allOf' in schema
+    ) {
+      const required: Set<string> = new Set()
+      const properties: Array<[string, any]> = []
+      const addComponent = (compSchema: any, isRequired: boolean) => {
+        if (compSchema.$ref !== undefined) {
+          compSchema = this._refs[compSchema.$ref]
+        }
 
-      let rule = '"{" space'
-      propPairs.forEach(([propName, propSchema], i) => {
-        const propRuleName = this.visit(
-          propSchema,
-          `${name}${name ? '-' : ''}${propName}`,
-        )
-        if (i > 0) {
-          rule += ' "," space'
+        if ('properties' in compSchema) {
+          for (const [propName, propSchema] of Object.entries(
+            compSchema.properties,
+          )) {
+            properties.push([propName, propSchema])
+            if (isRequired) {
+              required.add(propName)
+            }
+          }
         }
-        rule += ` ${formatLiteral(propName)} space ":" space ${propRuleName}`
-      });
-      rule += ' "}" space'
-
-      return this.addRule(ruleName, rule)
-    } else if (schemaType === 'array' && 'items' in schema) {
-      // TODO `prefixItems` keyword (from python implementation)
-      const itemRuleName = this.visit(
-        schema.items,
-        `${name}${name ? '-' : ''}item`,
-      )
-      const rule = `"[" space (${itemRuleName} ("," space ${itemRuleName})*)? "]" space`
-      return this.addRule(ruleName, rule)
+      }
+
+      for (const t of schema.allOf) {
+        if ('anyOf' in t) {
+          for (const tt of t.anyOf) {
+            addComponent(tt, false)
+          }
+        } else {
+          addComponent(t, true)
+        }
+      }
+
+      return this._addRule(
+        ruleName,
+        this._buildObjectRule(
+          properties,
+          required,
+          name,
+          /* additionalProperties= */ false,
+        ),
+      )
+    } else if (
+      (schemaType === undefined || schemaType === 'array') &&
+      ('items' in schema || 'prefixItems' in schema)
+    ) {
+      const items = schema.items ?? schema.prefixItems
+      if (Array.isArray(items)) {
+        const rules = items
+          .map((item, i) =>
+            this.visit(item, `${name ?? ''}${name ? '-' : ''}tuple-${i}`),
+          )
+          .join(' "," space ')
+        return this._addRule(ruleName, `"[" space ${rules} "]" space`)
+      } else {
+        const itemRuleName = this.visit(
+          items,
+          `${name ?? ''}${name ? '-' : ''}item`,
+        )
+        const minItems = schema.minItems || 0
+        const { maxItems } = schema
+        return this._addRule(
+          ruleName,
+          `"[" space ${buildRepetition(itemRuleName, minItems, maxItems, {
+            separatorRule: '"," space',
+          })} "]" space`,
+        )
+      }
+    } else if (
+      (schemaType === undefined || schemaType === 'string') &&
+      'pattern' in schema
+    ) {
+      return this._visitPattern(schema.pattern, ruleName)
+    } else if (
+      (schemaType === undefined || schemaType === 'string') &&
+      /^uuid[1-5]?$/.test(schema.format || '')
+    ) {
+      return this._addPrimitive(
+        ruleName === 'root' ? 'root' : schemaFormat,
+        PRIMITIVE_RULES['uuid'],
+      )
+    } else if (
+      (schemaType === undefined || schemaType === 'string') &&
+      `${schema.format}-string` in STRING_FORMAT_RULES
+    ) {
+      const primName = `${schema.format}-string`
+      return this._addRule(
+        ruleName,
+        this._addPrimitive(primName, STRING_FORMAT_RULES[primName]),
+      )
+    } else if (
+      schemaType === 'string' &&
+      ('minLength' in schema || 'maxLength' in schema)
+    ) {
+      const charRuleName = this._addPrimitive('char', PRIMITIVE_RULES['char'])
+      const minLen = schema.minLength || 0
+      const maxLen = schema.maxLength
+      return this._addRule(
+        ruleName,
+        `"\\"" ${buildRepetition(charRuleName, minLen, maxLen)} "\\"" space`,
+      )
+    } else if (schemaType === 'object' || Object.keys(schema).length === 0) {
+      return this._addRule(
+        ruleName,
+        this._addPrimitive('object', PRIMITIVE_RULES['object']),
+      )
     } else {
-      if (!PRIMITIVE_RULES[schemaType]) {
+      if (!(schemaType in PRIMITIVE_RULES)) {
         throw new Error(`Unrecognized schema: ${JSON.stringify(schema)}`)
       }
-      return this.addRule(
+      // TODO: support minimum, maximum, exclusiveMinimum, exclusiveMaximum at least for zero
+      return this._addPrimitive(
         ruleName === 'root' ? 'root' : schemaType,
-        PRIMITIVE_RULES[schemaType] || '',
+        PRIMITIVE_RULES[schemaType],
       )
     }
   }
 
-  public formatGrammar(): string {
-    let grammar = '';
-    this._rules.forEach((rule, name) => {
-      grammar += `${name} ::= ${rule}\n`;
-    });
-    return grammar;
+  _addPrimitive(name: string, rule: BuiltinRule | undefined) {
+    if (!rule) {
+      throw new Error(`Rule ${name} not known`)
+    }
+    const n = this._addRule(name, rule.content)
+    for (const dep of rule.deps) {
+      const depRule = PRIMITIVE_RULES[dep] || STRING_FORMAT_RULES[dep]
+      if (!depRule) {
+        throw new Error(`Rule ${dep} not known`)
+      }
+      if (!(dep in this._rules)) {
+        this._addPrimitive(dep, depRule)
+      }
+    }
+    return n
+  }
+
+  _buildObjectRule(
+    properties: any[],
+    required: Set<string>,
+    name: string,
+    additionalProperties: any,
+  ) {
+    const propOrder = this._propOrder
+    // sort by position in prop_order (if specified) then by original order
+    const sortedProps = properties
+      .map(([k]) => k)
+      .sort((a, b) => {
+        const orderA = propOrder[a] || Infinity
+        const orderB = propOrder[b] || Infinity
+        return (
+          orderA - orderB ||
+          properties.findIndex(([k]) => k === a) -
+            properties.findIndex(([k]) => k === b)
+        )
+      })
+
+    const propKvRuleNames: { [key: string]: string } = {}
+    for (const [propName, propSchema] of properties) {
+      const propRuleName = this.visit(
+        propSchema,
+        `${name ?? ''}${name ? '-' : ''}${propName}`,
+      )
+      propKvRuleNames[propName] = this._addRule(
+        `${name ?? ''}${name ? '-' : ''}${propName}-kv`,
+        `${formatLiteral(
+          JSON.stringify(propName),
+        )} space ":" space ${propRuleName}`,
+      )
+    }
+    const requiredProps = sortedProps.filter((k) => required.has(k))
+    const optionalProps = sortedProps.filter((k) => !required.has(k))
+
+    if (
+      typeof additionalProperties === 'object' ||
+      additionalProperties === true
+    ) {
+      const subName = `${name ?? ''}${name ? '-' : ''}additional`
+      const valueRule = this.visit(
+        additionalProperties === true ? {} : additionalProperties,
+        `${subName}-value`,
+      )
+      propKvRuleNames['*'] = this._addRule(
+        `${subName}-kv`,
+        `${this._addPrimitive(
+          'string',
+          PRIMITIVE_RULES['string'],
+        )} ":" space ${valueRule}`,
+      )
+      optionalProps.push('*')
+    }
+
+    let rule = '"{" space '
+    rule += requiredProps.map((k) => propKvRuleNames[k]).join(' "," space ')
+
+    if (optionalProps.length > 0) {
+      rule += ' ('
+      if (requiredProps.length > 0) {
+        rule += ' "," space ( '
+      }
+
+      const getRecursiveRefs = (ks: any[], firstIsOptional: boolean) => {
+        const [k, ...rest] = ks
+        const kvRuleName = propKvRuleNames[k]
+        let res
+        if (k === '*') {
+          res = this._addRule(
+            `${name ?? ''}${name ? '-' : ''}additional-kvs`,
+            `${kvRuleName} ( "," space ${kvRuleName} )*`,
+          )
+        } else if (firstIsOptional) {
+          res = `( "," space ${kvRuleName} )?`
+        } else {
+          res = kvRuleName
+        }
+        if (rest.length > 0) {
+          res += ` ${this._addRule(
+            `${name ?? ''}${name ? '-' : ''}${k}-rest`,
+            getRecursiveRefs(rest, true) || '',
+          )}`
+        }
+        return res
+      }
+
+      rule += optionalProps
+        .map((_: any, i: number) =>
+          getRecursiveRefs(optionalProps.slice(i), false),
+        )
+        .join(' | ')
+      if (requiredProps.length > 0) {
+        rule += ' )'
+      }
+      rule += ' )?'
+    }
+
+    rule += ' "}" space'
+
+    return rule
+  }
+
+  formatGrammar() {
+    let grammar = ''
+    for (const [name, rule] of Object.entries(this._rules).sort(([a], [b]) =>
+      a.localeCompare(b),
+    )) {
+      grammar += `${name} ::= ${rule}\n`
+    }
+    return grammar
   }
 }
 
-export const convertJsonSchemaToGrammar = (
-  { schema, propOrder }: { schema: any; propOrder?: PropOrder },
-): string => {
-  const converter = new SchemaGrammarConverter(propOrder)
+export const convertJsonSchemaToGrammar = ({
+  schema,
+  propOrder,
+  dotall,
+  allowFetch,
+}: {
+  schema: any
+  propOrder?: PropOrder
+  dotall?: boolean
+  allowFetch?: boolean
+}): string | Promise<string> => {
+  const converter = new SchemaGrammarConverter({
+    prop_order: propOrder,
+    dotall,
+    allow_fetch: allowFetch,
+  })
+
+  if (allowFetch) {
+    return converter.resolveRefs(schema, '').then(() => {
+      converter.visit(schema, '')
+      return converter.formatGrammar()
+    })
+  }
   converter.visit(schema, '')
   return converter.formatGrammar()
 }