` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
- "\n",
- "JSON Object:\n"
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\n"
]
},
"metadata": {},
@@ -198,15 +266,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
}
],
@@ -230,7 +297,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
@@ -239,15 +306,15 @@
"{ \n",
" 'user_orders' : [ \n",
" { 'user_id' : 1 , 'user_name' : 'John Smith' , 'num_orders' : 10 , 'last_order_date' : '2020-01-01' } ,\n",
- " { 'user_id' : 2 , 'user_name' : 'Jane Doe' , 'num_orders' : 5 , 'last_order_date' : '2020-02-01' } ,\n",
- " { 'user_id' : 3 , 'user_name' : 'Bob Jones' , 'num_orders' : 25 , 'last_order_date' : '2020-03-01' } ,\n",
- " { 'user_id' : 4 , 'user_name' : 'Alice Johnson' , 'num_orders' : 0 , 'last_order_date' : '2020-04-01' } ,\n",
- " { 'user_id' : 5 , 'user_name' : 'John Doe' , 'num_orders' : 15 , 'last_order_date' : '2020-05-01' } ,\n",
- " { 'user_id' : 6 , 'user_name' : 'Jane Smith' , 'num_orders' : 20 , 'last_order_date' : '2020-06-01' } ,\n",
- " { 'user_id' : 7 , 'user_name' : 'Bob Johnson' , 'num_orders' : 10 , 'last_order_date' : '2020-07-01' } ,\n",
- " { 'user_id' : 8 , 'user_name' : 'Alice Jones' , 'num_orders' : 30 , 'last_order_date' : '2020-08-01' } ,\n",
- " { 'user_id' : 9 , 'user_name' : 'John Johnson' , 'num_orders' : 40 , 'last_order_date' : '2020-09-01' } ,\n",
- " { 'user_id' : 10 , 'user_name' : 'Jane Jones' , 'num_orders' : 35 , 'last_order_date' : '2020-10-01' } \n",
+ " { 'user_id' : 2 , 'user_name' : 'Jane Doe' , 'num_orders' : 20 , 'last_order_date' : '2020-02-01' } ,\n",
+ " { 'user_id' : 3 , 'user_name' : 'Bob Jones' , 'num_orders' : 30 , 'last_order_date' : '2020-03-01' } ,\n",
+ " { 'user_id' : 4 , 'user_name' : 'Alice Smith' , 'num_orders' : 40 , 'last_order_date' : '2020-04-01' } ,\n",
+ " { 'user_id' : 5 , 'user_name' : 'John Doe' , 'num_orders' : 50 , 'last_order_date' : '2020-05-01' } ,\n",
+ " { 'user_id' : 6 , 'user_name' : 'Jane Jones' , 'num_orders' : 0 , 'last_order_date' : '2020-06-01' } ,\n",
+ " { 'user_id' : 7 , 'user_name' : 'Bob Smith' , 'num_orders' : 10 , 'last_order_date' : '2020-07-01' } ,\n",
+ " { 'user_id' : 8 , 'user_name' : 'Alice Doe' , 'num_orders' : 20 , 'last_order_date' : '2020-08-01' } ,\n",
+ " { 'user_id' : 9 , 'user_name' : 'John Jones' , 'num_orders' : 30 , 'last_order_date' : '2020-09-01' } ,\n",
+ " { 'user_id' : 10 , 'user_name' : 'Jane Smith' , 'num_orders' : 40 , 'last_order_date' : '2020-10-01' } \n",
" ] \n",
"} \n",
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
" │ │ Generate a dataset of fake user orders. Each row of the dataset should be valid. │ │\n",
@@ -296,14 +363,14 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <list name=\"user_orders\" description=\"Generate a list of user, and how many orders they have placed │ │\n",
- " │ │ in the past.\" format=\"length: min=10 max=10\"> │ │\n",
+ " │ │ <list name=\"user_orders\" format=\"length: min=10 max=10\" description=\"Generate a list of user, and │ │\n",
+ " │ │ how many orders they have placed in the past.\"> │ │\n",
" │ │ <object> │ │\n",
- " │ │ <string name=\"user_id\" description=\"The user's id.\" format=\"1-indexed\"/> │ │\n",
- " │ │ <string name=\"user_name\" description=\"The user's first name and last name\" │ │\n",
- " │ │ format=\"two-words\"/> │ │\n",
- " │ │ <integer name=\"num_orders\" description=\"The number of orders the user has placed\" │ │\n",
- " │ │ format=\"valid-range: 0 50\"/> │ │\n",
+ " │ │ <string name=\"user_id\" format=\"1-indexed\" description=\"The user's id.\"/> │ │\n",
+ " │ │ <string name=\"user_name\" format=\"two-words\" description=\"The user's first name and last │ │\n",
+ " │ │ name\"/> │ │\n",
+ " │ │ <integer name=\"num_orders\" format=\"valid-range: min=0 max=50\" description=\"The number of │ │\n",
+ " │ │ orders the user has placed\"/> │ │\n",
" │ │ <date name=\"last_order_date\" description=\"Date of last order\"/> │ │\n",
" │ │ </object> │ │\n",
" │ │ </list> │ │\n",
@@ -323,10 +390,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
- " │ │ │ │\n",
" │ │ { │ │\n",
" │ │ \"user_orders\": [ │ │\n",
" │ │ { │ │\n",
@@ -338,55 +413,55 @@
" │ │ { │ │\n",
" │ │ \"user_id\": 2, │ │\n",
" │ │ \"user_name\": \"Jane Doe\", │ │\n",
- " │ │ \"num_orders\": 5, │ │\n",
+ " │ │ \"num_orders\": 20, │ │\n",
" │ │ \"last_order_date\": \"2020-02-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 3, │ │\n",
" │ │ \"user_name\": \"Bob Jones\", │ │\n",
- " │ │ \"num_orders\": 25, │ │\n",
+ " │ │ \"num_orders\": 30, │ │\n",
" │ │ \"last_order_date\": \"2020-03-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 4, │ │\n",
- " │ │ \"user_name\": \"Alice Johnson\", │ │\n",
- " │ │ \"num_orders\": 0, │ │\n",
+ " │ │ \"user_name\": \"Alice Smith\", │ │\n",
+ " │ │ \"num_orders\": 40, │ │\n",
" │ │ \"last_order_date\": \"2020-04-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 5, │ │\n",
" │ │ \"user_name\": \"John Doe\", │ │\n",
- " │ │ \"num_orders\": 15, │ │\n",
+ " │ │ \"num_orders\": 50, │ │\n",
" │ │ \"last_order_date\": \"2020-05-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 6, │ │\n",
- " │ │ \"user_name\": \"Jane Smith\", │ │\n",
- " │ │ \"num_orders\": 20, │ │\n",
+ " │ │ \"user_name\": \"Jane Jones\", │ │\n",
+ " │ │ \"num_orders\": 0, │ │\n",
" │ │ \"last_order_date\": \"2020-06-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 7, │ │\n",
- " │ │ \"user_name\": \"Bob Johnson\", │ │\n",
+ " │ │ \"user_name\": \"Bob Smith\", │ │\n",
" │ │ \"num_orders\": 10, │ │\n",
" │ │ \"last_order_date\": \"2020-07-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 8, │ │\n",
- " │ │ \"user_name\": \"Alice Jones\", │ │\n",
- " │ │ \"num_orders\": 30, │ │\n",
+ " │ │ \"user_name\": \"Alice Doe\", │ │\n",
+ " │ │ \"num_orders\": 20, │ │\n",
" │ │ \"last_order_date\": \"2020-08-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 9, │ │\n",
- " │ │ \"user_name\": \"John Johnson\", │ │\n",
- " │ │ \"num_orders\": 40, │ │\n",
+ " │ │ \"user_name\": \"John Jones\", │ │\n",
+ " │ │ \"num_orders\": 30, │ │\n",
" │ │ \"last_order_date\": \"2020-09-01\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"user_id\": 10, │ │\n",
- " │ │ \"user_name\": \"Jane Jones\", │ │\n",
- " │ │ \"num_orders\": 35, │ │\n",
+ " │ │ \"user_name\": \"Jane Smith\", │ │\n",
+ " │ │ \"num_orders\": 40, │ │\n",
" │ │ \"last_order_date\": \"2020-10-01\" │ │\n",
" │ │ } │ │\n",
" │ │ ] │ │\n",
@@ -404,55 +479,55 @@
" │ │ { │ │\n",
" │ │ 'user_id': 2, │ │\n",
" │ │ 'user_name': 'Jane Doe', │ │\n",
- " │ │ 'num_orders': 5, │ │\n",
+ " │ │ 'num_orders': 20, │ │\n",
" │ │ 'last_order_date': '2020-02-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 3, │ │\n",
" │ │ 'user_name': 'Bob Jones', │ │\n",
- " │ │ 'num_orders': 25, │ │\n",
+ " │ │ 'num_orders': 30, │ │\n",
" │ │ 'last_order_date': '2020-03-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 4, │ │\n",
- " │ │ 'user_name': 'Alice Johnson', │ │\n",
- " │ │ 'num_orders': 0, │ │\n",
+ " │ │ 'user_name': 'Alice Smith', │ │\n",
+ " │ │ 'num_orders': 40, │ │\n",
" │ │ 'last_order_date': '2020-04-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 5, │ │\n",
" │ │ 'user_name': 'John Doe', │ │\n",
- " │ │ 'num_orders': 15, │ │\n",
+ " │ │ 'num_orders': 50, │ │\n",
" │ │ 'last_order_date': '2020-05-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 6, │ │\n",
- " │ │ 'user_name': 'Jane Smith', │ │\n",
- " │ │ 'num_orders': 20, │ │\n",
+ " │ │ 'user_name': 'Jane Jones', │ │\n",
+ " │ │ 'num_orders': 0, │ │\n",
" │ │ 'last_order_date': '2020-06-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 7, │ │\n",
- " │ │ 'user_name': 'Bob Johnson', │ │\n",
+ " │ │ 'user_name': 'Bob Smith', │ │\n",
" │ │ 'num_orders': 10, │ │\n",
" │ │ 'last_order_date': '2020-07-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 8, │ │\n",
- " │ │ 'user_name': 'Alice Jones', │ │\n",
- " │ │ 'num_orders': 30, │ │\n",
+ " │ │ 'user_name': 'Alice Doe', │ │\n",
+ " │ │ 'num_orders': 20, │ │\n",
" │ │ 'last_order_date': '2020-08-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 9, │ │\n",
- " │ │ 'user_name': 'John Johnson', │ │\n",
- " │ │ 'num_orders': 40, │ │\n",
+ " │ │ 'user_name': 'John Jones', │ │\n",
+ " │ │ 'num_orders': 30, │ │\n",
" │ │ 'last_order_date': '2020-09-01' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'user_id': 10, │ │\n",
- " │ │ 'user_name': 'Jane Jones', │ │\n",
- " │ │ 'num_orders': 35, │ │\n",
+ " │ │ 'user_name': 'Jane Smith', │ │\n",
+ " │ │ 'num_orders': 40, │ │\n",
" │ │ 'last_order_date': '2020-10-01' │ │\n",
" │ │ } │ │\n",
" │ │ ] │ │\n",
@@ -463,7 +538,7 @@
],
"text/plain": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mGenerate a dataset of fake user orders. Each row of the dataset should be valid.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -473,14 +548,14 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m
\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -500,10 +575,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
" You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
"\n",
- " @json_suffix_prompt_examples\n",
+ " ${gr.json_suffix_prompt_examples}\n",
" \n",
"\n",
"\n",
@@ -96,33 +96,63 @@
" Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
" `null`.\n",
"\n",
- " {{document}}\n",
+ " ${document}\n",
"\n",
" Extract information from this document and return a JSON that follows the correct schema.\n",
"\n",
- " @xml_prefix_prompt\n",
+ " ${gr.xml_prefix_prompt}\n",
"\n",
- " {output_schema}\n",
+ " ${output_schema}\n",
" \n",
" ```\n",
"\n",
+ "=== \"Pydantic with instructions\"\n",
+ " ```py\n",
+ " instructions = \"\"\"You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
+ "\n",
+ " ${gr.json_suffix_prompt_examples}\"\"\"\n",
+ "\n",
+ " prompt = \"\"\"Given the following document, answer the following questions. If the answer doesn't exist in the document, enter `null`.\n",
+ "\n",
+ " ${document}\n",
+ "\n",
+ " Extract information from this document and return a JSON that follows the correct schema.\n",
+ "\n",
+ " ${gr.xml_prefix_prompt}\n",
+ "\n",
+ " ${output_schema}\"\"\"\n",
+ " ```\n",
+ "\n",
"=== \"RAIL Spec without instruction tag\"\n",
" \n",
" ```xml\n",
" \n",
" Given the following document, answer the following questions. If the answer doesn't exist in the document, enter `null`.\n",
"\n",
- " {{document}}\n",
+ " ${document}\n",
"\n",
- " @xml_prefix_prompt\n",
+ " ${gr.xml_prefix_prompt}\n",
"\n",
- " {output_schema}\n",
+ " ${output_schema}\n",
"\n",
- " @json_suffix_prompt_v2_wo_none\n",
+ " ${gr.json_suffix_prompt_v2_wo_none}\n",
" \n",
" ```\n",
"\n",
- "After materialization, the two specs will look like this:\n",
+ "=== \"Pydantic without instructions\"\n",
+ " ```py\n",
+ " prompt = \"\"\"Given the following document, answer the following questions. If the answer doesn't exist in the document, enter `null`.\n",
+ "\n",
+ " ${document}\n",
+ "\n",
+ " ${gr.xml_prefix_prompt}\n",
+ "\n",
+ " ${output_schema}\n",
+ "\n",
+ " ${gr.json_suffix_prompt_v2_wo_none}\"\"\"\n",
+ " ```\n",
+ "\n",
+ "After materialization, the two variations of the RAIL specification will look like this when passed to the LLM (regarless if they were initialized from xml or a Pydantic model):\n",
"\n",
"=== \"RAIL Spec with instruction tag\"\n",
"\n",
@@ -143,13 +173,13 @@
" Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
" `null`.\n",
"\n",
- " {{document}}\n",
+ " ${document}\n",
"\n",
" Extract information from this document and return a JSON that follows the correct schema.\n",
"\n",
" Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
- " {output_schema}\n",
+ " ${output_schema}\n",
" \n",
" ```\n",
"\n",
@@ -159,11 +189,11 @@
" \n",
" Given the following document, answer the following questions. If the answer doesn't exist in the document, enter `null`.\n",
"\n",
- " {{document}}\n",
+ " ${document}\n",
"\n",
" Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
- " {output_schema}\n",
+ " ${output_schema}\n",
"\n",
" ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.\n",
"\n",
@@ -174,12 +204,12 @@
" \n",
" ```\n",
"\n",
- "Here's the final RAIL spec:"
+ "Here's the final RAIL spec as XML:"
]
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 37,
"metadata": {},
"outputs": [],
"source": [
@@ -196,14 +226,14 @@
" \n",
"
\n",
- " \n",
"You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
"\n",
- "@json_suffix_prompt_examples\n",
+ "${gr.json_suffix_prompt_examples}\n",
" \n",
"\n",
"\n",
@@ -211,19 +241,62 @@
"Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
"`null`.\n",
"\n",
- "{{document}}\n",
+ "${document}\n",
"\n",
"Extract information from this document and return a JSON that follows the correct schema.\n",
"\n",
- "@xml_prefix_prompt\n",
+ "${gr.xml_prefix_prompt}\n",
"\n",
- "{output_schema}\n",
+ "${output_schema}\n",
"\n",
"\n",
"\n",
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or using a Pydantic model for the output schema:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from guardrails.validators import LowerCase, TwoWords, OneLine\n",
+ "from pydantic import BaseModel, Field\n",
+ "from typing import List, Optional\n",
+ "\n",
+ "instructions = \"\"\"You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
+ "\n",
+ "${gr.json_suffix_prompt_examples}\"\"\"\n",
+ "\n",
+ "prompt = \"\"\"Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
+ "`null`.\n",
+ "\n",
+ "${document}\n",
+ "\n",
+ "Extract information from this document and return a JSON that follows the correct schema.\n",
+ "\n",
+ "${gr.xml_prefix_prompt}\n",
+ "\n",
+ "${output_schema}\"\"\"\n",
+ "\n",
+ "class Fee(BaseModel):\n",
+ " index: int = Field(validators=[(\"1-indexed\", \"noop\")])\n",
+ " name: str = Field(validators=[LowerCase(on_fail=\"fix\"), TwoWords(on_fail=\"reask\")])\n",
+ " explanation: str = Field(validators=[OneLine()])\n",
+ " value: float = Field(validators=[(\"percentage\", \"noop\")])\n",
+ "\n",
+ "class CreditCardAgreement(BaseModel):\n",
+ " fees: List[Fee] = Field(description=\"What fees and charges are associated with my account?\")\n",
+ " interest_rates: Optional[dict] = Field(description=\"What are the interest rates offered by the bank on savings and checking accounts, loans, and credit products?\")"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -237,18 +310,25 @@
"3. Compiles the schema and type info from the RAIL spec and adds it to the prompt."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Creating the guard from XML:"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 42,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/guardrails/guardrails/schema.py:197: UserWarning: Validator 1-indexed is not valid for element integer.\n",
+ "/Users/calebcourier/Projects/shreyar/guardrails/guardrails/schema.py:218: UserWarning: Validator 1-indexed is not valid for element integer.\n",
" warnings.warn(\n",
- "/Users/shreyarajpal/guardrails/guardrails/schema.py:197: UserWarning: Validator percentage is not valid for element float.\n",
+ "/Users/calebcourier/Projects/shreyar/guardrails/guardrails/schema.py:218: UserWarning: Validator percentage is not valid for element float.\n",
" warnings.warn(\n"
]
}
@@ -257,6 +337,22 @@
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or from the Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=CreditCardAgreement, instructions=instructions, prompt=prompt)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -268,17 +364,16 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 44,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "\n",
- "Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
+ "Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
"`null`.\n",
"\n",
- "{ document} \n",
+ "${ document} \n",
"\n",
"Extract information from this document and return a JSON that follows the correct schema.\n",
"\n",
@@ -299,15 +394,13 @@
"checking accounts, loans, and credit products?\" / > \n",
"< / output > \n",
"\n",
- "\n",
" \n"
],
"text/plain": [
- "\n",
"Given the following document, answer the following questions. If the answer doesn't exist in the document, enter \n",
"`null`.\n",
"\n",
- "\u001b[1m{\u001b[0mdocument\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mdocument\u001b[1m}\u001b[0m\n",
"\n",
"Extract information from this document and return a JSON that follows the correct schema.\n",
"\n",
@@ -327,7 +420,6 @@
"\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[1m>\u001b[0m\n",
- "\n",
"\n"
]
},
@@ -349,46 +441,42 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 45,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "\n",
- "You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
+ "You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
"\n",
"\n",
"ONLY return a valid JSON object ( no other text is necessary) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter `None `.\n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`.\n",
"\n",
"Here are examples of simple ( XML, JSON) pairs that show the expected behavior:\n",
- "- `< string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `< string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
- "\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
" \n"
],
"text/plain": [
- "\n",
"You are a helpful assistant only capable of communicating with valid JSON, and no other text.\n",
"\n",
"\n",
"ONLY return a valid JSON object \u001b[1m(\u001b[0mno other text is necessary\u001b[1m)\u001b[0m, where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[3;35mNone\u001b[0m`.\n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`.\n",
"\n",
"Here are examples of simple \u001b[1m(\u001b[0mXML, JSON\u001b[1m)\u001b[0m pairs that show the expected behavior:\n",
- "- `\u001b[1m<\u001b[0m\u001b[1;95mstring\u001b[0m\u001b[39m \u001b[0m\u001b[33mname\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m \u001b[0m\u001b[33mformat\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'two-words lower-case'\u001b[0m\u001b[39m \u001b[0m\u001b[35m/\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "- `\u001b[1m<\u001b[0m\u001b[1;95mstring\u001b[0m\u001b[39m \u001b[0m\u001b[33mname\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m \u001b[0m\u001b[33mformat\u001b[0m\u001b[39m=\u001b[0m\u001b[32m'two-words lower-case'\u001b[0m\u001b[39m \u001b[0m\u001b[35m/\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
- "\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n"
]
},
@@ -414,18 +502,9 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 46,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"import openai\n",
"\n",
@@ -449,165 +528,105 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 47,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "{ \n",
- " 'fees' : [ \n",
- " { 'index' : 1 , 'name' : 'annual membership' , 'explanation' : 'None' , 'value' : 0.0 } ,\n",
- " { \n",
- " 'index' : 2 ,\n",
- " 'name' : 'My Chase' ,\n",
- " 'explanation' : 'monthly fee of 0% of the amount of each eligible purchase transaction or amount \n",
- "selected to create a my chase plan while in the 0% intro purchase apr period. after that, monthly fee of 1.72% of \n",
- "the amount of each eligible purchase transaction or amount selected to create a my chase plan. the my chase plan \n",
- "fee will be determined at the time each my chase plan is created and will remain the same until the my chase plan \n",
+ "SkeletonReAsk ( \n",
+ " incorrect_value ={ \n",
+ " 'fees' : [ \n",
+ " { 'index' : 1 , 'name' : 'annual membership fee' , 'explanation' : 'None' , 'value' : 0.0 } ,\n",
+ " { \n",
+ " 'index' : 2 ,\n",
+ " 'name' : 'my chase plan fee' ,\n",
+ " 'explanation' : 'Monthly fee of 0% of the amount of each eligible purchase transaction or amount \n",
+ "selected to create a My Chase Plan while in the 0% Intro Purchase APR period. After that, monthly fee of 1.72% of \n",
+ "the amount of each eligible purchase transaction or amount selected to create a My Chase Plan. The My Chase Plan \n",
+ "Fee will be determined at the time each My Chase Plan is created and will remain the same until the My Chase Plan \n",
"is paid in full.' ,\n",
- " 'value' : 1.72 \n",
- " } ,\n",
- " { \n",
- " 'index' : 3 ,\n",
- " 'name' : 'balance transfers' ,\n",
- " 'explanation' : 'intro fee of either $5 or 3% of the amount of each transfer, whichever is greater, on \n",
- "transfers made within 60 days of account opening. after that: either $5 or 5% of the amount of each transfer, \n",
- "whichever is greater.' ,\n",
- " 'value' : 5.0 \n",
- " } ,\n",
- " { \n",
- " 'index' : 4 ,\n",
- " 'name' : 'cash advances' ,\n",
- " 'explanation' : 'Either $10 or 5% of the amount of each transaction, whichever is greater.' ,\n",
- " 'value' : 5.0 \n",
- " } ,\n",
- " { \n",
- " 'index' : 5 ,\n",
- " 'name' : 'foreign transactions' ,\n",
- " 'explanation' : '3% of the amount of each transaction in U.S. dollars.' ,\n",
- " 'value' : 3.0 \n",
- " } ,\n",
- " { 'index' : 6 , 'name' : 'Late Payment' , 'explanation' : 'Up to $40.' , 'value' : None } ,\n",
- " { 'index' : 7 , 'name' : 'over-the-credit-limit penalty' , 'explanation' : 'None' , 'value' : None } ,\n",
- " { 'index' : 8 , 'name' : 'Return Payment' , 'explanation' : 'Up to $40.' , 'value' : None } ,\n",
- " { 'index' : 9 , 'name' : 'Return Check' , 'explanation' : 'None' , 'value' : None } \n",
- " ] ,\n",
- " 'interest_rates' : { \n",
- " 'purchase annual percentage rate (apr)' : { \n",
- " 'introductory rate' : 0.0 ,\n",
- " 'introductory period' : 'first 18 months' ,\n",
- " 'standard rate' : 19.49 ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The interest rate charged on purchases made with the credit card.' \n",
- " } ,\n",
- " 'my chase loan sm apr' : { \n",
- " 'rate' : 19.49 ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The interest rate charged on My Chase Loan balances.' \n",
- " } ,\n",
- " 'balance transfer apr' : { \n",
- " 'introductory rate' : 0.0 ,\n",
- " 'introductory period' : 'first 18 months' ,\n",
- " 'standard rate' : 19.49 ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The interest rate charged on balances transferred to the credit card.' \n",
- " } ,\n",
- " 'cash advance apr' : { \n",
- " 'rate' : 29.49 ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The interest rate charged on cash advances taken with the credit card.' \n",
- " } ,\n",
- " 'penalty apr and when it applies' : { \n",
- " 'rate' : 'Up to 29.99%' ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The interest rate charged on the credit card when a penalty is applied.' \n",
- " } ,\n",
- " 'maximum apr' : { \n",
- " 'rate' : 29.99 ,\n",
- " 'variable' : True ,\n",
- " 'description' : 'The highest interest rate that can be charged on the credit card.' \n",
- " } \n",
- " } \n",
- "} \n",
+ " 'value' : 1.72 \n",
+ " } ,\n",
+ " { \n",
+ " 'index' : 3 ,\n",
+ " 'name' : 'balance transfers intro fee' ,\n",
+ " 'explanation' : 'Either $5 or 3% of the amount of each transfer, whichever is greater, on transfers \n",
+ "made within 60 days of account opening. After that: Either $5 or 5% of the amount of each transfer, whichever is \n",
+ "greater.' ,\n",
+ " 'value' : 5.0 \n",
+ " } ,\n",
+ " { \n",
+ " 'index' : 4 ,\n",
+ " 'name' : 'cash advances' ,\n",
+ " 'explanation' : 'Either $10 or 5% of the amount of each transaction, whichever is greater.' ,\n",
+ " 'value' : 5.0 \n",
+ " } ,\n",
+ " { \n",
+ " 'index' : 5 ,\n",
+ " 'name' : 'foreign transactions' ,\n",
+ " 'explanation' : '3% of the amount of each transaction in U.S. dollars.' ,\n",
+ " 'value' : 3.0 \n",
+ " } ,\n",
+ " { 'index' : 6 , 'name' : 'late payment' , 'explanation' : 'Up to $40.' , 'value' : 40.0 } ,\n",
+ " { 'index' : 7 , 'name' : 'over-the-credit-limit' , 'explanation' : 'None' , 'value' : None } ,\n",
+ " { 'index' : 8 , 'name' : 'return payment' , 'explanation' : 'Up to $40.' , 'value' : 40.0 } ,\n",
+ " { 'index' : 9 , 'name' : 'return check' , 'explanation' : 'None' , 'value' : None } \n",
+ " ] ,\n",
+ " 'interest_rates' : None \n",
+ " } ,\n",
+ " fail_results =[ \n",
+ " FailResult ( outcome ='fail' , metadata =None , error_message ='JSON does not match schema' , fix_value =None ) \n",
+ " ] \n",
+ ") \n",
"Logs\n",
"├── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
"│ │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
- "│ │ │ │ │\n",
"│ │ │ Given the following document, answer the following questions. If the answer doesn't exist in the │ │\n",
"│ │ │ document, enter │ │\n",
"│ │ │ `null`. │ │\n",
@@ -777,10 +795,8 @@
"│ │ │ savings and checking accounts, loans, and credit products?\"/> │ │\n",
"│ │ │ </output> │ │\n",
"│ │ │ │ │\n",
- "│ │ │ │ │\n",
"│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
"│ │ ╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮ │\n",
- "│ │ │ │ │\n",
"│ │ │ You are a helpful assistant only capable of communicating with valid JSON, and no other text. │ │\n",
"│ │ │ │ │\n",
"│ │ │ │ │\n",
@@ -797,256 +813,157 @@
"│ │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
"│ │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
"│ │ │ │ │\n",
- "│ │ │ │ │\n",
"│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ "│ │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ "│ │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ "│ │ │ ┃ Role ┃ Content ┃ │ │\n",
+ "│ │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ "│ │ │ └──────┴─────────┘ │ │\n",
+ "│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
"│ │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
"│ │ │ { │ │\n",
- "│ │ │ \"fees\": [ │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 1, │ │\n",
- "│ │ │ \"name\": \"annual membership fee\", │ │\n",
- "│ │ │ \"explanation\": \"None\", │ │\n",
- "│ │ │ \"value\": 0.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 2, │ │\n",
- "│ │ │ \"name\": \"my chase plan sm fee\", │ │\n",
- "│ │ │ \"explanation\": \"monthly fee of 0% of the amount of each eligible purchase transaction or │ │\n",
- "│ │ │ amount selected to create a my chase plan while in the 0% intro purchase apr period. after that, │ │\n",
- "│ │ │ monthly fee of 1.72% of the amount of each eligible purchase transaction or amount selected to create a │ │\n",
- "│ │ │ my chase plan. the my chase plan fee will be determined at the time each my chase plan is created and │ │\n",
- "│ │ │ will remain the same until the my chase plan is paid in full.\", │ │\n",
- "│ │ │ \"value\": 1.72 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 3, │ │\n",
- "│ │ │ \"name\": \"balance transfers transaction fee\", │ │\n",
- "│ │ │ \"explanation\": \"intro fee of either $5 or 3% of the amount of each transfer, whichever is │ │\n",
- "│ │ │ greater, on transfers made within 60 days of account opening. after that: either $5 or 5% of the amount │ │\n",
- "│ │ │ of each transfer, whichever is greater.\", │ │\n",
- "│ │ │ \"value\": 5.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 4, │ │\n",
- "│ │ │ \"name\": \"cash advances transaction fee\", │ │\n",
- "│ │ │ \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\", │ │\n",
- "│ │ │ \"value\": 5.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 5, │ │\n",
- "│ │ │ \"name\": \"foreign transactions transaction fee\", │ │\n",
- "│ │ │ \"explanation\": \"3% of the amount of each transaction in U.S. dollars.\", │ │\n",
- "│ │ │ \"value\": 3.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 6, │ │\n",
- "│ │ │ \"name\": \"late payment penalty fee\", │ │\n",
- "│ │ │ \"explanation\": \"Up to $40.\", │ │\n",
- "│ │ │ \"value\": null │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 7, │ │\n",
- "│ │ │ \"name\": \"over-the-credit-limit penalty fee\", │ │\n",
- "│ │ │ \"explanation\": \"None\", │ │\n",
- "│ │ │ \"value\": null │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 8, │ │\n",
- "│ │ │ \"name\": \"return payment penalty fee\", │ │\n",
- "│ │ │ \"explanation\": \"Up to $40.\", │ │\n",
- "│ │ │ \"value\": null │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ \"index\": 9, │ │\n",
- "│ │ │ \"name\": \"return check penalty fee\", │ │\n",
- "│ │ │ \"explanation\": \"None\", │ │\n",
- "│ │ │ \"value\": null │ │\n",
- "│ │ │ } │ │\n",
- "│ │ │ ], │ │\n",
- "│ │ │ \"interest_rates\": { │ │\n",
- "│ │ │ \"purchase annual percentage rate (apr)\": { │ │\n",
- "│ │ │ \"introductory rate\": 0.0, │ │\n",
- "│ │ │ \"introductory period\": \"first 18 months\", │ │\n",
- "│ │ │ \"standard rate\": 19.49, │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The interest rate charged on purchases made with the credit card.\" │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ \"my chase loan sm apr\": { │ │\n",
- "│ │ │ \"rate\": 19.49, │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The interest rate charged on My Chase Loan balances.\" │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ \"balance transfer apr\": { │ │\n",
- "│ │ │ \"introductory rate\": 0.0, │ │\n",
- "│ │ │ \"introductory period\": \"first 18 months\", │ │\n",
- "│ │ │ \"standard rate\": 19.49, │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The interest rate charged on balances transferred to the credit card.\" │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ \"cash advance apr\": { │ │\n",
- "│ │ │ \"rate\": 29.49, │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The interest rate charged on cash advances taken with the credit card.\" │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ \"penalty apr and when it applies\": { │ │\n",
- "│ │ │ \"rate\": \"Up to 29.99%\", │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The interest rate charged on the credit card when a penalty is applied.\" │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ \"maximum apr\": { │ │\n",
- "│ │ │ \"rate\": 29.99, │ │\n",
- "│ │ │ \"variable\": true, │ │\n",
- "│ │ │ \"description\": \"The highest interest rate that can be charged on the credit card.\" │ │\n",
- "│ │ │ } │ │\n",
+ "│ │ │ \"fees\": [ │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 1, │ │\n",
+ "│ │ │ \"name\": \"annual membership fee\", │ │\n",
+ "│ │ │ \"explanation\": \"None\", │ │\n",
+ "│ │ │ \"value\": 0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 2, │ │\n",
+ "│ │ │ \"name\": \"my chase plan fee\", │ │\n",
+ "│ │ │ \"explanation\": \"Monthly fee of 0% of the amount of each eligible purchase transaction or amount │ │\n",
+ "│ │ │ selected to create a My Chase Plan while in the 0% Intro Purchase APR period.\\nAfter that, monthly fee │ │\n",
+ "│ │ │ of 1.72% of the amount of each eligible purchase transaction or amount selected to create a My Chase │ │\n",
+ "│ │ │ Plan. The My Chase Plan Fee will be determined at the time each My Chase Plan is created and will │ │\n",
+ "│ │ │ remain the same until the My Chase Plan is paid in full.\", │ │\n",
+ "│ │ │ \"value\": 1.72 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 3, │ │\n",
+ "│ │ │ \"name\": \"balance transfers intro fee\", │ │\n",
+ "│ │ │ \"explanation\": \"Either $5 or 3% of the amount of each transfer, whichever is greater, on │ │\n",
+ "│ │ │ transfers made within 60 days of account opening. After that: Either $5 or 5% of the amount of each │ │\n",
+ "│ │ │ transfer, whichever is greater.\", │ │\n",
+ "│ │ │ \"value\": 5 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 4, │ │\n",
+ "│ │ │ \"name\": \"cash advances\", │ │\n",
+ "│ │ │ \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\", │ │\n",
+ "│ │ │ \"value\": 5 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 5, │ │\n",
+ "│ │ │ \"name\": \"foreign transactions\", │ │\n",
+ "│ │ │ \"explanation\": \"3% of the amount of each transaction in U.S. dollars.\", │ │\n",
+ "│ │ │ \"value\": 3 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 6, │ │\n",
+ "│ │ │ \"name\": \"late payment\", │ │\n",
+ "│ │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ "│ │ │ \"value\": 40 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 7, │ │\n",
+ "│ │ │ \"name\": \"over-the-credit-limit\", │ │\n",
+ "│ │ │ \"explanation\": \"None\", │ │\n",
+ "│ │ │ \"value\": null │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 8, │ │\n",
+ "│ │ │ \"name\": \"return payment\", │ │\n",
+ "│ │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ "│ │ │ \"value\": 40 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ \"index\": 9, │ │\n",
+ "│ │ │ \"name\": \"return check\", │ │\n",
+ "│ │ │ \"explanation\": \"None\", │ │\n",
+ "│ │ │ \"value\": null │ │\n",
"│ │ │ } │ │\n",
+ "│ │ │ ], │ │\n",
+ "│ │ │ \"interest_rates\": null │ │\n",
"│ │ │ } │ │\n",
"│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
"│ │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'fees': [ │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 1, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='annual membership fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='annual membership', │ │\n",
- "│ │ │ path=['fees', 0, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'None', │ │\n",
- "│ │ │ 'value': 0.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 2, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='my chase plan sm fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='my chase', │ │\n",
- "│ │ │ path=['fees', 1, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'monthly fee of 0% of the amount of each eligible purchase transaction or │ │\n",
- "│ │ │ amount selected to create a my chase plan while in the 0% intro purchase apr period. after that, │ │\n",
+ "│ │ │ SkeletonReAsk( │ │\n",
+ "│ │ │ incorrect_value={ │ │\n",
+ "│ │ │ 'fees': [ │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 1, │ │\n",
+ "│ │ │ 'name': 'annual membership fee', │ │\n",
+ "│ │ │ 'explanation': 'None', │ │\n",
+ "│ │ │ 'value': 0.0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 2, │ │\n",
+ "│ │ │ 'name': 'my chase plan fee', │ │\n",
+ "│ │ │ 'explanation': 'Monthly fee of 0% of the amount of each eligible purchase transaction │ │\n",
+ "│ │ │ or amount selected to create a My Chase Plan while in the 0% Intro Purchase APR period.\\nAfter that, │ │\n",
"│ │ │ monthly fee of 1.72% of the amount of each eligible purchase transaction or amount selected to create a │ │\n",
- "│ │ │ my chase plan. the my chase plan fee will be determined at the time each my chase plan is created and │ │\n",
- "│ │ │ will remain the same until the my chase plan is paid in full.', │ │\n",
- "│ │ │ 'value': 1.72 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 3, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='balance transfers transaction fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='balance transfers', │ │\n",
- "│ │ │ path=['fees', 2, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'intro fee of either $5 or 3% of the amount of each transfer, whichever is │ │\n",
- "│ │ │ greater, on transfers made within 60 days of account opening. after that: either $5 or 5% of the amount │ │\n",
- "│ │ │ of each transfer, whichever is greater.', │ │\n",
- "│ │ │ 'value': 5.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 4, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='cash advances transaction fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='cash advances', │ │\n",
- "│ │ │ path=['fees', 3, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is greater.', │ │\n",
- "│ │ │ 'value': 5.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 5, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='foreign transactions transaction fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='foreign transactions', │ │\n",
- "│ │ │ path=['fees', 4, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': '3% of the amount of each transaction in U.S. dollars.', │ │\n",
- "│ │ │ 'value': 3.0 │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 6, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='late payment penalty fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='late payment', │ │\n",
- "│ │ │ path=['fees', 5, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'Up to $40.', │ │\n",
- "│ │ │ 'value': None │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 7, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='over-the-credit-limit penalty fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='over-the-credit-limit penalty', │ │\n",
- "│ │ │ path=['fees', 6, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'None', │ │\n",
- "│ │ │ 'value': None │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 8, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='return payment penalty fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='return payment', │ │\n",
- "│ │ │ path=['fees', 7, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'Up to $40.', │ │\n",
- "│ │ │ 'value': None │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ { │ │\n",
- "│ │ │ 'index': 9, │ │\n",
- "│ │ │ 'name': ReAsk( │ │\n",
- "│ │ │ incorrect_value='return check penalty fee', │ │\n",
- "│ │ │ error_message='must be exactly two words', │ │\n",
- "│ │ │ fix_value='return check', │ │\n",
- "│ │ │ path=['fees', 8, 'name'] │ │\n",
- "│ │ │ ), │ │\n",
- "│ │ │ 'explanation': 'None', │ │\n",
- "│ │ │ 'value': None │ │\n",
- "│ │ │ } │ │\n",
- "│ │ │ ], │ │\n",
- "│ │ │ 'interest_rates': { │ │\n",
- "│ │ │ 'purchase annual percentage rate (apr)': { │ │\n",
- "│ │ │ 'introductory rate': 0.0, │ │\n",
- "│ │ │ 'introductory period': 'first 18 months', │ │\n",
- "│ │ │ 'standard rate': 19.49, │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The interest rate charged on purchases made with the credit card.' │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ 'my chase loan sm apr': { │ │\n",
- "│ │ │ 'rate': 19.49, │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The interest rate charged on My Chase Loan balances.' │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ 'balance transfer apr': { │ │\n",
- "│ │ │ 'introductory rate': 0.0, │ │\n",
- "│ │ │ 'introductory period': 'first 18 months', │ │\n",
- "│ │ │ 'standard rate': 19.49, │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The interest rate charged on balances transferred to the credit card.' │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ 'cash advance apr': { │ │\n",
- "│ │ │ 'rate': 29.49, │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The interest rate charged on cash advances taken with the credit card.' │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ 'penalty apr and when it applies': { │ │\n",
- "│ │ │ 'rate': 'Up to 29.99%', │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The interest rate charged on the credit card when a penalty is applied.' │ │\n",
- "│ │ │ }, │ │\n",
- "│ │ │ 'maximum apr': { │ │\n",
- "│ │ │ 'rate': 29.99, │ │\n",
- "│ │ │ 'variable': True, │ │\n",
- "│ │ │ 'description': 'The highest interest rate that can be charged on the credit card.' │ │\n",
- "│ │ │ } │ │\n",
- "│ │ │ } │ │\n",
- "│ │ │ } │ │\n",
+ "│ │ │ My Chase Plan. The My Chase Plan Fee will be determined at the time each My Chase Plan is created and │ │\n",
+ "│ │ │ will remain the same until the My Chase Plan is paid in full.', │ │\n",
+ "│ │ │ 'value': 1.72 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 3, │ │\n",
+ "│ │ │ 'name': 'balance transfers intro fee', │ │\n",
+ "│ │ │ 'explanation': 'Either $5 or 3% of the amount of each transfer, whichever is greater, │ │\n",
+ "│ │ │ on transfers made within 60 days of account opening. After that: Either $5 or 5% of the amount of each │ │\n",
+ "│ │ │ transfer, whichever is greater.', │ │\n",
+ "│ │ │ 'value': 5.0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 4, │ │\n",
+ "│ │ │ 'name': 'cash advances', │ │\n",
+ "│ │ │ 'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is │ │\n",
+ "│ │ │ greater.', │ │\n",
+ "│ │ │ 'value': 5.0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 5, │ │\n",
+ "│ │ │ 'name': 'foreign transactions', │ │\n",
+ "│ │ │ 'explanation': '3% of the amount of each transaction in U.S. dollars.', │ │\n",
+ "│ │ │ 'value': 3.0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 6, │ │\n",
+ "│ │ │ 'name': 'late payment', │ │\n",
+ "│ │ │ 'explanation': 'Up to $40.', │ │\n",
+ "│ │ │ 'value': 40.0 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 7, │ │\n",
+ "│ │ │ 'name': 'over-the-credit-limit', │ │\n",
+ "│ │ │ 'explanation': 'None', │ │\n",
+ "│ │ │ 'value': None │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 8, │ │\n",
+ "│ │ │ 'name': 'return payment', │ │\n",
+ "│ │ │ 'explanation': 'Up to $40.', │ │\n",
+ "│ │ │ 'value': 40 │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'index': 9, │ │\n",
+ "│ │ │ 'name': 'return check', │ │\n",
+ "│ │ │ 'explanation': 'None', │ │\n",
+ "│ │ │ 'value': None │ │\n",
+ "│ │ │ } │ │\n",
+ "│ │ │ ], │ │\n",
+ "│ │ │ 'interest_rates': None │ │\n",
+ "│ │ │ }, │ │\n",
+ "│ │ │ fail_results=[ │ │\n",
+ "│ │ │ FailResult( │ │\n",
+ "│ │ │ outcome='fail', │ │\n",
+ "│ │ │ metadata=None, │ │\n",
+ "│ │ │ error_message='JSON does not match schema', │ │\n",
+ "│ │ │ fix_value=None │ │\n",
+ "│ │ │ ) │ │\n",
+ "│ │ │ ] │ │\n",
+ "│ │ │ ) │ │\n",
"│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
"│ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
@@ -1055,61 +972,73 @@
" │ │ I was given the following JSON response, which had problems due to incorrect values. │ │\n",
" │ │ │ │\n",
" │ │ { │ │\n",
- " │ │ \"fees\": [ │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"annual membership fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
+ " │ │ \"incorrect_value\": { │ │\n",
+ " │ │ \"fees\": [ │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 1, │ │\n",
+ " │ │ \"name\": \"annual membership fee\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": 0.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 2, │ │\n",
+ " │ │ \"name\": \"my chase plan fee\", │ │\n",
+ " │ │ \"explanation\": \"Monthly fee of 0% of the amount of each eligible purchase transaction or amount │ │\n",
+ " │ │ selected to create a My Chase Plan while in the 0% Intro Purchase APR period.\\nAfter that, monthly fee │ │\n",
+ " │ │ of 1.72% of the amount of each eligible purchase transaction or amount selected to create a My Chase │ │\n",
+ " │ │ Plan. The My Chase Plan Fee will be determined at the time each My Chase Plan is created and will │ │\n",
+ " │ │ remain the same until the My Chase Plan is paid in full.\", │ │\n",
+ " │ │ \"value\": 1.72 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 3, │ │\n",
+ " │ │ \"name\": \"balance transfers intro fee\", │ │\n",
+ " │ │ \"explanation\": \"Either $5 or 3% of the amount of each transfer, whichever is greater, on │ │\n",
+ " │ │ transfers made within 60 days of account opening. After that: Either $5 or 5% of the amount of each │ │\n",
+ " │ │ transfer, whichever is greater.\", │ │\n",
+ " │ │ \"value\": 5.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 4, │ │\n",
+ " │ │ \"name\": \"cash advances\", │ │\n",
+ " │ │ \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\", │ │\n",
+ " │ │ \"value\": 5.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 5, │ │\n",
+ " │ │ \"name\": \"foreign transactions\", │ │\n",
+ " │ │ \"explanation\": \"3% of the amount of each transaction in U.S. dollars.\", │ │\n",
+ " │ │ \"value\": 3.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 6, │ │\n",
+ " │ │ \"name\": \"late payment\", │ │\n",
+ " │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ " │ │ \"value\": 40.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 7, │ │\n",
+ " │ │ \"name\": \"over-the-credit-limit\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": null │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 8, │ │\n",
+ " │ │ \"name\": \"return payment\", │ │\n",
+ " │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ " │ │ \"value\": 40 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"index\": 9, │ │\n",
+ " │ │ \"name\": \"return check\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": null │ │\n",
" │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"my chase plan sm fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"balance transfers transaction fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"cash advances transaction fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"foreign transactions transaction fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"late payment penalty fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"over-the-credit-limit penalty fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"return payment penalty fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"name\": { │ │\n",
- " │ │ \"incorrect_value\": \"return check penalty fee\", │ │\n",
- " │ │ \"error_message\": \"must be exactly two words\" │ │\n",
- " │ │ } │ │\n",
- " │ │ } │ │\n",
+ " │ │ ], │ │\n",
+ " │ │ \"interest_rates\": null │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ \"error_messages\": [ │ │\n",
+ " │ │ \"JSON does not match schema\" │ │\n",
" │ │ ] │ │\n",
" │ │ } │ │\n",
" │ │ │ │\n",
@@ -1121,9 +1050,14 @@
" │ │ <output> │ │\n",
" │ │ <list name=\"fees\" description=\"What fees and charges are associated with my account?\"> │ │\n",
" │ │ <object> │ │\n",
+ " │ │ <integer name=\"index\" format=\"1-indexed\"/> │ │\n",
" │ │ <string name=\"name\" format=\"lower-case; two-words\"/> │ │\n",
+ " │ │ <string name=\"explanation\" format=\"one-line\"/> │ │\n",
+ " │ │ <float name=\"value\" format=\"percentage\"/> │ │\n",
" │ │ </object> │ │\n",
" │ │ </list> │ │\n",
+ " │ │ <object name=\"interest_rates\" description=\"What are the interest rates offered by the bank on │ │\n",
+ " │ │ savings and checking accounts, loans, and credit products?\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -1133,19 +1067,11 @@
" │ │ requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, │ │\n",
" │ │ enter `null`. │ │\n",
" │ │ │ │\n",
- " │ │ Here are examples of simple (XML, JSON) pairs that show the expected behavior: │ │\n",
- " │ │ - `<string name='foo' format='two-words lower-case' />` => `{{'foo': 'example one'}}` │ │\n",
- " │ │ - `<list name='bar'><string format='upper-case' /></list>` => `{{\"bar\": ['STRING ONE', 'STRING TWO', │ │\n",
- " │ │ etc.]}}` │ │\n",
- " │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
- " │ │ format=\"1-indexed\" /></object>` => `{{'baz': {{'foo': 'Some String', 'index': 1}}}}` │ │\n",
- " │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
" │ │ You are a helpful assistant only capable of communicating with valid JSON, and no other text. │ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ │ ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the │ │\n",
" │ │ `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding │ │\n",
" │ │ XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. │ │\n",
@@ -1159,142 +1085,154 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ No message history. │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ { │ │\n",
" │ │ \"fees\": [ │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Annual Membership\" │ │\n",
+ " │ │ \"index\": 1, │ │\n",
+ " │ │ \"name\": \"annual membership fee\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": 0.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"My Chase Plan Sm\" │ │\n",
+ " │ │ \"index\": 2, │ │\n",
+ " │ │ \"name\": \"my chase plan fee\", │ │\n",
+ " │ │ \"explanation\": \"Monthly fee of 0% of the amount of each eligible purchase transaction or amount │ │\n",
+ " │ │ selected to create a My Chase Plan while in the 0% Intro Purchase APR period. After that, monthly fee │ │\n",
+ " │ │ of 1.72% of the amount of each eligible purchase transaction or amount selected to create a My Chase │ │\n",
+ " │ │ Plan. The My Chase Plan Fee will be determined at the time each My Chase Plan is created and will │ │\n",
+ " │ │ remain the same until the My Chase Plan is paid in full.\", │ │\n",
+ " │ │ \"value\": 1.72 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Balance Transfers\" │ │\n",
+ " │ │ \"index\": 3, │ │\n",
+ " │ │ \"name\": \"balance transfers intro fee\", │ │\n",
+ " │ │ \"explanation\": \"Either $5 or 3% of the amount of each transfer, whichever is greater, on │ │\n",
+ " │ │ transfers made within 60 days of account opening. After that: Either $5 or 5% of the amount of each │ │\n",
+ " │ │ transfer, whichever is greater.\", │ │\n",
+ " │ │ \"value\": 5.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Cash Advances\" │ │\n",
+ " │ │ \"index\": 4, │ │\n",
+ " │ │ \"name\": \"cash advances\", │ │\n",
+ " │ │ \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\", │ │\n",
+ " │ │ \"value\": 5.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Foreign Transactions\" │ │\n",
+ " │ │ \"index\": 5, │ │\n",
+ " │ │ \"name\": \"foreign transactions\", │ │\n",
+ " │ │ \"explanation\": \"3% of the amount of each transaction in U.S. dollars.\", │ │\n",
+ " │ │ \"value\": 3.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Late Payment Penalty\" │ │\n",
+ " │ │ \"index\": 6, │ │\n",
+ " │ │ \"name\": \"late payment\", │ │\n",
+ " │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ " │ │ \"value\": 40.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Over-The-Credit-Limit Penalty\" │ │\n",
+ " │ │ \"index\": 7, │ │\n",
+ " │ │ \"name\": \"over-the-credit-limit\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": null │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Return Payment Penalty\" │ │\n",
+ " │ │ \"index\": 8, │ │\n",
+ " │ │ \"name\": \"return payment\", │ │\n",
+ " │ │ \"explanation\": \"Up to $40.\", │ │\n",
+ " │ │ \"value\": 40.0 │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"name\": \"Return Check Penalty\" │ │\n",
+ " │ │ \"index\": 9, │ │\n",
+ " │ │ \"name\": \"return check\", │ │\n",
+ " │ │ \"explanation\": \"None\", │ │\n",
+ " │ │ \"value\": null │ │\n",
" │ │ } │ │\n",
- " │ │ ] │ │\n",
+ " │ │ ], │ │\n",
+ " │ │ \"interest_rates\": null │ │\n",
" │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
- " │ │ { │ │\n",
- " │ │ 'fees': [ │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 1, │ │\n",
- " │ │ 'name': 'annual membership', │ │\n",
- " │ │ 'explanation': 'None', │ │\n",
- " │ │ 'value': 0.0 │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 2, │ │\n",
- " │ │ 'name': 'My Chase', │ │\n",
- " │ │ 'explanation': 'monthly fee of 0% of the amount of each eligible purchase transaction or │ │\n",
- " │ │ amount selected to create a my chase plan while in the 0% intro purchase apr period. after that, │ │\n",
+ " │ │ SkeletonReAsk( │ │\n",
+ " │ │ incorrect_value={ │ │\n",
+ " │ │ 'fees': [ │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 1, │ │\n",
+ " │ │ 'name': 'annual membership fee', │ │\n",
+ " │ │ 'explanation': 'None', │ │\n",
+ " │ │ 'value': 0.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 2, │ │\n",
+ " │ │ 'name': 'my chase plan fee', │ │\n",
+ " │ │ 'explanation': 'Monthly fee of 0% of the amount of each eligible purchase transaction │ │\n",
+ " │ │ or amount selected to create a My Chase Plan while in the 0% Intro Purchase APR period. After that, │ │\n",
" │ │ monthly fee of 1.72% of the amount of each eligible purchase transaction or amount selected to create a │ │\n",
- " │ │ my chase plan. the my chase plan fee will be determined at the time each my chase plan is created and │ │\n",
- " │ │ will remain the same until the my chase plan is paid in full.', │ │\n",
- " │ │ 'value': 1.72 │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 3, │ │\n",
- " │ │ 'name': 'balance transfers', │ │\n",
- " │ │ 'explanation': 'intro fee of either $5 or 3% of the amount of each transfer, whichever is │ │\n",
- " │ │ greater, on transfers made within 60 days of account opening. after that: either $5 or 5% of the amount │ │\n",
- " │ │ of each transfer, whichever is greater.', │ │\n",
- " │ │ 'value': 5.0 │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 4, │ │\n",
- " │ │ 'name': 'cash advances', │ │\n",
- " │ │ 'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is greater.', │ │\n",
- " │ │ 'value': 5.0 │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 5, │ │\n",
- " │ │ 'name': 'foreign transactions', │ │\n",
- " │ │ 'explanation': '3% of the amount of each transaction in U.S. dollars.', │ │\n",
- " │ │ 'value': 3.0 │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 6, │ │\n",
- " │ │ 'name': 'Late Payment', │ │\n",
- " │ │ 'explanation': 'Up to $40.', │ │\n",
- " │ │ 'value': None │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 7, │ │\n",
- " │ │ 'name': 'over-the-credit-limit penalty', │ │\n",
- " │ │ 'explanation': 'None', │ │\n",
- " │ │ 'value': None │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 8, │ │\n",
- " │ │ 'name': 'Return Payment', │ │\n",
- " │ │ 'explanation': 'Up to $40.', │ │\n",
- " │ │ 'value': None │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ 'index': 9, │ │\n",
- " │ │ 'name': 'Return Check', │ │\n",
- " │ │ 'explanation': 'None', │ │\n",
- " │ │ 'value': None │ │\n",
- " │ │ } │ │\n",
- " │ │ ], │ │\n",
- " │ │ 'interest_rates': { │ │\n",
- " │ │ 'purchase annual percentage rate (apr)': { │ │\n",
- " │ │ 'introductory rate': 0.0, │ │\n",
- " │ │ 'introductory period': 'first 18 months', │ │\n",
- " │ │ 'standard rate': 19.49, │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The interest rate charged on purchases made with the credit card.' │ │\n",
- " │ │ }, │ │\n",
- " │ │ 'my chase loan sm apr': { │ │\n",
- " │ │ 'rate': 19.49, │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The interest rate charged on My Chase Loan balances.' │ │\n",
- " │ │ }, │ │\n",
- " │ │ 'balance transfer apr': { │ │\n",
- " │ │ 'introductory rate': 0.0, │ │\n",
- " │ │ 'introductory period': 'first 18 months', │ │\n",
- " │ │ 'standard rate': 19.49, │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The interest rate charged on balances transferred to the credit card.' │ │\n",
- " │ │ }, │ │\n",
- " │ │ 'cash advance apr': { │ │\n",
- " │ │ 'rate': 29.49, │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The interest rate charged on cash advances taken with the credit card.' │ │\n",
- " │ │ }, │ │\n",
- " │ │ 'penalty apr and when it applies': { │ │\n",
- " │ │ 'rate': 'Up to 29.99%', │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The interest rate charged on the credit card when a penalty is applied.' │ │\n",
- " │ │ }, │ │\n",
- " │ │ 'maximum apr': { │ │\n",
- " │ │ 'rate': 29.99, │ │\n",
- " │ │ 'variable': True, │ │\n",
- " │ │ 'description': 'The highest interest rate that can be charged on the credit card.' │ │\n",
- " │ │ } │ │\n",
- " │ │ } │ │\n",
- " │ │ } │ │\n",
+ " │ │ My Chase Plan. The My Chase Plan Fee will be determined at the time each My Chase Plan is created and │ │\n",
+ " │ │ will remain the same until the My Chase Plan is paid in full.', │ │\n",
+ " │ │ 'value': 1.72 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 3, │ │\n",
+ " │ │ 'name': 'balance transfers intro fee', │ │\n",
+ " │ │ 'explanation': 'Either $5 or 3% of the amount of each transfer, whichever is greater, │ │\n",
+ " │ │ on transfers made within 60 days of account opening. After that: Either $5 or 5% of the amount of each │ │\n",
+ " │ │ transfer, whichever is greater.', │ │\n",
+ " │ │ 'value': 5.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 4, │ │\n",
+ " │ │ 'name': 'cash advances', │ │\n",
+ " │ │ 'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is │ │\n",
+ " │ │ greater.', │ │\n",
+ " │ │ 'value': 5.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 5, │ │\n",
+ " │ │ 'name': 'foreign transactions', │ │\n",
+ " │ │ 'explanation': '3% of the amount of each transaction in U.S. dollars.', │ │\n",
+ " │ │ 'value': 3.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 6, │ │\n",
+ " │ │ 'name': 'late payment', │ │\n",
+ " │ │ 'explanation': 'Up to $40.', │ │\n",
+ " │ │ 'value': 40.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 7, │ │\n",
+ " │ │ 'name': 'over-the-credit-limit', │ │\n",
+ " │ │ 'explanation': 'None', │ │\n",
+ " │ │ 'value': None │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 8, │ │\n",
+ " │ │ 'name': 'return payment', │ │\n",
+ " │ │ 'explanation': 'Up to $40.', │ │\n",
+ " │ │ 'value': 40.0 │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'index': 9, │ │\n",
+ " │ │ 'name': 'return check', │ │\n",
+ " │ │ 'explanation': 'None', │ │\n",
+ " │ │ 'value': None │ │\n",
+ " │ │ } │ │\n",
+ " │ │ ], │ │\n",
+ " │ │ 'interest_rates': None │ │\n",
+ " │ │ }, │ │\n",
+ " │ │ fail_results=[ │ │\n",
+ " │ │ FailResult( │ │\n",
+ " │ │ outcome='fail', │ │\n",
+ " │ │ metadata=None, │ │\n",
+ " │ │ error_message='JSON does not match schema', │ │\n",
+ " │ │ fix_value=None │ │\n",
+ " │ │ ) │ │\n",
+ " │ │ ] │ │\n",
+ " │ │ ) │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
" \n"
@@ -1303,7 +1241,6 @@
"Logs\n",
"├── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
"│ │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
- "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mGiven the following document, answer the following questions. If the answer doesn't exist in the \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mdocument, enter \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m`null`.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -1451,10 +1388,8 @@
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255msavings and checking accounts, loans, and credit products?\"/>\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
"│ │ \u001b[48;2;255;240;242m╭─\u001b[0m\u001b[48;2;255;240;242m────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m Instructions \u001b[0m\u001b[48;2;255;240;242m─────────────────────────────────────────────\u001b[0m\u001b[48;2;255;240;242m─╮\u001b[0m │\n",
- "│ │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
"│ │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242mYou are a helpful assistant only capable of communicating with valid JSON, and no other text.\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
"│ │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
"│ │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
@@ -1471,256 +1406,157 @@
"│ │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m- `\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m
\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -1807,19 +1660,11 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mrequests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255menter `null`.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mHere are examples of simple (XML, JSON) pairs that show the expected behavior:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `
` => `{{\"bar\": ['STRING ONE', 'STRING TWO', \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255metc.]}}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
- "\n",
- "\n",
+ " return PassResult()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now we can use the validator in a Rail spec:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rail_str = \"\"\"\n",
+ "\n",
"\n",
"\n",
"\n",
@@ -83,7 +98,7 @@
"\n",
"How do I use OpenAI's Completion API?\n",
"\n",
- "@complete_json_suffix\n",
+ "${gr.complete_json_suffix}\n",
"\n",
"\n",
"\n",
@@ -91,6 +106,33 @@
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or in a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pydantic import BaseModel, Field\n",
+ "\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "\n",
+ "How do I use OpenAI's Completion API?\n",
+ "\n",
+ "${gr.complete_json_suffix}\n",
+ "\"\"\"\n",
+ "\n",
+ "class ScrubbedCode(BaseModel):\n",
+ " api_help: str = Field(description=\"Show an example curl command for using openai Completion API\", validators=[NoCodeSecrets(on_fail=\"fix\")])"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -106,17 +148,47 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"import guardrails as gd\n",
"\n",
- "from rich import print\n",
- "\n",
+ "from rich import print"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From the XML RAIL spec:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or from the Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=ScrubbedCode, prompt=prompt)"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -127,7 +199,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -141,23 +213,22 @@
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string name = \"api_help\" description = \"Show an example curl command for using openai Completion API\" \n",
- "format = \"no-code-secrets: \" / > \n",
+ " <string name = \"api_help\" format = \"no-code-secrets\" description = \"Show an example curl command for using openai \n",
+ "Completion API\" / > \n",
"< / output > \n",
"\n",
"\n",
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -207,15 +277,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
}
],
@@ -238,7 +307,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -267,14 +336,14 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -285,8 +354,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"api_help\" description=\"Show an example curl command for using openai Completion API\" │ │\n",
- " │ │ format=\"no-code-secrets: \"/> │ │\n",
+ " │ │ <string name=\"api_help\" format=\"no-code-secrets\" description=\"Show an example curl command for │ │\n",
+ " │ │ using openai Completion API\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -303,9 +372,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ {\"api_help\": \"curl -X POST -H 'Content-Type: application/json' -d '{\\\"prompt\\\": \\\"The quick brown │ │\n",
" │ │ fox\\\", \\\"max_tokens\\\": 10}' https://api.openai.com/v1/engines/completion/completions\"} │ │\n",
@@ -321,7 +399,7 @@
],
"text/plain": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -332,8 +410,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -350,9 +428,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
- "\n",
- "\n",
- "\n",
+ " return PassResult()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next we can define our RAIL spec either as a XML string:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rail_str = \"\"\"\n",
+ "\n",
"\n",
"\n",
" \n",
@@ -110,13 +119,52 @@
"\n",
"\n",
"Generate a recipe for vegan mac and cheese.\n",
- "@complete_json_suffix\n",
+ "${gr.complete_json_suffix}\n",
" \n",
"\n",
"
\n",
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "or a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pydantic import BaseModel, Field\n",
+ "from typing import List\n",
+ "\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "Generate a recipe for vegan mac and cheese.\n",
+ "${gr.complete_json_suffix}\n",
+ "\"\"\"\n",
+ "\n",
+ "class Ingredient(BaseModel):\n",
+ " index: int = Field(validators=[(\"1-indexed\", \"noop\")])\n",
+ " name: str = Field(validators=[IsVegan(on_fail=\"fix\")])\n",
+ " brand: str = Field(description=\"Suggested brand for the ingredient (if any)\")\n",
+ " optional: bool = Field(description=\"Is the ingredient necessary?\")\n",
+ " quantity: float = Field(validators=[(\"units-imperial\", \"noop\")])\n",
+ " units: str = Field(validators=[(\"units-imperial\", \"noop\")])\n",
+ "\n",
+ "class Instruction(BaseModel):\n",
+ " index: int = Field(validators=[(\"1-indexed\", \"noop\")])\n",
+ " step: str\n",
+ "\n",
+ "class Recipe(BaseModel):\n",
+ " ingredients: List[Ingredient] = Field(description=\"What are the ingredients for the recipe?\")\n",
+ " instructions: List[Instruction] = Field(description=\"What are the instructions for the recipe?\")"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -140,20 +188,27 @@
"3. Compiles the schema and type info from the RAIL spec and adds it to the prompt."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From the XML string RAIL spec:"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/guardrails/guardrails/schema.py:187: UserWarning: Validator 1-indexed is not valid for element integer.\n",
+ "/home/zayd/workspace/guardrails-poc/.venv/lib/python3.11/site-packages/guardrails/schema.py:218: UserWarning: Validator 1-indexed is not valid for element integer.\n",
" warnings.warn(\n",
- "/Users/shreyarajpal/guardrails/guardrails/schema.py:187: UserWarning: Validator units-imperial is not valid for element float.\n",
+ "/home/zayd/workspace/guardrails-poc/.venv/lib/python3.11/site-packages/guardrails/schema.py:218: UserWarning: Validator units-imperial is not valid for element float.\n",
" warnings.warn(\n",
- "/Users/shreyarajpal/guardrails/guardrails/schema.py:187: UserWarning: Validator units-imperial is not valid for element string.\n",
+ "/home/zayd/workspace/guardrails-poc/.venv/lib/python3.11/site-packages/guardrails/schema.py:218: UserWarning: Validator units-imperial is not valid for element string.\n",
" warnings.warn(\n"
]
}
@@ -162,6 +217,22 @@
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From the Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=Recipe, prompt=prompt)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -173,7 +244,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
@@ -207,15 +278,14 @@
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \n"
],
@@ -248,15 +318,14 @@
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -277,15 +346,14 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
}
],
@@ -308,7 +376,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -316,7 +384,7 @@
"text/html": [
"{ \n",
" 'ingredients' : [ \n",
- " { 'index' : 1 , 'name' : 'macaroni' , 'brand' : 'Barilla' , 'optional' : False , 'quantity' : 8.0 , 'units' : 'oz' } ,\n",
+ " { 'index' : 1 , 'name' : 'macaroni' , 'brand' : None , 'optional' : False , 'quantity' : 8.0 , 'units' : 'oz' } ,\n",
" { \n",
" 'index' : 2 ,\n",
" 'name' : 'vegan butter' ,\n",
@@ -328,46 +396,32 @@
" { \n",
" 'index' : 3 ,\n",
" 'name' : 'all-purpose flour' ,\n",
- " 'brand' : 'Gold Medal' ,\n",
+ " 'brand' : None ,\n",
" 'optional' : False ,\n",
" 'quantity' : 2.0 ,\n",
" 'units' : 'tbsp' \n",
" } ,\n",
" { 'index' : 4 , 'name' : 'vegan milk' , 'brand' : 'Oatly' , 'optional' : False , 'quantity' : 2.0 , 'units' : 'cups' } ,\n",
- " { 'index' : 5 , 'name' : 'vegan cheese' , 'brand' : 'Daiya' , 'optional' : False , 'quantity' : 8.0 , 'units' : 'oz' } ,\n",
- " { \n",
- " 'index' : 6 ,\n",
- " 'name' : 'nutritional yeast' ,\n",
- " 'brand' : \"Bob's Red Mill\" ,\n",
- " 'optional' : False ,\n",
- " 'quantity' : 2.0 ,\n",
- " 'units' : 'tbsp' \n",
- " } ,\n",
" { \n",
- " 'index' : 7 ,\n",
- " 'name' : 'garlic powder' ,\n",
- " 'brand' : 'McCormick' ,\n",
+ " 'index' : 5 ,\n",
+ " 'name' : 'vegan cheese' ,\n",
+ " 'brand' : 'Daiya' ,\n",
" 'optional' : False ,\n",
" 'quantity' : 1.0 ,\n",
- " 'units' : 'tsp' \n",
+ " 'units' : 'cup' \n",
" } ,\n",
" { \n",
- " 'index' : 8 ,\n",
- " 'name' : 'onion powder' ,\n",
- " 'brand' : 'McCormick' ,\n",
+ " 'index' : 6 ,\n",
+ " 'name' : 'nutritional yeast' ,\n",
+ " 'brand' : None ,\n",
" 'optional' : False ,\n",
- " 'quantity' : 1.0 ,\n",
- " 'units' : 'tsp' \n",
+ " 'quantity' : 2.0 ,\n",
+ " 'units' : 'tbsp' \n",
" } ,\n",
- " { 'index' : 9 , 'name' : 'salt' , 'brand' : 'Morton' , 'optional' : False , 'quantity' : 1.0 , 'units' : 'tsp' } ,\n",
- " { \n",
- " 'index' : 10 ,\n",
- " 'name' : 'black pepper' ,\n",
- " 'brand' : 'McCormick' ,\n",
- " 'optional' : False ,\n",
- " 'quantity' : 0.5 ,\n",
- " 'units' : 'tsp' \n",
- " } \n",
+ " { 'index' : 7 , 'name' : 'garlic powder' , 'brand' : None , 'optional' : False , 'quantity' : 1.0 , 'units' : 'tsp' } ,\n",
+ " { 'index' : 8 , 'name' : 'onion powder' , 'brand' : None , 'optional' : False , 'quantity' : 1.0 , 'units' : 'tsp' } ,\n",
+ " { 'index' : 9 , 'name' : 'salt' , 'brand' : None , 'optional' : False , 'quantity' : 0.5 , 'units' : 'tsp' } ,\n",
+ " { 'index' : 10 , 'name' : 'black pepper' , 'brand' : None , 'optional' : False , 'quantity' : 0.5 , 'units' : 'tsp' } \n",
" ] ,\n",
" 'instructions' : [ \n",
" { \n",
@@ -375,23 +429,26 @@
" 'step' : 'Bring a large pot of salted water to a boil. Add the macaroni and cook according to package \n",
"instructions.' \n",
" } ,\n",
- " { 'index' : 2 , 'step' : 'Drain the macaroni and set aside.' } ,\n",
+ " { \n",
+ " 'index' : 2 ,\n",
+ " 'step' : 'Meanwhile, melt the vegan butter in a medium saucepan over medium heat. Add the flour and \n",
+ "whisk until combined. Cook for 1 minute, stirring constantly.' \n",
+ " } ,\n",
" { \n",
" 'index' : 3 ,\n",
- " 'step' : 'In a medium saucepan, melt the vegan butter over medium heat. Add the flour and whisk until \n",
- "combined and bubbly.' \n",
+ " 'step' : 'Slowly add the vegan milk, whisking constantly until the mixture is smooth. Cook for 3-4 \n",
+ "minutes, stirring constantly, until the mixture is thickened.' \n",
" } ,\n",
" { \n",
" 'index' : 4 ,\n",
- " 'step' : 'Slowly add the vegan milk, whisking constantly until the mixture is smooth and thickened.' \n",
+ " 'step' : 'Add the vegan cheese, nutritional yeast, garlic powder, onion powder, salt, and pepper. Stir \n",
+ "until the cheese is melted and the sauce is smooth.' \n",
" } ,\n",
" { \n",
" 'index' : 5 ,\n",
- " 'step' : 'Add the vegan cheese, nutritional yeast, garlic powder, onion powder, salt, and pepper. Stir \n",
- "until the cheese is melted and the sauce is smooth.' \n",
+ " 'step' : 'Drain the macaroni and add it to the sauce. Stir until the macaroni is evenly coated.' \n",
" } ,\n",
- " { 'index' : 6 , 'step' : 'Add the cooked macaroni to the sauce and stir until combined.' } ,\n",
- " { 'index' : 7 , 'step' : 'Serve the vegan mac and cheese warm.' } \n",
+ " { 'index' : 6 , 'step' : 'Serve the vegan mac and cheese warm.' } \n",
" ] \n",
"} \n",
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
" │ │ Generate a recipe for vegan mac and cheese. │ │\n",
@@ -537,16 +583,25 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ { │ │\n",
" │ │ \"ingredients\": [ │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 1, │ │\n",
" │ │ \"name\": \"macaroni\", │ │\n",
- " │ │ \"brand\": \"Barilla\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 8.0, │ │\n",
" │ │ \"units\": \"oz\" │ │\n",
@@ -562,7 +617,7 @@
" │ │ { │ │\n",
" │ │ \"index\": 3, │ │\n",
" │ │ \"name\": \"all-purpose flour\", │ │\n",
- " │ │ \"brand\": \"Gold Medal\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 2.0, │ │\n",
" │ │ \"units\": \"tbsp\" │ │\n",
@@ -580,13 +635,13 @@
" │ │ \"name\": \"vegan cheese\", │ │\n",
" │ │ \"brand\": \"Daiya\", │ │\n",
" │ │ \"optional\": false, │ │\n",
- " │ │ \"quantity\": 8.0, │ │\n",
- " │ │ \"units\": \"oz\" │ │\n",
+ " │ │ \"quantity\": 1.0, │ │\n",
+ " │ │ \"units\": \"cup\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 6, │ │\n",
" │ │ \"name\": \"nutritional yeast\", │ │\n",
- " │ │ \"brand\": \"Bob's Red Mill\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 2.0, │ │\n",
" │ │ \"units\": \"tbsp\" │ │\n",
@@ -594,7 +649,7 @@
" │ │ { │ │\n",
" │ │ \"index\": 7, │ │\n",
" │ │ \"name\": \"garlic powder\", │ │\n",
- " │ │ \"brand\": \"McCormick\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 1.0, │ │\n",
" │ │ \"units\": \"tsp\" │ │\n",
@@ -602,7 +657,7 @@
" │ │ { │ │\n",
" │ │ \"index\": 8, │ │\n",
" │ │ \"name\": \"onion powder\", │ │\n",
- " │ │ \"brand\": \"McCormick\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 1.0, │ │\n",
" │ │ \"units\": \"tsp\" │ │\n",
@@ -610,15 +665,15 @@
" │ │ { │ │\n",
" │ │ \"index\": 9, │ │\n",
" │ │ \"name\": \"salt\", │ │\n",
- " │ │ \"brand\": \"Morton\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
- " │ │ \"quantity\": 1.0, │ │\n",
+ " │ │ \"quantity\": 0.5, │ │\n",
" │ │ \"units\": \"tsp\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 10, │ │\n",
" │ │ \"name\": \"black pepper\", │ │\n",
- " │ │ \"brand\": \"McCormick\", │ │\n",
+ " │ │ \"brand\": null, │ │\n",
" │ │ \"optional\": false, │ │\n",
" │ │ \"quantity\": 0.5, │ │\n",
" │ │ \"units\": \"tsp\" │ │\n",
@@ -632,29 +687,26 @@
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 2, │ │\n",
- " │ │ \"step\": \"Drain the macaroni and set aside.\" │ │\n",
+ " │ │ \"step\": \"Meanwhile, melt the vegan butter in a medium saucepan over medium heat. Add the │ │\n",
+ " │ │ flour and whisk until combined. Cook for 1 minute, stirring constantly.\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 3, │ │\n",
- " │ │ \"step\": \"In a medium saucepan, melt the vegan butter over medium heat. Add the flour and │ │\n",
- " │ │ whisk until combined and bubbly.\" │ │\n",
+ " │ │ \"step\": \"Slowly add the vegan milk, whisking constantly until the mixture is smooth. Cook │ │\n",
+ " │ │ for 3-4 minutes, stirring constantly, until the mixture is thickened.\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"index\": 4, │ │\n",
- " │ │ \"step\": \"Slowly add the vegan milk, whisking constantly until the mixture is smooth and │ │\n",
- " │ │ thickened.\" │ │\n",
- " │ │ }, │ │\n",
- " │ │ { │ │\n",
- " │ │ \"index\": 5, │ │\n",
" │ │ \"step\": \"Add the vegan cheese, nutritional yeast, garlic powder, onion powder, salt, and │ │\n",
" │ │ pepper. Stir until the cheese is melted and the sauce is smooth.\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"index\": 6, │ │\n",
- " │ │ \"step\": \"Add the cooked macaroni to the sauce and stir until combined.\" │ │\n",
+ " │ │ \"index\": 5, │ │\n",
+ " │ │ \"step\": \"Drain the macaroni and add it to the sauce. Stir until the macaroni is evenly │ │\n",
+ " │ │ coated.\" │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ \"index\": 7, │ │\n",
+ " │ │ \"index\": 6, │ │\n",
" │ │ \"step\": \"Serve the vegan mac and cheese warm.\" │ │\n",
" │ │ } │ │\n",
" │ │ ] │ │\n",
@@ -666,7 +718,7 @@
" │ │ { │ │\n",
" │ │ 'index': 1, │ │\n",
" │ │ 'name': 'macaroni', │ │\n",
- " │ │ 'brand': 'Barilla', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 8.0, │ │\n",
" │ │ 'units': 'oz' │ │\n",
@@ -682,7 +734,7 @@
" │ │ { │ │\n",
" │ │ 'index': 3, │ │\n",
" │ │ 'name': 'all-purpose flour', │ │\n",
- " │ │ 'brand': 'Gold Medal', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 2.0, │ │\n",
" │ │ 'units': 'tbsp' │ │\n",
@@ -700,13 +752,13 @@
" │ │ 'name': 'vegan cheese', │ │\n",
" │ │ 'brand': 'Daiya', │ │\n",
" │ │ 'optional': False, │ │\n",
- " │ │ 'quantity': 8.0, │ │\n",
- " │ │ 'units': 'oz' │ │\n",
+ " │ │ 'quantity': 1.0, │ │\n",
+ " │ │ 'units': 'cup' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'index': 6, │ │\n",
" │ │ 'name': 'nutritional yeast', │ │\n",
- " │ │ 'brand': \"Bob's Red Mill\", │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 2.0, │ │\n",
" │ │ 'units': 'tbsp' │ │\n",
@@ -714,7 +766,7 @@
" │ │ { │ │\n",
" │ │ 'index': 7, │ │\n",
" │ │ 'name': 'garlic powder', │ │\n",
- " │ │ 'brand': 'McCormick', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 1.0, │ │\n",
" │ │ 'units': 'tsp' │ │\n",
@@ -722,7 +774,7 @@
" │ │ { │ │\n",
" │ │ 'index': 8, │ │\n",
" │ │ 'name': 'onion powder', │ │\n",
- " │ │ 'brand': 'McCormick', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 1.0, │ │\n",
" │ │ 'units': 'tsp' │ │\n",
@@ -730,15 +782,15 @@
" │ │ { │ │\n",
" │ │ 'index': 9, │ │\n",
" │ │ 'name': 'salt', │ │\n",
- " │ │ 'brand': 'Morton', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
- " │ │ 'quantity': 1.0, │ │\n",
+ " │ │ 'quantity': 0.5, │ │\n",
" │ │ 'units': 'tsp' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ 'index': 10, │ │\n",
" │ │ 'name': 'black pepper', │ │\n",
- " │ │ 'brand': 'McCormick', │ │\n",
+ " │ │ 'brand': None, │ │\n",
" │ │ 'optional': False, │ │\n",
" │ │ 'quantity': 0.5, │ │\n",
" │ │ 'units': 'tsp' │ │\n",
@@ -750,27 +802,27 @@
" │ │ 'step': 'Bring a large pot of salted water to a boil. Add the macaroni and cook according │ │\n",
" │ │ to package instructions.' │ │\n",
" │ │ }, │ │\n",
- " │ │ {'index': 2, 'step': 'Drain the macaroni and set aside.'}, │ │\n",
" │ │ { │ │\n",
- " │ │ 'index': 3, │ │\n",
- " │ │ 'step': 'In a medium saucepan, melt the vegan butter over medium heat. Add the flour and │ │\n",
- " │ │ whisk until combined and bubbly.' │ │\n",
+ " │ │ 'index': 2, │ │\n",
+ " │ │ 'step': 'Meanwhile, melt the vegan butter in a medium saucepan over medium heat. Add the │ │\n",
+ " │ │ flour and whisk until combined. Cook for 1 minute, stirring constantly.' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ 'index': 4, │ │\n",
- " │ │ 'step': 'Slowly add the vegan milk, whisking constantly until the mixture is smooth and │ │\n",
- " │ │ thickened.' │ │\n",
+ " │ │ 'index': 3, │ │\n",
+ " │ │ 'step': 'Slowly add the vegan milk, whisking constantly until the mixture is smooth. Cook │ │\n",
+ " │ │ for 3-4 minutes, stirring constantly, until the mixture is thickened.' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ 'index': 5, │ │\n",
+ " │ │ 'index': 4, │ │\n",
" │ │ 'step': 'Add the vegan cheese, nutritional yeast, garlic powder, onion powder, salt, and │ │\n",
" │ │ pepper. Stir until the cheese is melted and the sauce is smooth.' │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
- " │ │ 'index': 6, │ │\n",
- " │ │ 'step': 'Add the cooked macaroni to the sauce and stir until combined.' │ │\n",
+ " │ │ 'index': 5, │ │\n",
+ " │ │ 'step': 'Drain the macaroni and add it to the sauce. Stir until the macaroni is evenly │ │\n",
+ " │ │ coated.' │ │\n",
" │ │ }, │ │\n",
- " │ │ {'index': 7, 'step': 'Serve the vegan mac and cheese warm.'} │ │\n",
+ " │ │ {'index': 6, 'step': 'Serve the vegan mac and cheese warm.'} │ │\n",
" │ │ ] │ │\n",
" │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
@@ -779,7 +831,7 @@
],
"text/plain": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mGenerate a recipe for vegan mac and cheese.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -820,16 +872,25 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
"\n",
"\n",
- " \n",
+ " \n",
" \n",
" \n",
" \n",
- " \n",
- " \n",
+ " \n",
" \n",
" \n",
@@ -53,14 +62,52 @@
"\n",
"You are a human in an enchanted forest. You come across opponents of different types, and you should fight smaller opponents and run away from bigger ones.\n",
"\n",
- "You run into a {{opp_type}}. What do you do?\n",
+ "You run into a ${opp_type}. What do you do?\n",
"\n",
- "@complete_json_suffix_v2 \n",
+ "${gr.complete_json_suffix_v2}\n",
"\n",
" \n",
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Pydantic model option:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from guardrails.validators import ValidChoices\n",
+ "from pydantic import BaseModel, Field\n",
+ "from typing import Literal, Union\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "You are a human in an enchanted forest. You come across opponents of different types, and you should fight smaller opponents and run away from bigger ones.\n",
+ "\n",
+ "You run into a ${opp_type}. What do you do?\n",
+ "\n",
+ "${gr.complete_json_suffix_v2}\"\"\"\n",
+ "\n",
+ "class Fight(BaseModel):\n",
+ " chosen_action: Literal['fight']\n",
+ " weapon: str = Field(validators=[ValidChoices(['crossbow', 'machine gun'], on_fail=\"reask\")])\n",
+ "\n",
+ "class Flight(BaseModel):\n",
+ " chosen_action: Literal['flight']\n",
+ " flight_direction: str = Field(validators=[ValidChoices(['north','south','east','west'], on_fail=\"exception\")])\n",
+ " distance: int = Field(validators=[ValidChoices([1,2,3,4], on_fail=\"exception\")])\n",
+ "\n",
+ "class FightOrFlight(BaseModel):\n",
+ " action: Union[Fight, Flight] = Field(discriminator='chosen_action')\n",
+ " "
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -75,9 +122,21 @@
]
},
{
- "cell_type": "code",
- "execution_count": 3,
+ "cell_type": "markdown",
"metadata": {},
+ "source": [
+ "From XML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-08-23T15:09:28.590929Z",
+ "start_time": "2023-08-23T15:09:27.966099Z"
+ }
+ },
"outputs": [],
"source": [
"import guardrails as gd\n",
@@ -91,13 +150,38 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "The `Guard` object compiles the output schema and adds it to the prompt. We can see the final prompt below:"
+ "Or from Pydantic:"
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 21,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import guardrails as gd\n",
+ "\n",
+ "from rich import print\n",
+ "\n",
+ "guard = gd.Guard.from_pydantic(output_class=FightOrFlight, prompt=prompt)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
"metadata": {},
+ "source": [
+ "The `Guard` object compiles the output schema and adds it to the prompt. We can see the final prompt below:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 22,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-08-23T15:09:32.364711Z",
+ "start_time": "2023-08-23T15:09:32.356959Z"
+ }
+ },
"outputs": [
{
"data": {
@@ -106,18 +190,21 @@
"You are a human in an enchanted forest. You come across opponents of different types, and you should fight smaller \n",
"opponents and run away from bigger ones.\n",
"\n",
- "You run into a { opp_type} . What do you do?\n",
+ "You run into a ${ opp_type} . What do you do?\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string name = \"action\" choices = \"fight,flight\" / > \n",
- " <string format = \"valid-choices: choices=['crossbow', 'machine gun']\" name = \"fight\" if = \"action==fight\" / > \n",
- " <object name = \"flight\" if = \"action==flight\" > \n",
- " <string name = \"flight_direction\" format = \"valid-choices: choices=['north', 'south', 'east', 'west']\" / > \n",
- " <integer name = \"distance\" format = \"valid-choices: choices=[1, 2, 3, 4]\" / > \n",
- " < / object > \n",
+ " <choice name = \"action\" discriminator = \"chosen_action\" > \n",
+ " <case name = \"fight\" > \n",
+ " <string name = \"weapon\" format = \"valid-choices: choices=['crossbow', 'machine gun']\" / > \n",
+ " < / case > \n",
+ " <case name = \"flight\" > \n",
+ " <string name = \"flight_direction\" format = \"valid-choices: choices=['north', 'south', 'east', 'west']\" / > \n",
+ " <integer name = \"distance\" format = \"valid-choices: choices=[1, 2, 3, 4]\" / > \n",
+ " < / case > \n",
+ " < / choice > \n",
"< / output > \n",
"\n",
"\n",
@@ -127,15 +214,11 @@
"specific types. Be correct and concise. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` => ` {{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} ` \n",
- "- `<string name = \"foo\" choices = \"bar,baz\" / ><case name = \"bar\" on = \"foo\" ><string format = \"valid-choices: ['a', \n",
- "'b']\" / >< / case ><case name = \"baz\" on = \"foo\" ><object><string name = \"qux\" format = \"valid-choices: ['corge', \n",
- "'grault']\" / ><integer name = \"quux\" format = \"valid-choices: [1, 2, 3, 4]\" / >< / object >< / case >` = > `{{ 'foo' : 'baz' , 'baz' :\n",
- "{{ 'qux' : 'corge' , 'quux' : 1 }}}} `\n",
- "JSON Object:\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
+ "\n",
" \n"
],
"text/plain": [
@@ -143,18 +226,21 @@
"You are a human in an enchanted forest. You come across opponents of different types, and you should fight smaller \n",
"opponents and run away from bigger ones.\n",
"\n",
- "You run into a \u001b[1m{\u001b[0mopp_type\u001b[1m}\u001b[0m. What do you do?\n",
+ "You run into a $\u001b[1m{\u001b[0mopp_type\u001b[1m}\u001b[0m. What do you do?\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>\u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mcase\u001b[0m\u001b[39m>\u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
+ "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mcase\u001b[0m\u001b[39m>\u001b[0m\n",
+ "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mchoice\u001b[0m\u001b[39m>\u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
@@ -164,15 +250,11 @@
"\u001b[39mspecific types. Be correct and concise.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'baz'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'Some String'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'index'\u001b[0m\u001b[39m: \u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mcase\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mcase\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'baz'\u001b[0m, \u001b[32m'baz'\u001b[0m:\n",
- "\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'qux'\u001b[0m: \u001b[32m'corge'\u001b[0m, \u001b[32m'quux'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
- "JSON Object:\n"
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\n"
]
},
"metadata": {},
@@ -191,34 +273,45 @@
]
},
{
- "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
- "We can now wrap the LsLM API call with the `Guard` object. This will ensure that the LLM generates an output that is compliant with the RAIL spec.\n",
+ "We can now wrap the LLM API call with the `Guard` object. This will ensure that the LLM generates an output that is compliant with the RAIL spec.\n",
"\n",
"To start, we test with a 'giant' as an opponent, and look at the output."
]
},
{
"cell_type": "code",
- "execution_count": 24,
- "metadata": {},
- "outputs": [],
+ "execution_count": 23,
+ "metadata": {
+ "ExecuteTime": {
+ "end_time": "2023-08-23T15:10:08.998121Z",
+ "start_time": "2023-08-23T15:10:08.792027Z"
+ }
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
+ ]
+ }
+ ],
"source": [
"import openai\n",
"\n",
"raw_llm_response, validated_response = guard(\n",
- " openai.Completion.create,\n",
+ " openai.ChatCompletion.create,\n",
" prompt_params={'opp_type': 'giant'},\n",
- " engine=\"text-davinci-003\",\n",
+ " model=\"gpt-3.5-turbo\",\n",
" max_tokens=256,\n",
" temperature=0.0,\n",
")"
]
},
{
- "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
@@ -231,17 +324,17 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
- "{ 'action' : 'flight' , 'flight' : { 'flight_direction' : 'north' , 'distance' : 1 }} \n",
+ "{ 'action' : { 'chosen_action' : 'flight' , 'flight_direction' : 'north' , 'distance' : 1 }} \n",
"│ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"action\" choices=\"fight,flight\"/> │ │\n",
- " │ │ <string format=\"valid-choices: choices=['crossbow', 'machine gun']\" name=\"fight\" │ │\n",
- " │ │ if=\"action==fight\"/> │ │\n",
- " │ │ <object name=\"flight\" if=\"action==flight\"> │ │\n",
- " │ │ <string name=\"flight_direction\" format=\"valid-choices: choices=['north', 'south', 'east', │ │\n",
+ " │ │ <choice name=\"action\" discriminator=\"chosen_action\"> │ │\n",
+ " │ │ <case name=\"fight\"> │ │\n",
+ " │ │ <string name=\"weapon\" format=\"valid-choices: choices=['crossbow', 'machine gun']\"/> │ │\n",
+ " │ │ </case> │ │\n",
+ " │ │ <case name=\"flight\"> │ │\n",
+ " │ │ <string name=\"flight_direction\" format=\"valid-choices: choices=['north', 'south', 'east', │ │\n",
" │ │ 'west']\"/> │ │\n",
- " │ │ <integer name=\"distance\" format=\"valid-choices: choices=[1, 2, 3, 4]\"/> │ │\n",
- " │ │ </object> │ │\n",
+ " │ │ <integer name=\"distance\" format=\"valid-choices: choices=[1, 2, 3, 4]\"/> │ │\n",
+ " │ │ </case> │ │\n",
+ " │ │ </choice> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -304,18 +398,35 @@
" │ │ etc.]}` │ │\n",
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
- " │ │ - `<string name=\"foo\" choices=\"bar,baz\"/><case name=\"bar\" on=\"foo\"><string format=\"valid-choices: ['a', │ │\n",
- " │ │ 'b']\"/></case><case name=\"baz\" on=\"foo\"><object><string name=\"qux\" format=\"valid-choices: ['corge', │ │\n",
- " │ │ 'grault']\"/><integer name=\"quux\" format=\"valid-choices: [1, 2, 3, 4]\"/></object></case>` => `{'foo': │ │\n",
- " │ │ 'baz', 'baz': {'qux': 'corge', 'quux': 1}}` │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮ │\n",
+ " │ │ You are a helpful assistant, able to express yourself purely through JSON, strictly and precisely │ │\n",
+ " │ │ adhering to the provided XML schemas. │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
- " │ │ │ │\n",
- " │ │ {\"action\": \"flight\", \"flight\": {\"flight_direction\": \"north\", \"distance\": 1}} │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"action\": { │ │\n",
+ " │ │ \"chosen_action\": \"flight\", │ │\n",
+ " │ │ \"flight_direction\": \"north\", │ │\n",
+ " │ │ \"distance\": 1 │ │\n",
+ " │ │ } │ │\n",
+ " │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
- " │ │ {'action': 'flight', 'flight': {'flight_direction': 'north', 'distance': 1}} │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'action': { │ │\n",
+ " │ │ 'chosen_action': 'flight', │ │\n",
+ " │ │ 'flight_direction': 'north', │ │\n",
+ " │ │ 'distance': 1 │ │\n",
+ " │ │ } │ │\n",
+ " │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m
\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -357,18 +470,35 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255metc.]}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `{ 'action' : 'fight' , 'fight' : 'crossbow' } \n",
+ "{ 'action' : { 'chosen_action' : 'fight' , 'weapon' : 'crossbow' }} \n",
"│ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"action\" choices=\"fight,flight\"/> │ │\n",
- " │ │ <string format=\"valid-choices: choices=['crossbow', 'machine gun']\" name=\"fight\" │ │\n",
- " │ │ if=\"action==fight\"/> │ │\n",
- " │ │ <object name=\"flight\" if=\"action==flight\"> │ │\n",
- " │ │ <string name=\"flight_direction\" format=\"valid-choices: choices=['north', 'south', 'east', │ │\n",
+ " │ │ <choice name=\"action\" discriminator=\"chosen_action\"> │ │\n",
+ " │ │ <case name=\"fight\"> │ │\n",
+ " │ │ <string name=\"weapon\" format=\"valid-choices: choices=['crossbow', 'machine gun']\"/> │ │\n",
+ " │ │ </case> │ │\n",
+ " │ │ <case name=\"flight\"> │ │\n",
+ " │ │ <string name=\"flight_direction\" format=\"valid-choices: choices=['north', 'south', 'east', │ │\n",
" │ │ 'west']\"/> │ │\n",
- " │ │ <integer name=\"distance\" format=\"valid-choices: choices=[1, 2, 3, 4]\"/> │ │\n",
- " │ │ </object> │ │\n",
+ " │ │ <integer name=\"distance\" format=\"valid-choices: choices=[1, 2, 3, 4]\"/> │ │\n",
+ " │ │ </case> │ │\n",
+ " │ │ </choice> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -480,18 +619,28 @@
" │ │ etc.]}` │ │\n",
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
- " │ │ - `<string name=\"foo\" choices=\"bar,baz\"/><case name=\"bar\" on=\"foo\"><string format=\"valid-choices: ['a', │ │\n",
- " │ │ 'b']\"/></case><case name=\"baz\" on=\"foo\"><object><string name=\"qux\" format=\"valid-choices: ['corge', │ │\n",
- " │ │ 'grault']\"/><integer name=\"quux\" format=\"valid-choices: [1, 2, 3, 4]\"/></object></case>` => `{'foo': │ │\n",
- " │ │ 'baz', 'baz': {'qux': 'corge', 'quux': 1}}` │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮ │\n",
+ " │ │ You are a helpful assistant, able to express yourself purely through JSON, strictly and precisely │ │\n",
+ " │ │ adhering to the provided XML schemas. │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
- " │ │ │ │\n",
- " │ │ {\"action\": \"fight\", \"fight\": \"crossbow\"} │ │\n",
+ " │ │ { │ │\n",
+ " │ │ \"action\": { │ │\n",
+ " │ │ \"chosen_action\": \"fight\", │ │\n",
+ " │ │ \"weapon\": \"crossbow\" │ │\n",
+ " │ │ } │ │\n",
+ " │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
- " │ │ {'action': 'fight', 'fight': 'crossbow'} │ │\n",
+ " │ │ {'action': {'chosen_action': 'fight', 'weapon': 'crossbow'}} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -533,18 +684,28 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255metc.]}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `{ nl_instruction} \n",
+ "${ nl_instruction} \n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string name = \"generated_sql\" description = \"Generate SQL for the given natural language instruction.\" / > \n",
+ " <string name = \"generated_sql\" format = \"bug-free-sql\" description = \"Generate SQL for the given natural language \n",
+ "instruction.\" / > \n",
"< / output > \n",
"\n",
"\n",
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
"
\n"
],
@@ -172,28 +237,28 @@
"\n",
"Generate a valid SQL query for the following natural language instruction:\n",
"\n",
- "\u001b[1m{\u001b[0mnl_instruction\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mnl_instruction\u001b[1m}\u001b[0m\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -222,15 +287,14 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 34,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
}
],
@@ -259,7 +323,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 35,
"metadata": {},
"outputs": [
{
@@ -282,14 +346,14 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 36,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -302,8 +366,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"generated_sql\" description=\"Generate SQL for the given natural language │ │\n",
- " │ │ instruction.\"/> │ │\n",
+ " │ │ <string name=\"generated_sql\" format=\"bug-free-sql\" description=\"Generate SQL for the given natural │ │\n",
+ " │ │ language instruction.\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -320,9 +384,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ {\"generated_sql\": \"SELECT name FROM employee ORDER BY salary DESC LIMIT 1\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
@@ -334,7 +407,7 @@
],
"text/plain": [
"Logs\n",
- "└── ╭────────────────────────────────────────────────── Step 1 ───────────────────────────────────────────────────╮\n",
+ "└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -347,8 +420,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -365,9 +438,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
"\n",
- "\n",
- "\n",
"\n",
" \n",
@@ -77,9 +73,9 @@
"\n",
"Summarize the following document:\n",
"\n",
- "{{document}}\n",
+ "${document}\n",
"\n",
- "@complete_json_suffix\n",
+ "${gr.complete_json_suffix}\n",
" \n",
" \n",
"\"\"\""
@@ -139,7 +135,7 @@
"\n",
"Summarize the following document:\n",
"\n",
- "{ document} \n",
+ "${ document} \n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
@@ -152,15 +148,14 @@
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \n"
],
@@ -168,7 +163,7 @@
"\n",
"Summarize the following document:\n",
"\n",
- "\u001b[1m{\u001b[0mdocument\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mdocument\u001b[1m}\u001b[0m\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
@@ -181,15 +176,14 @@
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -231,28 +225,27 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
},
{
"data": {
"text/html": [
- "Validated Output: { 'summary' : 'All legislative powers are vested in a Congress of the United States, which consists \n",
- "of a Senate and House of Representatives. The House of Representatives is composed of members chosen every two \n",
- "years by the people of the several states, and must meet certain qualifications. Representatives and direct taxes \n",
- "are apportioned among the states according to their respective numbers. The House of Representatives chooses their \n",
- "speaker and other officers, and has the sole power of impeachment. When vacancies happen in the representation from \n",
- "any state, the executive authority thereof shall issue writs of election to fill such vacancies.' } \n",
+ "Validated Output: { 'summary' : 'The US Congress consists of a Senate and House of Representatives, with the House of \n",
+ "Representatives being chosen every two years by the people of the several states. Representatives must be at least \n",
+ "25 years old and have been a citizen of the US for seven years. Representation and taxes are apportioned among the \n",
+ "states according to their population, and the number of representatives cannot exceed one for every 30,000 people. \n",
+ "Vacancies are filled by the executive authority of the state. The House of Representatives chooses its speaker and \n",
+ "other officers, and has the sole power of impeachment.' } \n",
" \n"
],
"text/plain": [
- "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'summary'\u001b[0m: \u001b[32m'All legislative powers are vested in a Congress of the United States, which consists\u001b[0m\n",
- "\u001b[32mof a Senate and House of Representatives. The House of Representatives is composed of members chosen every two \u001b[0m\n",
- "\u001b[32myears by the people of the several states, and must meet certain qualifications. Representatives and direct taxes \u001b[0m\n",
- "\u001b[32mare apportioned among the states according to their respective numbers. The House of Representatives chooses their \u001b[0m\n",
- "\u001b[32mspeaker and other officers, and has the sole power of impeachment. When vacancies happen in the representation from\u001b[0m\n",
- "\u001b[32many state, the executive authority thereof shall issue writs of election to fill such vacancies.'\u001b[0m\u001b[1m}\u001b[0m\n"
+ "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'summary'\u001b[0m: \u001b[32m'The US Congress consists of a Senate and House of Representatives, with the House of\u001b[0m\n",
+ "\u001b[32mRepresentatives being chosen every two years by the people of the several states. Representatives must be at least \u001b[0m\n",
+ "\u001b[32m25 years old and have been a citizen of the US for seven years. Representation and taxes are apportioned among the \u001b[0m\n",
+ "\u001b[32mstates according to their population, and the number of representatives cannot exceed one for every 30,000 people. \u001b[0m\n",
+ "\u001b[32mVacancies are filled by the executive authority of the state. The House of Representatives chooses its speaker and \u001b[0m\n",
+ "\u001b[32mother officers, and has the sole power of impeachment.'\u001b[0m\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
@@ -283,7 +276,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -348,27 +341,36 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
- " │ │ {\"summary\": \"All legislative powers are vested in a Congress of the United States, which consists of a │ │\n",
- " │ │ Senate and House of Representatives. The House of Representatives is composed of members chosen every │ │\n",
- " │ │ two years by the people of the several states, and must meet certain qualifications. Representatives │ │\n",
- " │ │ and direct taxes are apportioned among the states according to their respective numbers. The House of │ │\n",
- " │ │ Representatives chooses their speaker and other officers, and has the sole power of impeachment. When │ │\n",
- " │ │ vacancies happen in the representation from any state, the executive authority thereof shall issue │ │\n",
- " │ │ writs of election to fill such vacancies.\"} │ │\n",
+ " │ │ {\"summary\": \"The US Congress consists of a Senate and House of Representatives, with the House of │ │\n",
+ " │ │ Representatives being chosen every two years by the people of the several states. Representatives must │ │\n",
+ " │ │ be at least 25 years old and have been a citizen of the US for seven years. Representation and taxes │ │\n",
+ " │ │ are apportioned among the states according to their population, and the number of representatives │ │\n",
+ " │ │ cannot exceed one for every 30,000 people. Vacancies are filled by the executive authority of the │ │\n",
+ " │ │ state. The House of Representatives chooses its speaker and other officers, and has the sole power of │ │\n",
+ " │ │ impeachment.\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
" │ │ { │ │\n",
- " │ │ 'summary': 'All legislative powers are vested in a Congress of the United States, which consists of │ │\n",
- " │ │ a Senate and House of Representatives. The House of Representatives is composed of members chosen every │ │\n",
- " │ │ two years by the people of the several states, and must meet certain qualifications. Representatives │ │\n",
- " │ │ and direct taxes are apportioned among the states according to their respective numbers. The House of │ │\n",
- " │ │ Representatives chooses their speaker and other officers, and has the sole power of impeachment. When │ │\n",
- " │ │ vacancies happen in the representation from any state, the executive authority thereof shall issue │ │\n",
- " │ │ writs of election to fill such vacancies.' │ │\n",
+ " │ │ 'summary': 'The US Congress consists of a Senate and House of Representatives, with the House of │ │\n",
+ " │ │ Representatives being chosen every two years by the people of the several states. Representatives must │ │\n",
+ " │ │ be at least 25 years old and have been a citizen of the US for seven years. Representation and taxes │ │\n",
+ " │ │ are apportioned among the states according to their population, and the number of representatives │ │\n",
+ " │ │ cannot exceed one for every 30,000 people. Vacancies are filled by the executive authority of the │ │\n",
+ " │ │ state. The House of Representatives chooses its speaker and other officers, and has the sole power of │ │\n",
+ " │ │ impeachment.' │ │\n",
" │ │ } │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
@@ -434,27 +436,36 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `│ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
+ " │ │ <string name=\"summary\" description=\"Summarize the given document faithfully.\"/> │ │\n",
+ " │ │ </string> │ │\n",
" │ │ │ │\n",
- " │ │ The string \"None`\" is removed from the list \"baz\" in the example above. │ │\n",
- " │ │ │ │\n",
- " │ │ The object \"baz\" is created with the value \"Some String\" which is set to \"{'foo': 'Some String', │ │\n",
- " │ │ 'index': 1}\". │ │\n",
+ " │ │ The House of Representatives shall chuse their Speaker and other Officers; and shall have the sole │ │\n",
+ " │ │ Power of Impeachment. │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
" │ │ None │ │\n",
@@ -660,15 +680,24 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- ` \n",
- "\n",
- "\n",
+ " return PassResult()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Next we define our RAIL spec either as XML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "rail_str = \"\"\"\n",
+ "\n",
"\n",
"\n",
" \n",
- "\n",
"Translate the given statement into english language:\n",
"\n",
- "{{statement_to_be_translated}}\n",
+ "${statement_to_be_translated}\n",
"\n",
- "@complete_json_suffix\n",
+ "${gr.complete_json_suffix}\n",
"\n",
"\n",
- "\n",
" \n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or as a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pydantic import BaseModel, Field\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "Translate the given statement into english language:\n",
"\n",
+ "${statement_to_be_translated}\n",
"\n",
- "\"\"\""
+ "${gr.complete_json_suffix}\n",
+ "\"\"\"\n",
+ "\n",
+ "class Translation(BaseModel):\n",
+ " translated_statement: str = Field(\n",
+ " description=\"Translate the given statement into english language\",\n",
+ " validators=[IsProfanityFree(on_fail=\"fix\")]\n",
+ " )"
]
},
{
@@ -143,19 +173,49 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import guardrails as gd\n",
+ "\n",
+ "from rich import print"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From XML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
- "import guardrails as gd\n",
- "\n",
- "from rich import print\n",
- "\n",
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or from our Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=Translation, prompt=prompt)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -165,7 +225,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 10,
"metadata": {
"tags": []
},
@@ -174,63 +234,59 @@
"data": {
"text/html": [
"\n",
- "\n",
"Translate the given statement into english language:\n",
"\n",
- "{ statement_to_be_translated} \n",
+ "${ statement_to_be_translated} \n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string name = \"translated_statement\" description = \"Translate the given statement into english language\" \n",
- "format = \"is-profanity-free: \" / > \n",
+ " <string name = \"translated_statement\" format = \"is-profanity-free\" description = \"Translate the given statement into \n",
+ "english language\" / > \n",
"< / output > \n",
"\n",
"\n",
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \n"
],
"text/plain": [
- "\n",
"\n",
"Translate the given statement into english language:\n",
"\n",
- "\u001b[1m{\u001b[0mstatement_to_be_translated\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mstatement_to_be_translated\u001b[1m}\u001b[0m\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -266,15 +322,14 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
},
{
@@ -314,7 +369,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
@@ -324,7 +379,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ │ Translate the given statement into english language: │ │\n",
" │ │ │ │\n",
" │ │ quesadilla de pollo │ │\n",
@@ -334,8 +388,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"translated_statement\" description=\"Translate the given statement into english │ │\n",
- " │ │ language\" format=\"is-profanity-free: \"/> │ │\n",
+ " │ │ <string name=\"translated_statement\" format=\"is-profanity-free\" description=\"Translate the given │ │\n",
+ " │ │ statement into english language\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -352,9 +406,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ {\"translated_statement\": \"Chicken Quesadilla\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
@@ -369,7 +432,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mTranslate the given statement into english language:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mquesadilla de pollo\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -379,8 +441,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -397,9 +459,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ │ Translate the given statement into english language: │ │\n",
" │ │ │ │\n",
" │ │ убей себя │ │\n",
@@ -498,8 +567,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"translated_statement\" description=\"Translate the given statement into english │ │\n",
- " │ │ language\" format=\"is-profanity-free: \"/> │ │\n",
+ " │ │ <string name=\"translated_statement\" format=\"is-profanity-free\" description=\"Translate the given │ │\n",
+ " │ │ statement into english language\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -516,9 +585,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ {\"translated_statement\": \"Kill yourself\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
@@ -533,7 +611,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mTranslate the given statement into english language:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mубей себя\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -543,8 +620,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -561,9 +638,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
"\n",
- "\n",
- "\n",
"\n",
" \n",
- "\n",
"Translate the given statement into the English language:\n",
"\n",
- "{{statement_to_be_translated}}\n",
+ "${statement_to_be_translated}\n",
"\n",
- "@complete_json_suffix\n",
+ "${gr.complete_json_suffix}\n",
"\n",
"\n",
"\n",
@@ -133,6 +123,37 @@
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or as a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 14,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pydantic import BaseModel, Field\n",
+ "from guardrails.validators import IsHighQualityTranslation\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "Translate the given statement into the English language:\n",
+ "\n",
+ "${statement_to_be_translated}\n",
+ "\n",
+ "${gr.complete_json_suffix}\n",
+ "\"\"\"\n",
+ "\n",
+ "class Translation(BaseModel):\n",
+ " translated_statement: str = Field(\n",
+ " description=\"Translate the given statement into the English language\",\n",
+ " validators=[IsHighQualityTranslation(on_fail=\"fix\")]\n",
+ " )"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -158,19 +179,49 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import guardrails as gd\n",
+ "\n",
+ "from rich import print"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From our RAIL string:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
- "import guardrails as gd\n",
- "\n",
- "from rich import print\n",
- "\n",
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or from our Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=Translation, prompt=prompt)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -180,7 +231,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 18,
"metadata": {
"tags": []
},
@@ -189,63 +240,59 @@
"data": {
"text/html": [
"\n",
- "\n",
"Translate the given statement into the English language:\n",
"\n",
- "{ statement_to_be_translated} \n",
+ "${ statement_to_be_translated} \n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string name = \"translated_statement\" description = \"Translate the given statement into the English language\" \n",
- "format = \"is-high-quality-translation: \" / > \n",
+ " <string name = \"translated_statement\" format = \"is-high-quality-translation\" description = \"Translate the given \n",
+ "statement into the English language\" / > \n",
"< / output > \n",
"\n",
"\n",
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \n"
],
"text/plain": [
- "\n",
"\n",
"Translate the given statement into the English language:\n",
"\n",
- "\u001b[1m{\u001b[0mstatement_to_be_translated\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mstatement_to_be_translated\u001b[1m}\u001b[0m\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -281,15 +328,14 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
},
{
@@ -309,9 +355,11 @@
"source": [
"import openai\n",
"\n",
+ "statement = \"これは簡単に翻訳できるかもしれない。\"\n",
"raw_llm_response, validated_response = guard(\n",
" openai.Completion.create,\n",
- " prompt_params={'statement_to_be_translated': 'これは簡単に翻訳できるかもしれない。'},\n",
+ " prompt_params={'statement_to_be_translated': statement},\n",
+ " metadata={'translation_source': statement},\n",
" engine='text-davinci-003',\n",
" max_tokens=2048,\n",
" temperature=0\n",
@@ -330,7 +378,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -340,7 +388,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ │ Translate the given statement into the English language: │ │\n",
" │ │ │ │\n",
" │ │ これは簡単に翻訳できるかもしれない。 │ │\n",
@@ -350,8 +397,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"translated_statement\" description=\"Translate the given statement into the English │ │\n",
- " │ │ language\" format=\"is-high-quality-translation: \"/> │ │\n",
+ " │ │ <string name=\"translated_statement\" format=\"is-high-quality-translation\" description=\"Translate the │ │\n",
+ " │ │ given statement into the English language\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -368,9 +415,18 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ {\"translated_statement\": \"This may be easy to translate.\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
@@ -385,7 +441,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mTranslate the given statement into the English language:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mこれは簡単に翻訳できるかもしれない。\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -395,8 +450,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -413,9 +468,18 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `Validated Output: { 'translated_statement' : '' } \n",
+ "Validated Output: { 'translated_statement' : \"It is so bad that it's a letdown.\" } \n",
" \n"
],
"text/plain": [
- "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'translated_statement'\u001b[0m: \u001b[32m''\u001b[0m\u001b[1m}\u001b[0m\n"
+ "Validated Output: \u001b[1m{\u001b[0m\u001b[32m'translated_statement'\u001b[0m: \u001b[32m\"It is so bad that it's a letdown.\"\u001b[0m\u001b[1m}\u001b[0m\n"
]
},
"metadata": {},
@@ -476,6 +539,7 @@
"raw_llm_response, validated_response = guard(\n",
" openai.Completion.create,\n",
" prompt_params={'statement_to_be_translated': 'ドン引きするほど翻訳が悪い。'},\n",
+ " metadata={'translation_source': 'ドン引きするほど翻訳が悪い。'},\n",
" engine='text-davinci-003',\n",
" max_tokens=2048,\n",
" temperature=0\n",
@@ -494,7 +558,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -504,7 +568,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ ╭──────────────────────────────────────────────── Prompt ─────────────────────────────────────────────────╮ │\n",
" │ │ │ │\n",
- " │ │ │ │\n",
" │ │ Translate the given statement into the English language: │ │\n",
" │ │ │ │\n",
" │ │ ドン引きするほど翻訳が悪い。 │ │\n",
@@ -514,8 +577,8 @@
" │ │ it into. │ │\n",
" │ │ │ │\n",
" │ │ <output> │ │\n",
- " │ │ <string name=\"translated_statement\" description=\"Translate the given statement into the English │ │\n",
- " │ │ language\" format=\"is-high-quality-translation: \"/> │ │\n",
+ " │ │ <string name=\"translated_statement\" format=\"is-high-quality-translation\" description=\"Translate the │ │\n",
+ " │ │ given statement into the English language\"/> │ │\n",
" │ │ </output> │ │\n",
" │ │ │ │\n",
" │ │ │ │\n",
@@ -532,14 +595,23 @@
" │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
" │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
" │ │ │ │\n",
- " │ │ JSON Object: │ │\n",
+ " │ │ │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ " │ │ ┃ Role ┃ Content ┃ │ │\n",
+ " │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ " │ │ └──────┴─────────┘ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
- " │ │ {\"translated_statement\": \"It's so bad that it's a letdown.\"} │ │\n",
+ " │ │ {\"translated_statement\": \"It is so bad that it's a letdown.\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭─────────────────────────────────────────── Validated Output ────────────────────────────────────────────╮ │\n",
- " │ │ {'translated_statement': ''} │ │\n",
+ " │ │ {'translated_statement': \"It is so bad that it's a letdown.\"} │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯\n",
" \n"
@@ -549,7 +621,6 @@
"└── ╭────────────────────────────────────────────────── Step 0 ───────────────────────────────────────────────────╮\n",
" │ \u001b[48;2;240;248;255m╭─\u001b[0m\u001b[48;2;240;248;255m───────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m Prompt \u001b[0m\u001b[48;2;240;248;255m────────────────────────────────────────────────\u001b[0m\u001b[48;2;240;248;255m─╮\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mTranslate the given statement into the English language:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mドン引きするほど翻訳が悪い。\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -559,8 +630,8 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mit into.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -577,14 +648,23 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `\n",
- "\n",
- "\n",
- "\n",
+ " return PassResult()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then we can define our RAIL spec either as XML:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "rail_str = \"\"\"\n",
+ "\n",
"\n",
"\n",
" \n",
"Generate a move for the chess board. The board is currently in the following state:\n",
- "{{board_state}}\n",
- "@complete_json_suffix\n",
+ "${board_state}\n",
+ "${gr.complete_json_suffix}\n",
" \n",
"\n",
" \n",
"\"\"\""
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Or as a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pydantic import BaseModel, Field\n",
+ "\n",
+ "prompt = \"\"\"\n",
+ "Generate a move for the chess board. The board is currently in the following state:\n",
+ "${board_state}\n",
+ "${gr.complete_json_suffix}\n",
+ "\"\"\"\n",
+ "\n",
+ "class ChessMove(BaseModel):\n",
+ " move: str = Field(\n",
+ " description=\"A move in standard algebraic notation.\",\n",
+ " validators=[IsValidChessMove(on_fail=\"reask\")]\n",
+ " )"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -124,15 +175,38 @@
"3. Compiles the schema and type info from the RAIL spec and adds it to the prompt."
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From XML:"
+ ]
+ },
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"guard = gd.Guard.from_rail_string(rail_str)"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "From a Pydantic model:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "guard = gd.Guard.from_pydantic(output_class=ChessMove, prompt=prompt)"
+ ]
+ },
{
"attachments": {},
"cell_type": "markdown",
@@ -143,7 +217,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -151,60 +225,52 @@
"text/html": [
"\n",
"Generate a move for the chess board. The board is currently in the following state:\n",
- "{ board_state} \n",
+ "${ board_state} \n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"< output > \n",
- " <string description = \"A move in standard algebraic notation.\" name = \"move\" required = \"true\" \n",
- "format = \"is-valid-chess-move\" / > \n",
+ " <string name = \"move\" format = \"is-valid-chess-move\" description = \"A move in standard algebraic notation.\" / > \n",
"< / output > \n",
"\n",
"\n",
- "\n",
- "\n",
"ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
"attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
"MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. If you are unsure anywhere, enter ` None `. \n",
+ "specific types. Be correct and concise. If you are unsure anywhere, enter `null`. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
- "JSON Object:\n",
"\n",
" \n"
],
"text/plain": [
"\n",
"Generate a move for the chess board. The board is currently in the following state:\n",
- "\u001b[1m{\u001b[0mboard_state\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mboard_state\u001b[1m}\u001b[0m\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
"\n",
"\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
+ "\u001b[39m \u001b[0m\n",
"\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
"\n",
"\n",
- "\n",
- "\n",
"\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
"\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
"\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `\u001b[0m\u001b[3;35mNone\u001b[0m\u001b[39m`.\u001b[0m\n",
+ "\u001b[39mspecific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
- "JSON Object:\n",
"\n"
]
},
@@ -226,17 +292,26 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
"data": {
- "image/svg+xml": "r n b q k b n r\np p p p p p p p\n. . . . . . . .\n. . . . . . . .\n. . . . . . . .\n. . . . . . . .\nP P P P P P P P\nR N B Q K B N R r n b q k b n r\n",
+ "p p p p p p p p\n",
+ ". . . . . . . .\n",
+ ". . . . . . . .\n",
+ ". . . . . . . .\n",
+ ". . . . . . . .\n",
+ "P P P P P P P P\n",
+ "R N B Q K B N R r n b q k b n r\np p p p p p p p\n. . . . . . . .\n. . . . . . . .\n. . . . P . . .\n. . . . . . . .\nP P P P . P P P\nR N B Q K B N R r n b q k b n r\n",
+ "p p p p p p p p\n",
+ ". . . . . . . .\n",
+ ". . . . . . . .\n",
+ ". . . . P . . .\n",
+ ". . . . . . . .\n",
+ "P P P P . P P P\n",
+ "R N B Q K B N R r n b q k b n r\np p p p . p p p\n. . . . . . . .\n. . . . p . . .\n. . . . P . . .\n. . . . . . . .\nP P P P . P P P\nR N B Q K B N R r n b q k b n r\n",
+ "p p p p . p p p\n",
+ ". . . . . . . .\n",
+ ". . . . p . . .\n",
+ ". . . . P . . .\n",
+ ". . . . . . . .\n",
+ "P P P P . P P P\n",
+ "R N B Q K B N R r n b q k b n r\np p p p . p p p\n. . . . . . . .\n. . . . p . . .\n. . . . P . . .\n. . . . . N . .\nP P P P . P P P\nR N B Q K B . R r n b q k b n r\n",
+ "p p p p . p p p\n",
+ ". . . . . . . .\n",
+ ". . . . p . . .\n",
+ ". . . . P . . .\n",
+ ". . . . . N . .\n",
+ "P P P P . P P P\n",
+ "R N B Q K B . R r . b q k b n r\np p p p . p p p\n. . n . . . . .\n. . . . p . . .\n. . . . P . . .\n. . . . . N . .\nP P P P . P P P\nR N B Q K B . R r . b q k b n r\n",
+ "p p p p . p p p\n",
+ ". . n . . . . .\n",
+ ". . . . p . . .\n",
+ ". . . . P . . .\n",
+ ". . . . . N . .\n",
+ "P P P P . P P P\n",
+ "R N B Q K B . R
+ This is a prompt with an & character.
+
+```
+
+You need to escape the `&` character like this:
+
+```xml
+
+ This is a prompt with an & character.
+
+```
+
+If you're still encountering issues, please [open an issue](https://github.com/ShreyaR/guardrails/issues/new) and we'll help you out!
diff --git a/docs/getting_started.ipynb b/docs/guardrails_ai/getting_started.ipynb
similarity index 76%
rename from docs/getting_started.ipynb
rename to docs/guardrails_ai/getting_started.ipynb
index bb0a80f7b..52681cb98 100644
--- a/docs/getting_started.ipynb
+++ b/docs/guardrails_ai/getting_started.ipynb
@@ -18,7 +18,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
@@ -64,12 +64,10 @@
"\n",
"`RAIL` is a flavor of XML (standing for **R**eliable **AI** markup **L**anguage) that describes the expected structure and type of the output of the LLM, the quality criteria for the output to be valid and corrective actions to be taken if the output is invalid.\n",
"\n",
- "A `RAIL` spec is composed of 3 main components:\n",
+ "A `RAIL` spec is composed of 2 main components:\n",
"\n",
"1. Output schema\n",
"2. Prompt\n",
- "3. (Optional) Script\n",
- "\n",
"\n",
"### Output schema\n",
"\n",
@@ -206,9 +204,9 @@
"Given the following doctor's notes about a patient,\n",
"please extract a dictionary that contains the patient's information. \n",
"\n",
- "{{doctors_notes}} \n",
+ "${doctors_notes} \n",
"\n",
- "@complete_json_suffix_v2 \n",
+ "${gr.complete_json_suffix_v2} \n",
"\n",
"```\n",
"\n",
@@ -234,7 +232,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -263,15 +261,15 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "/Users/shreyarajpal/anaconda3/envs/tiff-env/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar. (This may have returned Python scalars in past versions.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n",
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
]
},
{
@@ -285,7 +283,7 @@
" { 'symptom' : 'macular rash' , 'affected area' : 'head' } ,\n",
" { 'symptom' : 'itchy, flaky, slightly scaly' , 'affected area' : 'neck' } \n",
" ] ,\n",
- " 'current_meds' : [{ 'medication' : 'OTC steroid cream' , 'response' : 'Moderate' }] \n",
+ " 'current_meds' : [{ 'medication' : 'OTC steroid cream' , 'response' : 'Moderate response' }] \n",
" } \n",
"} \n",
"
\n"
@@ -299,7 +297,7 @@
" \u001b[1m{\u001b[0m\u001b[32m'symptom'\u001b[0m: \u001b[32m'macular rash'\u001b[0m, \u001b[32m'affected area'\u001b[0m: \u001b[32m'head'\u001b[0m\u001b[1m}\u001b[0m,\n",
" \u001b[1m{\u001b[0m\u001b[32m'symptom'\u001b[0m: \u001b[32m'itchy, flaky, slightly scaly'\u001b[0m, \u001b[32m'affected area'\u001b[0m: \u001b[32m'neck'\u001b[0m\u001b[1m}\u001b[0m\n",
" \u001b[1m]\u001b[0m,\n",
- " \u001b[32m'current_meds'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'medication'\u001b[0m: \u001b[32m'OTC steroid cream'\u001b[0m, \u001b[32m'response'\u001b[0m: \u001b[32m'Moderate'\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n",
+ " \u001b[32m'current_meds'\u001b[0m: \u001b[1m[\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'medication'\u001b[0m: \u001b[32m'OTC steroid cream'\u001b[0m, \u001b[32m'response'\u001b[0m: \u001b[32m'Moderate response'\u001b[0m\u001b[1m}\u001b[0m\u001b[1m]\u001b[0m\n",
" \u001b[1m}\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
@@ -351,7 +349,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -404,7 +402,17 @@
"│ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ \n",
"│ │ \n",
"│ │ \n",
+ "│ │ \n",
+ "│ Json Output: │ \n",
+ "│ │ \n",
+ "│ │ \n",
"╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ \n",
+ "╭──────────────────────────────────────────────── Message History ────────────────────────────────────────────────╮ \n",
+ "│ ┏━━━━━━┳━━━━━━━━━┓ │ \n",
+ "│ ┃ Role ┃ Content ┃ │ \n",
+ "│ ┡━━━━━━╇━━━━━━━━━┩ │ \n",
+ "│ └──────┴─────────┘ │ \n",
+ "╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ \n",
"╭──────────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────────╮ \n",
"│ { │ \n",
"│ \"patient_info\": { │ \n",
@@ -423,7 +431,7 @@
"│ \"current_meds\": [ │ \n",
"│ { │ \n",
"│ \"medication\": \"OTC steroid cream\", │ \n",
- "│ \"response\": \"Moderate\" │ \n",
+ "│ \"response\": \"Moderate response\" │ \n",
"│ } │ \n",
"│ ] │ \n",
"│ } │ \n",
@@ -437,25 +445,41 @@
"│ 'symptoms': [ │ \n",
"│ { │ \n",
"│ 'symptom': 'macular rash', │ \n",
- "│ 'affected area': ReAsk( │ \n",
+ "│ 'affected area': FieldReAsk( │ \n",
"│ incorrect_value='face & hair', │ \n",
- "│ error_message=\"Value face & hair is not in choices ['head', 'neck', 'chest'].\", │ \n",
- "│ fix_value=None, │ \n",
+ "│ fail_results=[ │ \n",
+ "│ FailResult( │ \n",
+ "│ outcome='fail', │ \n",
+ "│ metadata=None, │ \n",
+ "│ error_message=\"Value face & hair is not in choices ['head', 'neck', 'chest'].\", │ \n",
+ "│ fix_value=None │ \n",
+ "│ ) │ \n",
+ "│ ], │ \n",
"│ path=['patient_info', 'symptoms', 0, 'affected area'] │ \n",
"│ ) │ \n",
"│ }, │ \n",
"│ { │ \n",
"│ 'symptom': 'itchy, flaky, slightly scaly', │ \n",
- "│ 'affected area': ReAsk( │ \n",
+ "│ 'affected area': FieldReAsk( │ \n",
"│ incorrect_value='beard, eyebrows & nares', │ \n",
- "│ error_message=\"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\", │ \n",
- "│ fix_value=None, │ \n",
+ "│ fail_results=[ │ \n",
+ "│ FailResult( │ \n",
+ "│ outcome='fail', │ \n",
+ "│ metadata=None, │ \n",
+ "│ error_message=\"Value beard, eyebrows & nares is not in choices ['head', 'neck', │ \n",
+ "│ 'chest'].\", │ \n",
+ "│ fix_value=None │ \n",
+ "│ ) │ \n",
+ "│ ], │ \n",
"│ path=['patient_info', 'symptoms', 1, 'affected area'] │ \n",
"│ ) │ \n",
"│ } │ \n",
"│ ], │ \n",
"│ 'current_meds': [ │ \n",
- "│ {'medication': 'OTC steroid cream', 'response': 'Moderate'} │ \n",
+ "│ { │ \n",
+ "│ 'medication': 'OTC steroid cream', │ \n",
+ "│ 'response': 'Moderate response' │ \n",
+ "│ } │ \n",
"│ ] │ \n",
"│ } │ \n",
"│ } │ \n",
@@ -510,7 +534,17 @@
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mformat=\"1-indexed\" /> │ { │ \n",
"│ \"affected area\": { │ \n",
"│ \"incorrect_value\": \"face & hair\", │ \n",
- "│ \"error_message\": \"Value face & hair is not in choices ['head', 'neck', 'chest'].\" │ \n",
+ "│ \"error_messages\": [ │ \n",
+ "│ \"Value face & hair is not in choices ['head', 'neck', 'chest'].\" │ \n",
+ "│ ] │ \n",
"│ } │ \n",
"│ }, │ \n",
"│ { │ \n",
"│ \"affected area\": { │ \n",
"│ \"incorrect_value\": \"beard, eyebrows & nares\", │ \n",
- "│ \"error_message\": \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\" │ \n",
+ "│ \"error_messages\": [ │ \n",
+ "│ \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\" │ \n",
+ "│ ] │ \n",
"│ } │ \n",
"│ } │ \n",
"│ ] │ \n",
@@ -642,13 +696,30 @@
"│ JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects │ \n",
"│ and specific types. Be correct and concise. If you are unsure anywhere, enter `null`. │ \n",
"│ │ \n",
- "│ Here are examples of simple (XML, JSON) pairs that show the expected behavior: │ \n",
- "│ - `<string name='foo' format='two-words lower-case' />` => `{{'foo': 'example one'}}` │ \n",
- "│ - `<list name='bar'><string format='upper-case' /></list>` => `{{\"bar\": ['STRING ONE', 'STRING TWO', etc.]}}` │ \n",
- "│ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ \n",
- "│ format=\"1-indexed\" /></object>` => `{{'baz': {{'foo': 'Some String', 'index': 1}}}}` │ \n",
+ "│ │ \n",
+ "│ Json Output: │ \n",
+ "│ │ \n",
"│ │ \n",
"╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ \n",
+ "╭───────────────────────────────────────────────── Instructions ──────────────────────────────────────────────────╮ \n",
+ "│ │ \n",
+ "│ You are a helpful assistant only capable of communicating with valid JSON, and no other text. │ \n",
+ "│ │ \n",
+ "│ ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` │ \n",
+ "│ attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The │ \n",
+ "│ JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects │ \n",
+ "│ and specific types. Be correct and concise. If you are unsure anywhere, enter `null`. │ \n",
+ "│ │ \n",
+ "│ Here are examples of simple (XML, JSON) pairs that show the expected behavior: │ \n",
+ "│ - `<string name='foo' format='two-words lower-case' />` => `{'foo': 'example one'}` │ \n",
+ "│ - `<list name='bar'><string format='upper-case' /></list>` => `{\"bar\": ['STRING ONE', 'STRING TWO', etc.]}` │ \n",
+ "│ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ \n",
+ "│ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ \n",
+ "│ │ \n",
+ "╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ \n",
+ "╭──────────────────────────────────────────────── Message History ────────────────────────────────────────────────╮ \n",
+ "│ No message history. │ \n",
+ "╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ \n",
"╭──────────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────────╮ \n",
"│ { │ \n",
"│ \"patient_info\": { │ \n",
@@ -676,7 +747,10 @@
"│ } │ \n",
"│ ], │ \n",
"│ 'current_meds': [ │ \n",
- "│ {'medication': 'OTC steroid cream', 'response': 'Moderate'} │ \n",
+ "│ { │ \n",
+ "│ 'medication': 'OTC steroid cream', │ \n",
+ "│ 'response': 'Moderate response' │ \n",
+ "│ } │ \n",
"│ ] │ \n",
"│ } │ \n",
"│ } │ \n",
@@ -694,13 +768,17 @@
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"affected area\": {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"incorrect_value\": \"face & hair\",\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
- "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_message\": \"Value face & hair is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_messages\": [\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"Value face & hair is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m },\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"affected area\": {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"incorrect_value\": \"beard, eyebrows & nares\",\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
- "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_message\": \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_messages\": [\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
+ "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
@@ -730,13 +808,30 @@
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mJSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mand specific types. Be correct and concise. If you are unsure anywhere, enter `null`.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
"\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
- "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mHere are examples of simple (XML, JSON) pairs that show the expected behavior:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
- "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `
` => `{{\"bar\": ['STRING ONE', 'STRING TWO', etc.]}}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m\n",
- "\u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `
` => `{\"bar\": ['STRING ONE', 'STRING TWO', etc.]}`\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m\n",
+ "\u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m- `│ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
"│ │ │ │ │\n",
"│ │ │ │ │\n",
+ "│ │ │ │ │\n",
+ "│ │ │ Json Output: │ │\n",
+ "│ │ │ │ │\n",
+ "│ │ │ │ │\n",
"│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ "│ │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ "│ │ │ ┏━━━━━━┳━━━━━━━━━┓ │ │\n",
+ "│ │ │ ┃ Role ┃ Content ┃ │ │\n",
+ "│ │ │ ┡━━━━━━╇━━━━━━━━━┩ │ │\n",
+ "│ │ │ └──────┴─────────┘ │ │\n",
+ "│ │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
"│ │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
"│ │ │ { │ │\n",
"│ │ │ \"patient_info\": { │ │\n",
@@ -877,7 +985,7 @@
"│ │ │ \"current_meds\": [ │ │\n",
"│ │ │ { │ │\n",
"│ │ │ \"medication\": \"OTC steroid cream\", │ │\n",
- "│ │ │ \"response\": \"Moderate\" │ │\n",
+ "│ │ │ \"response\": \"Moderate response\" │ │\n",
"│ │ │ } │ │\n",
"│ │ │ ] │ │\n",
"│ │ │ } │ │\n",
@@ -891,26 +999,42 @@
"│ │ │ 'symptoms': [ │ │\n",
"│ │ │ { │ │\n",
"│ │ │ 'symptom': 'macular rash', │ │\n",
- "│ │ │ 'affected area': ReAsk( │ │\n",
+ "│ │ │ 'affected area': FieldReAsk( │ │\n",
"│ │ │ incorrect_value='face & hair', │ │\n",
- "│ │ │ error_message=\"Value face & hair is not in choices ['head', 'neck', 'chest'].\", │ │\n",
- "│ │ │ fix_value=None, │ │\n",
+ "│ │ │ fail_results=[ │ │\n",
+ "│ │ │ FailResult( │ │\n",
+ "│ │ │ outcome='fail', │ │\n",
+ "│ │ │ metadata=None, │ │\n",
+ "│ │ │ error_message=\"Value face & hair is not in choices ['head', 'neck', │ │\n",
+ "│ │ │ 'chest'].\", │ │\n",
+ "│ │ │ fix_value=None │ │\n",
+ "│ │ │ ) │ │\n",
+ "│ │ │ ], │ │\n",
"│ │ │ path=['patient_info', 'symptoms', 0, 'affected area'] │ │\n",
"│ │ │ ) │ │\n",
"│ │ │ }, │ │\n",
"│ │ │ { │ │\n",
"│ │ │ 'symptom': 'itchy, flaky, slightly scaly', │ │\n",
- "│ │ │ 'affected area': ReAsk( │ │\n",
+ "│ │ │ 'affected area': FieldReAsk( │ │\n",
"│ │ │ incorrect_value='beard, eyebrows & nares', │ │\n",
- "│ │ │ error_message=\"Value beard, eyebrows & nares is not in choices ['head', 'neck', │ │\n",
- "│ │ │ 'chest'].\", │ │\n",
- "│ │ │ fix_value=None, │ │\n",
+ "│ │ │ fail_results=[ │ │\n",
+ "│ │ │ FailResult( │ │\n",
+ "│ │ │ outcome='fail', │ │\n",
+ "│ │ │ metadata=None, │ │\n",
+ "│ │ │ error_message=\"Value beard, eyebrows & nares is not in choices ['head', │ │\n",
+ "│ │ │ 'neck', 'chest'].\", │ │\n",
+ "│ │ │ fix_value=None │ │\n",
+ "│ │ │ ) │ │\n",
+ "│ │ │ ], │ │\n",
"│ │ │ path=['patient_info', 'symptoms', 1, 'affected area'] │ │\n",
"│ │ │ ) │ │\n",
"│ │ │ } │ │\n",
"│ │ │ ], │ │\n",
"│ │ │ 'current_meds': [ │ │\n",
- "│ │ │ {'medication': 'OTC steroid cream', 'response': 'Moderate'} │ │\n",
+ "│ │ │ { │ │\n",
+ "│ │ │ 'medication': 'OTC steroid cream', │ │\n",
+ "│ │ │ 'response': 'Moderate response' │ │\n",
+ "│ │ │ } │ │\n",
"│ │ │ ] │ │\n",
"│ │ │ } │ │\n",
"│ │ │ } │ │\n",
@@ -927,13 +1051,17 @@
" │ │ { │ │\n",
" │ │ \"affected area\": { │ │\n",
" │ │ \"incorrect_value\": \"face & hair\", │ │\n",
- " │ │ \"error_message\": \"Value face & hair is not in choices ['head', 'neck', 'chest'].\" │ │\n",
+ " │ │ \"error_messages\": [ │ │\n",
+ " │ │ \"Value face & hair is not in choices ['head', 'neck', 'chest'].\" │ │\n",
+ " │ │ ] │ │\n",
" │ │ } │ │\n",
" │ │ }, │ │\n",
" │ │ { │ │\n",
" │ │ \"affected area\": { │ │\n",
" │ │ \"incorrect_value\": \"beard, eyebrows & nares\", │ │\n",
- " │ │ \"error_message\": \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\" │ │\n",
+ " │ │ \"error_messages\": [ │ │\n",
+ " │ │ \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\" │ │\n",
+ " │ │ ] │ │\n",
" │ │ } │ │\n",
" │ │ } │ │\n",
" │ │ ] │ │\n",
@@ -964,14 +1092,32 @@
" │ │ requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, │ │\n",
" │ │ enter `null`. │ │\n",
" │ │ │ │\n",
- " │ │ Here are examples of simple (XML, JSON) pairs that show the expected behavior: │ │\n",
- " │ │ - `<string name='foo' format='two-words lower-case' />` => `{{'foo': 'example one'}}` │ │\n",
- " │ │ - `<list name='bar'><string format='upper-case' /></list>` => `{{\"bar\": ['STRING ONE', 'STRING TWO', │ │\n",
- " │ │ etc.]}}` │ │\n",
- " │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
- " │ │ format=\"1-indexed\" /></object>` => `{{'baz': {{'foo': 'Some String', 'index': 1}}}}` │ │\n",
+ " │ │ │ │\n",
+ " │ │ Json Output: │ │\n",
+ " │ │ │ │\n",
" │ │ │ │\n",
" │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭───────────────────────────────────────────── Instructions ──────────────────────────────────────────────╮ │\n",
+ " │ │ │ │\n",
+ " │ │ You are a helpful assistant only capable of communicating with valid JSON, and no other text. │ │\n",
+ " │ │ │ │\n",
+ " │ │ ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the │ │\n",
+ " │ │ `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding │ │\n",
+ " │ │ XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. │ │\n",
+ " │ │ requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, │ │\n",
+ " │ │ enter `null`. │ │\n",
+ " │ │ │ │\n",
+ " │ │ Here are examples of simple (XML, JSON) pairs that show the expected behavior: │ │\n",
+ " │ │ - `<string name='foo' format='two-words lower-case' />` => `{'foo': 'example one'}` │ │\n",
+ " │ │ - `<list name='bar'><string format='upper-case' /></list>` => `{\"bar\": ['STRING ONE', 'STRING TWO', │ │\n",
+ " │ │ etc.]}` │ │\n",
+ " │ │ - `<object name='baz'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" │ │\n",
+ " │ │ format=\"1-indexed\" /></object>` => `{'baz': {'foo': 'Some String', 'index': 1}}` │ │\n",
+ " │ │ │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
+ " │ ╭──────────────────────────────────────────── Message History ────────────────────────────────────────────╮ │\n",
+ " │ │ No message history. │ │\n",
+ " │ ╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯ │\n",
" │ ╭──────────────────────────────────────────── Raw LLM Output ─────────────────────────────────────────────╮ │\n",
" │ │ { │ │\n",
" │ │ \"patient_info\": { │ │\n",
@@ -999,7 +1145,10 @@
" │ │ } │ │\n",
" │ │ ], │ │\n",
" │ │ 'current_meds': [ │ │\n",
- " │ │ {'medication': 'OTC steroid cream', 'response': 'Moderate'} │ │\n",
+ " │ │ { │ │\n",
+ " │ │ 'medication': 'OTC steroid cream', │ │\n",
+ " │ │ 'response': 'Moderate response' │ │\n",
+ " │ │ } │ │\n",
" │ │ ] │ │\n",
" │ │ } │ │\n",
" │ │ } │ │\n",
@@ -1059,7 +1208,17 @@
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mformat=\"1-indexed\" />
` => `{'baz': {'foo': 'Some String', 'index': 1}}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mJson Output:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;248;255m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m╭─\u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m Message History \u001b[0m\u001b[48;2;231;223;235m───────────────────────────────────────────\u001b[0m\u001b[48;2;231;223;235m─╮\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m┏━━━━━━┳━━━━━━━━━┓\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m┃\u001b[0m\u001b[1;48;2;231;223;235m \u001b[0m\u001b[1;48;2;231;223;235mRole\u001b[0m\u001b[1;48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m┃\u001b[0m\u001b[1;48;2;231;223;235m \u001b[0m\u001b[1;48;2;231;223;235mContent\u001b[0m\u001b[1;48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m┃\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m┡━━━━━━╇━━━━━━━━━┩\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m│\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m└──────┴─────────┘\u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m \u001b[0m\u001b[48;2;231;223;235m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;231;223;235m╰─────────────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m╭─\u001b[0m\u001b[48;2;245;245;220m───────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m Raw LLM Output \u001b[0m\u001b[48;2;245;245;220m────────────────────────────────────────────\u001b[0m\u001b[48;2;245;245;220m─╮\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m{\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \"patient_info\": {\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1078,7 +1237,7 @@
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \"current_meds\": [\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m {\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \"medication\": \"OTC steroid cream\",\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
- "│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \"response\": \"Moderate\"\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \"response\": \"Moderate response\"\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m }\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m ]\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
"│ │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m }\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1092,26 +1251,42 @@
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'symptoms': [\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m {\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'symptom': 'macular rash',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'affected area': ReAsk(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'affected area': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m incorrect_value='face & hair',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m error_message=\"Value face & hair is not in choices ['head', 'neck', 'chest'].\",\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fix_value=None,\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fail_results=[\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m FailResult(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m outcome='fail',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m metadata=None,\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m error_message=\"Value face & hair is not in choices ['head', 'neck', \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m'chest'].\",\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fix_value=None\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m )\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m ],\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m path=['patient_info', 'symptoms', 0, 'affected area']\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m )\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m },\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m {\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'symptom': 'itchy, flaky, slightly scaly',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'affected area': ReAsk(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'affected area': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m incorrect_value='beard, eyebrows & nares',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m error_message=\"Value beard, eyebrows & nares is not in choices ['head', 'neck', \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m'chest'].\",\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fix_value=None,\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fail_results=[\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m FailResult(\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m outcome='fail',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m metadata=None,\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m error_message=\"Value beard, eyebrows & nares is not in choices ['head', \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m'neck', 'chest'].\",\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m fix_value=None\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m )\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m ],\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m path=['patient_info', 'symptoms', 1, 'affected area']\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m )\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m }\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m ],\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'current_meds': [\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
- "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m {'medication': 'OTC steroid cream', 'response': 'Moderate'}\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m {\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'medication': 'OTC steroid cream',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m 'response': 'Moderate response'\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+ "│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m }\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m ]\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m }\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
"│ │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m}\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
@@ -1128,13 +1303,17 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"affected area\": {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"incorrect_value\": \"face & hair\",\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_message\": \"Value face & hair is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_messages\": [\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"Value face & hair is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m },\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"affected area\": {\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"incorrect_value\": \"beard, eyebrows & nares\",\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_message\": \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"error_messages\": [\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \"Value beard, eyebrows & nares is not in choices ['head', 'neck', 'chest'].\"\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
+ " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m }\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m ]\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
@@ -1165,14 +1344,32 @@
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mrequests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255menter `null`.\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
" │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255mHere are examples of simple (XML, JSON) pairs that show the expected behavior:\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `
` => `{{\"bar\": ['STRING ONE', 'STRING TWO', \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255metc.]}}`\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m│\u001b[0m │\n",
- " │ \u001b[48;2;240;248;255m│\u001b[0m\u001b[48;2;240;248;255m \u001b[0m\u001b[48;2;240;248;255m- `
` => `{\"bar\": ['STRING ONE', 'STRING TWO', \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+ " │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242metc.]}`\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m│\u001b[0m │\n",
+ " │ \u001b[48;2;255;240;242m│\u001b[0m\u001b[48;2;255;240;242m \u001b[0m\u001b[48;2;255;240;242m- `{ doctors_notes} \n",
+ "${ doctors_notes} \n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
@@ -155,10 +155,10 @@
"specific types. Be correct and concise. \n",
"\n",
"Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
+ "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` { 'foo' : 'example one' } ` \n",
+ "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` { \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]} ` \n",
"- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
+ "/ >< / object >` = > `{ 'baz' : { 'foo' : 'Some String' , 'index' : 1 }} `\n",
"\n",
"\n",
" \n"
@@ -169,7 +169,7 @@
"Given the following doctor's notes about a patient, please extract a dictionary that contains the patient's \n",
"information.\n",
"\n",
- "\u001b[1m{\u001b[0mdoctors_notes\u001b[1m}\u001b[0m\n",
+ "$\u001b[1m{\u001b[0mdoctors_notes\u001b[1m}\u001b[0m\n",
"\n",
"\n",
"Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
@@ -189,10 +189,10 @@
"\u001b[39mspecific types. Be correct and concise.\u001b[0m\n",
"\n",
"\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
+ "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
"\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
+ "\u001b[35m/\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
"\n",
"\n"
]
@@ -202,14 +202,14 @@
}
],
"source": [
- "print(output_parser.guard.base_prompt)"
+ "print(output_parser.guard.prompt)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "We can now create a LangChain `PromptTemplate` from this output parser."
+ "We can now create a LangChain `PromptTemplate` from this output parser. Note that the `PromptTemplate` class from LangChain utilizes f-strings. In order to prevent it from trying to treat our example json as variables that should be substituted we will escape our prompt."
]
},
{
@@ -221,7 +221,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 5,
"metadata": {
"ExecuteTime": {
"end_time": "2023-06-21T16:39:54.395309Z",
@@ -231,7 +231,7 @@
"outputs": [],
"source": [
"prompt = PromptTemplate(\n",
- " template=output_parser.guard.base_prompt,\n",
+ " template=output_parser.guard.prompt.escape(),\n",
" input_variables=output_parser.guard.prompt.variable_names,\n",
")"
]
@@ -245,7 +245,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 7,
"metadata": {
"ExecuteTime": {
"end_time": "2023-06-21T16:39:57.246325Z",
@@ -266,9 +266,16 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 8,
"metadata": {},
"outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Async event loop found, but guard was invoked synchronously.For validator parallelization, please call `validate_async` instead.\n"
+ ]
+ },
{
"data": {
"text/html": [
@@ -276,8 +283,7 @@
" 'patient_info' : { \n",
" 'gender' : 'Male' ,\n",
" 'age' : 49 ,\n",
- " 'symptoms' : 'Chronic macular rash to face & hair, worse in beard, eyebrows & nares. Itchy, flaky, slightly \n",
- "scaly. Moderate response to OTC steroid cream' \n",
+ " 'symptoms' : 'Itchy, flaky, slightly scaly. Moderate response to OTC steroid cream' \n",
" } \n",
"} \n",
"\n"
@@ -287,8 +293,7 @@
" \u001b[32m'patient_info'\u001b[0m: \u001b[1m{\u001b[0m\n",
" \u001b[32m'gender'\u001b[0m: \u001b[32m'Male'\u001b[0m,\n",
" \u001b[32m'age'\u001b[0m: \u001b[1;36m49\u001b[0m,\n",
- " \u001b[32m'symptoms'\u001b[0m: \u001b[32m'Chronic macular rash to face & hair, worse in beard, eyebrows & nares. Itchy, flaky, slightly \u001b[0m\n",
- "\u001b[32mscaly. Moderate response to OTC steroid cream'\u001b[0m\n",
+ " \u001b[32m'symptoms'\u001b[0m: \u001b[32m'Itchy, flaky, slightly scaly. Moderate response to OTC steroid cream'\u001b[0m\n",
" \u001b[1m}\u001b[0m\n",
"\u001b[1m}\u001b[0m\n"
]
@@ -318,7 +323,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.0"
+ "version": "3.11.4"
}
},
"nbformat": 4,
diff --git a/docs/integrations/llm_api_wrappers.md b/docs/integrations/llm_api_wrappers.md
new file mode 100644
index 000000000..7a2c2a165
--- /dev/null
+++ b/docs/integrations/llm_api_wrappers.md
@@ -0,0 +1,133 @@
+# Use Guardrails with LLM APIs
+
+Guardrails' `Guard` wrappers provide a simple way to add Guardrails to your LLM API calls. The wrappers are designed to be used with any LLM API.
+
+
+Here are some examples of how to use the wrappers with different LLM providers and models:
+
+## OpenAI
+
+### Completion Models (e.g. GPT-3)
+
+```python
+import openai
+import guardrails as gd
+
+
+# Create a Guard class
+guard = gd.Guard.from_rail(...)
+
+# Wrap openai API call
+raw_llm_output, guardrail_output = guard(
+ openai.Completion.create,
+ prompt_params={"prompt_param_1": "value_1", "prompt_param_2": "value_2", ..},
+ engine="text-davinci-003",
+ max_tokens=100,
+ temperature=0.0,
+)
+```
+
+### ChatCompletion Models (e.g. ChatGPT)
+
+```python
+import openai
+import guardrails as gd
+
+# Create a Guard class
+guard = gd.Guard.from_rail(...)
+
+# Wrap openai API call
+raw_llm_output, guardrail_output = guard(
+ openai.ChatCompletion.create,
+ prompt_params={"prompt_param_1": "value_1", "prompt_param_2": "value_2", ..},
+ system_prompt="You are a helpful assistant...",
+ model="gpt-3.5-turbo",
+ max_tokens=100,
+ temperature=0.0,
+)
+```
+
+## Cohere
+
+### Generate (e.g. GPT-3)
+
+```python
+import cohere
+import guardrails as gd
+
+# Create a Guard class
+guard = gd.Guard.from_rail(...)
+
+# Create a Cohere client
+cohere_client = cohere.Client(api_key="my_api_key")
+
+# Wrap cohere API call
+raw_llm_output, guardrail_output = guard(
+ cohere_client.generate,
+ prompt_params={"prompt_param_1": "value_1", "prompt_param_2": "value_2", ..},
+ model="command-nightly",
+ max_tokens=100,
+ ...
+)
+```
+
+## Using Manifest
+[Manifest](https://github.com/HazyResearch/manifest) is a wrapper around most model APIs and supports hosting local models. It can be used as a LLM API.
+
+```python
+import guardrails as gd
+import manifest
+
+# Create a Guard class
+guard = gd.Guard.from_rail(...)
+
+# Create a Manifest client - this one points to GPT-4
+# and caches responses in SQLLite
+manifest = manifest.Manifest(
+ client_name="openai",
+ engine="gpt-4",
+ cache_name="sqlite",
+ cache_connection="my_manifest_cache.db"
+)
+
+# Wrap openai API call
+raw_llm_output, guardrail_output = guard(
+ manifest,
+ prompt_params={"prompt_param_1": "value_1", "prompt_param_2": "value_2", ..},
+ max_tokens=100,
+ temperature=0.0,
+)
+```
+
+
+## Using a custom LLM API
+
+```python
+import guardrails as gd
+
+# Create a Guard class
+guard = gd.Guard.from_rail(...)
+
+# Function that takes the prompt as a string and returns the LLM output as string
+def my_llm_api(prompt: str, **kwargs) -> str:
+ """Custom LLM API wrapper.
+
+ Args:
+ prompt (str): The prompt to be passed to the LLM API
+ **kwargs: Any additional arguments to be passed to the LLM API
+
+ Returns:
+ str: The output of the LLM API
+ """
+
+ # Call your LLM API here
+ return ...
+
+
+# Wrap your LLM API call
+raw_llm_output, guardrail_output = guard(
+ my_llm_api,
+ prompt_params={"prompt_param_1": "value_1", "prompt_param_2": "value_2", ..},
+ **kwargs,
+)
+```
diff --git a/docs/integrations/pydantic_validation.ipynb b/docs/integrations/pydantic_validation.ipynb
deleted file mode 100644
index 81d55adb9..000000000
--- a/docs/integrations/pydantic_validation.ipynb
+++ /dev/null
@@ -1,570 +0,0 @@
-{
- "cells": [
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Validating LLM Outputs with Pydantic\n",
- "\n",
- "!!! note\n",
- " To download this example as a Jupyter notebook, click [here](https://github.com/ShreyaR/guardrails/blob/main/docs/integrations/pydantic_validation.ipynb).\n",
- "\n",
- "In this example, we will use Guardrails with Pydantic.\n",
- "\n",
- "## Objective\n",
- "\n",
- "We want to generate synthetic data that is consistent with a `Person` Pydantic BaseModel."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import guardrails as gd\n",
- "\n",
- "from rich import print"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Step 1: Create the RAIL Spec\n",
- "\n",
- "Ordinarily, we would create an RAIL spec in a separate file. For the purposes of this example, we will create the spec in this notebook as a string following the RAIL syntax. For more information on RAIL, see the [RAIL documentation](../rail/output.md).\n",
- "\n",
- "Here, we define a Pydantic model for a `Person` with the following fields:\n",
- "\n",
- "- `name`: a string \n",
- "- `age`: an integer \n",
- "- `zip_code`: a string zip code\n",
- "\n",
- "and write very simple validators for the fields as an example. As a way to show how LLM reasking can be used to generate data that is consistent with the Pydantic model, we can define a validator that asks for a zip code in California (including being perversely opposed to the \"90210\" zip code). If this validator fails, the LLM will be sent the error message and will reask the question.\n",
- "\n",
- "This Pydantic model could also be any model that you already have in your codebase, and just needs to be decorated with `@register_pydantic`.\n",
- "\n",
- "\n",
- "To use this model in the `` specification, we used the special\n",
- "`pydantic` tag. This tag takes the name of the Pydantic model, as well as the\n",
- "`on-fail-pydantic` attribute, which specifies what to do when the output\n",
- "does not validate against the Pydantic model."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "rail_str = \"\"\"\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- " \n",
- "
\n",
- " \n",
- "\n",
- "\n",
- "\n",
- "Generate data for possible users in accordance with the specification below.\n",
- "\n",
- "@xml_prefix_prompt\n",
- "\n",
- "{output_schema}\n",
- "\n",
- "@complete_json_suffix_v2 \n",
- "\n",
- " \n",
- "\"\"\""
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Step 2: Create a `Guard` object with the RAIL Spec\n",
- "\n",
- "We create a `gd.Guard` object that will check, validate and correct the output of the LLM. This object:\n",
- "\n",
- "1. Enforces the quality criteria specified in the RAIL spec.\n",
- "2. Takes corrective action when the quality criteria are not met.\n",
- "3. Compiles the schema and type info from the RAIL spec and adds it to the prompt."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "guard = gd.Guard.from_rail_string(rail_str)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We see the prompt that will be sent to the LLM."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "Generate data for possible users in accordance with the specification below.\n",
- "\n",
- "\n",
- "Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
- "\n",
- "\n",
- "< output > \n",
- " <list name = \"people\" description = \"A list of 3 people.\" > \n",
- " <object description = \"Information about a person.\" pydantic = \"Person\" ><string name = \"name\" description = \"The \n",
- "name of the person.\" / ><integer name = \"age\" description = \"The age of the person.\" \n",
- "format = \"age-must-be-between-0-and-150\" / ><string name = \"zip_code\" description = \"The zip code of the person.\" \n",
- "format = \"zip-code-must-be-numeric; zip-code-in-california\" / >< / object >< / list > \n",
- "< / output > \n",
- "\n",
- "\n",
- "Given below is XML that describes the information to extract from this document and the tags to extract it into. \n",
- "\n",
- "<output> \n",
- " <list name = \"people\" description = \"A list of 3 people.\" > \n",
- " <object description = \"Information about a person.\" pydantic = \"Person\" ><string name = \"name\" description = \"The \n",
- "name of the person.\" / ><integer name = \"age\" description = \"The age of the person.\" \n",
- "format = \"age-must-be-between-0-and-150\" / ><string name = \"zip_code\" description = \"The zip code of the person.\" \n",
- "format = \"zip-code-must-be-numeric; zip-code-in-california\" / >< / object >< / list > \n",
- "< / output > \n",
- "\n",
- "ONLY return a valid JSON object ( no other text is necessary ) , where the key of the field in JSON is the `name` \n",
- "attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON \n",
- "MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \n",
- "specific types. Be correct and concise. \n",
- "\n",
- "Here are examples of simple ( XML, JSON ) pairs that show the expected behavior: \n",
- "- `<string name = 'foo' format = 'two-words lower-case' / >` => ` {{ 'foo' : 'example one' }} ` \n",
- "- `<list name = 'bar' ><string format = 'upper-case' / >< / list >` => ` {{ \"bar\" : [ 'STRING ONE' , 'STRING TWO' , etc. ]}} ` \n",
- "- `<object name = 'baz' ><string name = \"foo\" format = \"capitalize two-words\" / ><integer name = \"index\" format = \"1-indexed\" \n",
- "/ >< / object >` = > `{{ 'baz' : {{ 'foo' : 'Some String' , 'index' : 1 }}}} `\n",
- "\n",
- "JSON Object:\n",
- " \n"
- ],
- "text/plain": [
- "\n",
- "Generate data for possible users in accordance with the specification below.\n",
- "\n",
- "\n",
- "Given below is XML that describes the information to extract from this document and the tags to extract it into.\n",
- "\n",
- "\n",
- "\u001b[1m<\u001b[0m\u001b[1;95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\n",
- "\n",
- "\u001b[39mGiven below is XML that describes the information to extract from this document and the tags to extract it into.\u001b[0m\n",
- "\n",
- "\u001b[39m\u001b[0m\n",
- "\u001b[39m \u001b[0m\n",
- "\u001b[39m <\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m><\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>\u001b[0m\n",
- "\u001b[39m<\u001b[0m\u001b[35m/\u001b[0m\u001b[95moutput\u001b[0m\u001b[39m>\u001b[0m\n",
- "\n",
- "\u001b[39mONLY return a valid JSON object \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mno other text is necessary\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m, where the key of the field in JSON is the `name` \u001b[0m\n",
- "\u001b[39mattribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON\u001b[0m\n",
- "\u001b[39mMUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and \u001b[0m\n",
- "\u001b[39mspecific types. Be correct and concise.\u001b[0m\n",
- "\n",
- "\u001b[39mHere are examples of simple \u001b[0m\u001b[1;39m(\u001b[0m\u001b[39mXML, JSON\u001b[0m\u001b[1;39m)\u001b[0m\u001b[39m pairs that show the expected behavior:\u001b[0m\n",
- "\u001b[39m- `` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m'foo'\u001b[0m\u001b[39m: \u001b[0m\u001b[32m'example one'\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mlist\u001b[0m\u001b[39m>` => `\u001b[0m\u001b[1;39m{\u001b[0m\u001b[1;39m{\u001b[0m\u001b[32m\"bar\"\u001b[0m\u001b[39m: \u001b[0m\u001b[1;39m[\u001b[0m\u001b[32m'STRING ONE'\u001b[0m\u001b[39m, \u001b[0m\u001b[32m'STRING TWO'\u001b[0m\u001b[39m, etc.\u001b[0m\u001b[1;39m]\u001b[0m\u001b[1;39m}\u001b[0m\u001b[1;39m}\u001b[0m\u001b[39m`\u001b[0m\n",
- "\u001b[39m- `<\u001b[0m\u001b[35m/\u001b[0m\u001b[95mobject\u001b[0m\u001b[39m>` =\u001b[0m\u001b[1m>\u001b[0m `\u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'baz'\u001b[0m: \u001b[1m{\u001b[0m\u001b[1m{\u001b[0m\u001b[32m'foo'\u001b[0m: \u001b[32m'Some String'\u001b[0m, \u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m`\n",
- "\n",
- "JSON Object:\n"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "print(guard.base_prompt)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "!!! note\n",
- " Notice that the prompt replaces the `pydantic` tag with the schema, validator and type information from the Pydantic model. This e.g. tells the LLM that we want that `zip-code-must-be-numeric` and `zip-code-in-california`. Guardrails will even automatically read the docstrings from the Pydantic model and add them to the prompt!"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Step 3: Wrap the LLM API call with `Guard`"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/krandiash/opt/anaconda3/envs/guardrails/lib/python3.9/site-packages/eliot/json.py:22: FutureWarning: In the future `np.bool` will be defined as the corresponding NumPy scalar.\n",
- " if isinstance(o, (numpy.bool, numpy.bool_)):\n"
- ]
- }
- ],
- "source": [
- "import openai\n",
- "\n",
- "raw_llm_response, validated_response = guard(\n",
- " openai.Completion.create,\n",
- " engine=\"text-davinci-003\",\n",
- " max_tokens=512,\n",
- " temperature=0.5,\n",
- " num_reasks=2,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "{ \n",
- " 'people' : [ \n",
- " Person ( name ='John Doe' , age =25 , zip_code ='90000' ) ,\n",
- " Person ( name ='Jane Doe' , age =30 , zip_code ='94105' ) ,\n",
- " Person ( name ='John Smith' , age =40 , zip_code ='90001' ) \n",
- " ] \n",
- "} \n",
- "GuardHistory ( \n",
- " history =[ \n",
- " GuardLogs ( \n",
- " prompt ='\\nGenerate data for possible users in accordance with the specification below.\\n\\n\\nGiven below \n",
- "is XML that describes the information to extract from this document and the tags to extract it \n",
- "into.\\n\\n\\n<output>\\n <list name=\"people\" description=\"A list of 3 people.\">\\n <object \n",
- "description=\"Information about a person.\" pydantic=\"Person\"><string name=\"name\" description=\"The name of the \n",
- "person.\"/><integer name=\"age\" description=\"The age of the person.\" format=\"age-must-be-between-0-and-150\"/><string \n",
- "name=\"zip_code\" description=\"The zip code of the person.\" format=\"zip-code-must-be-numeric; \n",
- "zip-code-in-california\"/></object></list>\\n</output>\\n\\n\\nGiven below is XML that describes the information to \n",
- "extract from this document and the tags to extract it into.\\n\\n<output>\\n <list name=\"people\" description=\"A \n",
- "list of 3 people.\">\\n <object description=\"Information about a person.\" pydantic=\"Person\"><string \n",
- "name=\"name\" description=\"The name of the person.\"/><integer name=\"age\" description=\"The age of the person.\" \n",
- "format=\"age-must-be-between-0-and-150\"/><string name=\"zip_code\" description=\"The zip code of the person.\" \n",
- "format=\"zip-code-must-be-numeric; zip-code-in-california\"/></object></list>\\n</output>\\n\\nONLY return a valid JSON \n",
- "object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the \n",
- "corresponding XML, and the value is of the type specified by the corresponding XML\\'s tag. The JSON MUST conform to \n",
- "the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be \n",
- "correct and concise.\\n\\nHere are examples of simple (XML, JSON) pairs that show the expected behavior:\\n- `<string \n",
- "name=\\'foo\\' format=\\'two-words lower-case\\' />` => `{\\'foo\\': \\'example one\\'}`\\n- `<list name=\\'bar\\'><string \n",
- "format=\\'upper-case\\' /></list>` => `{\"bar\": [\\'STRING ONE\\', \\'STRING TWO\\', etc.]}`\\n- `<object \n",
- "name=\\'baz\\'><string name=\"foo\" format=\"capitalize two-words\" /><integer name=\"index\" format=\"1-indexed\" \n",
- "/></object>` => `{\\'baz\\': {\\'foo\\': \\'Some String\\', \\'index\\': 1}}`\\n\\nJSON Object:' , \n",
- " output = ' \\n{\\n \"people\": [\\n {\\n \"name\": \"John Doe\",\\n \"age\": 25,\\n \n",
- "\"zip_code\": \"90210\"\\n },\\n {\\n \"name\": \"Jane Doe\",\\n \"age\": 30,\\n \n",
- "\"zip_code\": \"94105\"\\n },\\n {\\n \"name\": \"John Smith\",\\n \"age\": 40,\\n \n",
- "\"zip_code\": \"90001\"\\n }\\n ]\\n}' , \n",
- " output_as_dict = { \n",
- " 'people' : [ \n",
- " { 'name' : 'John Doe' , 'age' : 25 , 'zip_code' : '90210' } , \n",
- " { 'name' : 'Jane Doe' , 'age' : 30 , 'zip_code' : '94105' } , \n",
- " { 'name' : 'John Smith' , 'age' : 40 , 'zip_code' : '90001' } \n",
- " ] \n",
- " } , \n",
- " validated_output = { \n",
- " 'people' : [ \n",
- " { \n",
- " 'name' : 'John Doe' , \n",
- " 'age' : 25 , \n",
- " 'zip_code' : ReAsk ( \n",
- " incorrect_value = '90210' , \n",
- " error_message = 'Zip code must not be Beverly Hills.' , \n",
- " fix_value = None , \n",
- " path = [ 'people' , 0 ] \n",
- " ) \n",
- " } , \n",
- " Person ( name = 'Jane Doe' , age = 30 , zip_code = '94105' ) , \n",
- " Person ( name = 'John Smith' , age = 40 , zip_code = '90001' ) \n",
- " ] \n",
- " } , \n",
- " reasks = [ \n",
- " ReAsk ( \n",
- " incorrect_value = '90210' , \n",
- " error_message = 'Zip code must not be Beverly Hills.' , \n",
- " fix_value = None , \n",
- " path = [ 'people' , 0 ] \n",
- " ) \n",
- " ] \n",
- " ) , \n",
- " GuardLogs ( \n",
- " prompt = '\\nI was given the following JSON response, which had problems due to incorrect values.\\n\\n{\\n \n",
- "\"people\": [\\n {\\n \"name\": \"John Doe\",\\n \"age\": 25,\\n \"zip_code\": {\\n \"incorrect_value\": \n",
- "\"90210\",\\n \"error_message\": \"Zip code must not be Beverly Hills.\"\\n }\\n }\\n ]\\n}\\n\\nHelp me correct \n",
- "the incorrect values based on the given error messages.\\n\\nGiven below is XML that describes the information to \n",
- "extract from this document and the tags to extract it into.\\n\\n<output>\\n <list name=\"people\" description=\"A \n",
- "list of 3 people.\">\\n <object description=\"Information about a person.\" pydantic=\"Person\"><string \n",
- "name=\"name\" description=\"The name of the person.\"/><integer name=\"age\" description=\"The age of the person.\" \n",
- "format=\"age-must-be-between-0-and-150\"/><string name=\"zip_code\" description=\"The zip code of the person.\" \n",
- "format=\"zip-code-must-be-numeric; zip-code-in-california\"/></object></list>\\n</output>\\n\\nONLY return a valid JSON \n",
- "object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the \n",
- "corresponding XML, and the value is of the type specified by the corresponding XML\\'s tag. The JSON MUST conform to \n",
- "the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be \n",
- "correct and concise. If you are unsure anywhere, enter `null`.\\n\\nHere are examples of simple (XML, JSON) pairs \n",
- "that show the expected behavior:\\n- `<string name=\\'foo\\' format=\\'two-words lower-case\\' />` => `{{\\'foo\\': \n",
- "\\'example one\\'}}`\\n- `<list name=\\'bar\\'><string format=\\'upper-case\\' /></list>` => `{{\"bar\": [\\'STRING ONE\\', \n",
- "\\'STRING TWO\\', etc.]}}`\\n- `<object name=\\'baz\\'><string name=\"foo\" format=\"capitalize two-words\" /><integer \n",
- "name=\"index\" format=\"1-indexed\" /></object>` => `{{\\'baz\\': {{\\'foo\\': \\'Some String\\', \\'index\\': 1}}}}`\\n\\nJSON \n",
- "Object:' ,\n",
- " output ='\\n{\\n \"people\": [\\n {\\n \"name\": \"John Doe\",\\n \"age\": 25,\\n \n",
- "\"zip_code\": \"90000\"\\n }\\n ]\\n}' ,\n",
- " output_as_dict ={ 'people' : [{ 'name' : 'John Doe' , 'age' : 25 , 'zip_code' : '90000' }]} ,\n",
- " validated_output ={ \n",
- " 'people' : [ \n",
- " Person ( name ='John Doe' , age =25 , zip_code ='90000' ) ,\n",
- " Person ( name ='Jane Doe' , age =30 , zip_code ='94105' ) ,\n",
- " Person ( name ='John Smith' , age =40 , zip_code ='90001' ) \n",
- " ] \n",
- " } ,\n",
- " reasks =[] \n",
- " ) \n",
- " ] \n",
- ") \n",
- "\\n
\\n
\\n\\n\\nGiven below is XML that describes the information to \u001b[0m\n",
- "\u001b[32mextract from this document and the tags to extract it into.\\n\\n\\n \\n
\\n \\n\\nONLY return a valid JSON \u001b[0m\n",
- "\u001b[32mobject \u001b[0m\u001b[32m(\u001b[0m\u001b[32mno other text is necessary\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, where the key of the field in JSON is the `name` attribute of the \u001b[0m\n",
- "\u001b[32mcorresponding XML, and the value is of the type specified by the corresponding XML\\'s tag. The JSON MUST conform to\u001b[0m\n",
- "\u001b[32mthe XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be \u001b[0m\n",
- "\u001b[32mcorrect and concise.\\n\\nHere are examples of simple \u001b[0m\u001b[32m(\u001b[0m\u001b[32mXML, JSON\u001b[0m\u001b[32m)\u001b[0m\u001b[32m pairs that show the expected behavior:\\n- `
` => `\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"bar\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m\\'STRING ONE\\', \\'STRING TWO\\', etc.\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m`\\n- `\\n \\n
\\n \\n\\nONLY return a valid JSON \u001b[0m\n",
- "\u001b[32mobject \u001b[0m\u001b[32m(\u001b[0m\u001b[32mno other text is necessary\u001b[0m\u001b[32m)\u001b[0m\u001b[32m, where the key of the field in JSON is the `name` attribute of the \u001b[0m\n",
- "\u001b[32mcorresponding XML, and the value is of the type specified by the corresponding XML\\'s tag. The JSON MUST conform to\u001b[0m\n",
- "\u001b[32mthe XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be \u001b[0m\n",
- "\u001b[32mcorrect and concise. If you are unsure anywhere, enter `null`.\\n\\nHere are examples of simple \u001b[0m\u001b[32m(\u001b[0m\u001b[32mXML, JSON\u001b[0m\u001b[32m)\u001b[0m\u001b[32m pairs \u001b[0m\n",
- "\u001b[32mthat show the expected behavior:\\n- `
` => `\u001b[0m\u001b[32m{\u001b[0m\u001b[32m{\u001b[0m\u001b[32m\"bar\": \u001b[0m\u001b[32m[\u001b[0m\u001b[32m\\'STRING ONE\\', \u001b[0m\n",
- "\u001b[32m\\'STRING TWO\\', etc.\u001b[0m\u001b[32m]\u001b[0m\u001b[32m}\u001b[0m\u001b[32m}\u001b[0m\u001b[32m`\\n- `
-
-
-
-
-```
-
-## 🧭 Adding a custom `DataType`
-
-Coming soon!
diff --git a/docs/recipe/generating_structured_data.ipynb b/docs/recipe/generating_structured_data.ipynb
new file mode 100644
index 000000000..984c5d8b1
--- /dev/null
+++ b/docs/recipe/generating_structured_data.ipynb
@@ -0,0 +1,21 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Hi"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/recipe/validating_string_data.ipynb b/docs/recipe/validating_string_data.ipynb
new file mode 100644
index 000000000..984c5d8b1
--- /dev/null
+++ b/docs/recipe/validating_string_data.ipynb
@@ -0,0 +1,21 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "Hi"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ },
+ "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/guardrails/applications/text2sql.py b/guardrails/applications/text2sql.py
index b478acd19..3ab788baf 100644
--- a/guardrails/applications/text2sql.py
+++ b/guardrails/applications/text2sql.py
@@ -15,24 +15,24 @@
REASK_PROMPT = """
You are a data scientist whose job is to write SQL queries.
-@complete_json_suffix_v2
+${gr.complete_json_suffix_v2}
Here's schema about the database that you can use to generate the SQL query.
Try to avoid using joins if the data can be retrieved from the same table.
-{{db_info}}
+${db_info}
I will give you a list of examples.
-{{examples}}
+${examples}
I want to create a query for the following instruction:
-{{nl_instruction}}
+${nl_instruction}
For this instruction, I was given the following JSON, which has some incorrect values.
-{previous_response}
+${previous_response}
Help me correct the incorrect values based on the given error messages.
"""
diff --git a/guardrails/applications/text2sql.rail b/guardrails/applications/text2sql.rail
index 5bfd8edf0..27732b5df 100644
--- a/guardrails/applications/text2sql.rail
+++ b/guardrails/applications/text2sql.rail
@@ -12,7 +12,7 @@
You are a data scientist whose job is to write SQL queries.
-@complete_json_suffix_v2
+${gr.complete_json_suffix_v2}
@@ -20,15 +20,15 @@ You are a data scientist whose job is to write SQL queries.
Here's schema about the database that you can use to generate the SQL query.
Try to avoid using joins if the data can be retrieved from the same table.
-{{db_info}}
+${db_info}
I will give you a list of examples. Write a SQL query similar to the examples below:
-{{examples}}
+${examples}
INSTRUCTIONS:
---------
-{{nl_instruction}}
+${nl_instruction}
QUERY:
---------
diff --git a/guardrails/constants.xml b/guardrails/constants.xml
index b4dc26d13..19f3b4a9a 100644
--- a/guardrails/constants.xml
+++ b/guardrails/constants.xml
@@ -23,7 +23,7 @@ ONLY return a valid JSON object (no other text is necessary). The JSON MUST conf
I was given the following JSON response, which had problems due to incorrect values.
-{previous_response}
+${previous_response}
Help me correct the incorrect values based on the given error messages.
@@ -31,7 +31,7 @@ Help me correct the incorrect values based on the given error messages.
I was given the following JSON response, which had problems due to incorrect values.
-{previous_response}
+${previous_response}
Help me correct the incorrect values based on the given error messages.
@@ -39,7 +39,7 @@ Help me correct the incorrect values based on the given error messages.
Given below is XML that describes the information to extract from this document and the tags to extract it into.
-{output_schema}
+${output_schema}
ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.
@@ -47,41 +47,41 @@ ONLY return a valid JSON object (no other text is necessary), where the key of t
Given below is XML that describes the information to extract from this document and the tags to extract it into.
-{output_schema}
+${output_schema}
ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.
Here are examples of simple (XML, JSON) pairs that show the expected behavior:
-- ``]]> => `{{{{'foo': 'example one'}}}}`
-- `
]]>` => `{{{{"bar": ['STRING ONE', 'STRING TWO', etc.]}}}}`
-- `
]]>` => `{"bar": ['STRING ONE', 'STRING TWO', etc.]}`
+- `