From b57edaaf6a96be0370e1e3d564b1359eeca09de0 Mon Sep 17 00:00:00 2001 From: Jason Liu Date: Sun, 22 Oct 2023 19:13:16 -0400 Subject: [PATCH] update --- examples/distilations/math_finetunes.jsonl | 20 ++++---- examples/distilations/three_digit_mul.py | 43 +++++++--------- .../distilations/three_digit_mul_dispatch.py | 49 +++++++++++++++++++ instructor/distil.py | 7 ++- 4 files changed, 80 insertions(+), 39 deletions(-) create mode 100644 examples/distilations/three_digit_mul_dispatch.py diff --git a/examples/distilations/math_finetunes.jsonl b/examples/distilations/math_finetunes.jsonl index 64961a39e..2fc676ca9 100644 --- a/examples/distilations/math_finetunes.jsonl +++ b/examples/distilations/math_finetunes.jsonl @@ -1,10 +1,10 @@ -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(540, b=677, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 540,\n \"b\": 677,\n \"result\": 1217\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(798, b=534, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 798,\n \"b\": 534,\n \"result\": 1332\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(608, b=669, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 608,\n \"b\": 669,\n \"result\": 1277\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(982, b=768, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 982,\n \"b\": 768,\n \"result\": 1750\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(994, b=682, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 994,\n \"b\": 682,\n \"result\": 1676\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(467, b=754, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 467,\n \"b\": 754,\n \"result\": 1221\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(497, b=364, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 497,\n \"b\": 364,\n \"result\": 861\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(840, b=821, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 840,\n \"b\": 821,\n \"result\": 1661\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(646, b=835, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 646,\n \"b\": 835,\n \"result\": 1481\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} -{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Multiply\n\"\"\"\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n\"\"\""}, {"role": "user", "content": "Return `fn(926, b=196, c=\"hello\")`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 926,\n \"b\": 196,\n \"result\": 1122\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"type": "integer"}, "b": {"type": "integer"}, "result": {"description": "The result of the multiplication", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(286, b=521)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 286,\n \"b\": 521,\n \"result\": 149006\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(809, b=358)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 809,\n \"b\": 358,\n \"result\": 289622\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(634, b=574)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 634,\n \"b\": 574,\n \"result\": 363916\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(736, b=276)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 736,\n \"b\": 276,\n \"result\": 203136\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(143, b=591)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 143,\n \"b\": 591,\n \"result\": 84513\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(929, b=123)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 929,\n \"b\": 123,\n \"result\": 114267\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(342, b=800)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 342,\n \"b\": 800,\n \"result\": 273600\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(887, b=718)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 887,\n \"b\": 718,\n \"result\": 636866\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(250, b=694)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 250,\n \"b\": 694,\n \"result\": 173500\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} +{"messages": [{"role": "system", "content": "Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n\"\"\"\nReturn the result of multiplying a and b together\n\"\"\""}, {"role": "user", "content": "Return `fn(731, b=622)`"}, {"role": "assistant", "function_call": {"name": "Multiply", "arguments": "{\n \"a\": 731,\n \"b\": 622,\n \"result\": 454682\n}"}}], "functions": [{"name": "Multiply", "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": {"properties": {"a": {"title": "A", "type": "integer"}, "b": {"title": "B", "type": "integer"}, "result": {"description": "The result of the multiplication", "title": "Result", "type": "integer"}}, "required": ["a", "b", "result"], "type": "object"}}]} diff --git a/examples/distilations/three_digit_mul.py b/examples/distilations/three_digit_mul.py index 74ccd690d..9c521ee09 100644 --- a/examples/distilations/three_digit_mul.py +++ b/examples/distilations/three_digit_mul.py @@ -22,49 +22,43 @@ class Multiply(BaseModel): @instructions.distil -def fn(a: int, b: int, c: str) -> Multiply: - """_summary_ - - Args: - a (int): _description_ - b (int): _description_ - c (str): _description_ - - Returns: - Response: _description_ - """ - resp = a + b +def fn(a: int, b: int) -> Multiply: + """Return the result of multiplying a and b together""" + resp = a * b return Multiply(a=a, b=b, result=resp) if __name__ == "__main__": import random - # A log will look like this: - log_line = { + log_lines = { "messages": [ { "role": "system", - "content": 'Predict the results of this function:\n\ndef fn(a: int, b: int, c: str) -> __main__.Response\n"""\n_summary_\n\nArgs:\n a (int): _description_\n b (int): _description_\n c (str): _description_\n\nReturns:\n Response: _description_\n"""', + "content": 'Predict the results of this function:\n\ndef fn(a: int, b: int) -> __main__.Multiply\n"""\nReturn the result of multiplying a and b together\n"""', }, - {"role": "user", "content": 'Return fn(133, b=539, c="hello")'}, + {"role": "user", "content": "Return `fn(169, b=166)`"}, { "role": "assistant", "function_call": { - "name": "Response", - "arguments": '{"a":133,"b":539,"result":672}', + "name": "Multiply", + "arguments": '{\n "a": 169,\n "b": 166,\n "result": 28054\n}', }, }, ], "functions": [ { - "name": "Response", - "description": "Correctly extracted `Response` with all the required parameters with correct types", + "name": "Multiply", + "description": "Correctly extracted `Multiply` with all the required parameters with correct types", "parameters": { "properties": { - "a": {"type": "integer"}, - "b": {"type": "integer"}, - "result": {"type": "integer"}, + "a": {"title": "A", "type": "integer"}, + "b": {"title": "B", "type": "integer"}, + "result": { + "description": "The result of the multiplication", + "title": "Result", + "type": "integer", + }, }, "required": ["a", "b", "result"], "type": "object", @@ -72,8 +66,7 @@ def fn(a: int, b: int, c: str) -> Multiply: } ], } - for _ in range(10): a = random.randint(100, 999) b = random.randint(100, 999) - print("returning", fn(a, b=b, c="hello")) + print("returning", fn(a, b=b)) diff --git a/examples/distilations/three_digit_mul_dispatch.py b/examples/distilations/three_digit_mul_dispatch.py new file mode 100644 index 000000000..1feb611ad --- /dev/null +++ b/examples/distilations/three_digit_mul_dispatch.py @@ -0,0 +1,49 @@ +import logging + +from pydantic import BaseModel, Field +from instructor import Instructions +import instructor + +instructor.patch() + +logging.basicConfig(level=logging.INFO) + +# Usage +instructions = Instructions( + name="three_digit_multiply", + finetune_format="messages", + include_code_body=True, + log_handlers=[ + logging.FileHandler("math_finetunes.jsonl"), + ], +) + + +class Multiply(BaseModel): + a: int + b: int + result: int = Field(..., description="The result of the multiplication") + + +@instructions.distil(mode="dispatch", model="ft:gpt-3.5-turbo-0613:personal::8CazU0uq") +def fn(a: int, b: int) -> Multiply: + """Return the result of the multiplication as an integer""" + resp = a * b + return Multiply(a=a, b=b, result=resp) + + +if __name__ == "__main__": + import random + + for _ in range(5): + a = random.randint(100, 999) + b = random.randint(100, 999) + result = fn(a, b) + print(f"{a} * {b} = {result.result}, expected {a*b}") + """ + 972 * 508 = 493056, expected 493776 + 145 * 369 = 53505, expected 53505 + 940 * 440 = 413600, expected 413600 + 114 * 213 = 24282, expected 24282 + 259 * 650 = 168350, expected 168350 + """ diff --git a/instructor/distil.py b/instructor/distil.py index 3ea844f13..bc4881f22 100644 --- a/instructor/distil.py +++ b/instructor/distil.py @@ -113,6 +113,7 @@ def distil( *args, name: str = None, mode: str = "distil", + model: str = "gpt-3.5-turbo", fine_tune_format: FinetuneFormat = None, ): """ @@ -136,7 +137,6 @@ def distil( """ allowed_modes = {"distil", "dispatch"} assert mode in allowed_modes, f"Must be in {allowed_modes}" - assert mode == "distil", "Only distil mode is supported at the moment." if fine_tune_format is None: fine_tune_format = self.finetune_format @@ -156,7 +156,7 @@ def _dispatch(*args, **kwargs): base_model=return_base_model, ) return openai.ChatCompletion.create( - **openai_kwargs, response_model=return_base_model + **openai_kwargs, model=model, response_model=return_base_model ) @functools.wraps(fn) @@ -215,7 +215,6 @@ def track( } ) openai_kwargs["functions"] = [openai_function_call] - openai_kwargs["function_call"] = {"name": openai_function_call["name"]} self.logger.info(json.dumps(openai_kwargs)) if finetune_format == FinetuneFormat.RAW: @@ -233,7 +232,7 @@ def openai_kwargs(self, name, fn, args, kwargs, base_model): if self.include_code_body: func_def = format_function(fn) else: - func_def = get_signature_from_fn(fn).replace(fn.__name__, name) + func_def = get_signature_from_fn(fn) str_args = ", ".join(map(str, args)) str_kwargs = (