From 5951c0cb3fcb0ca29f4220794abb97dfee92aa60 Mon Sep 17 00:00:00 2001 From: Rohit Prasad Date: Wed, 25 Sep 2024 21:08:54 -0700 Subject: [PATCH] Update README and some examples. --- examples/client.ipynb | 38 +- examples/llm_reasoning.ipynb | 921 ++--------------------------------- 2 files changed, 53 insertions(+), 906 deletions(-) diff --git a/examples/client.ipynb b/examples/client.ipynb index 9294f6fe..c721d7bf 100644 --- a/examples/client.ipynb +++ b/examples/client.ipynb @@ -11,7 +11,7 @@ "source": [ "# Client Examples\n", "\n", - "Client provides a uniform interface for interacting with LLMs from various providers. It adapts the official python libraries from providers such as Mistral, OpenAI, Groq, Anthropic, Fireworks, Replicate, etc. to conform to the OpenAI chat completion interface.\n", + "Client provides a uniform interface for interacting with LLMs from various providers. It adapts the official python libraries from providers such as Mistral, OpenAI, Groq, Anthropic, AWS, etc to conform to the OpenAI chat completion interface. It directly calls the REST endpoints in some cases.\n", "\n", "Below are some examples of how to use Client to interact with different LLMs." ] @@ -55,15 +55,12 @@ " for key, value in additional_env_vars.items():\n", " os.environ[key] = value\n", "\n", - "# Define additional API keys and AWS credentials\n", + "# Define additional API keys and credentials\n", "additional_keys = {\n", " 'GROQ_API_KEY': 'xxx',\n", - " 'FIREWORKS_API_KEY': 'xxx', \n", - " 'REPLICATE_API_KEY': 'xxx', \n", - " 'TOGETHER_API_KEY': 'xxx', \n", - " 'OCTO_API_KEY': 'xxx',\n", " 'AWS_ACCESS_KEY_ID': 'xxx',\n", " 'AWS_SECRET_ACCESS_KEY': 'xxx',\n", + " 'ANTHROPIC_API_KEY': 'xxx',\n", "}\n", "\n", "# Configure environment\n", @@ -111,9 +108,6 @@ "metadata": {}, "outputs": [], "source": [ - "# print(os.environ['AWS_SECRET_ACCESS_KEY'])\n", - "# print(os.environ['AWS_ACCESS_KEY_ID'])\n", - "# print(os.environ['AWS_REGION'])\n", "aws_bedrock_llama3_8b = \"aws-bedrock:meta.llama3-1-8b-instruct-v1:0\"\n", "response = client.chat.completions.create(model=aws_bedrock_llama3_8b, messages=messages)\n", "print(response.choices[0].message.content)" @@ -126,15 +120,16 @@ "metadata": {}, "outputs": [], "source": [ - "# client2 = ai.Client({\"azure\" : {\n", - "# \"api_key\": os.environ[\"AZURE_API_KEY\"],\n", - "# }});\n", + "# IMP NOTE: Azure expects model endpoint to be passed in the format of \"azure:\".\n", + "# The model name is the deployment name in Project/Deployments.\n", + "# In the exmaple below, the model is \"mistral-large-2407\", but the name given to the\n", + "# deployment is \"aisuite-mistral-large-2407\" under the deployments section in Azure.\n", "client2 = ai.Client()\n", "client2.configure({\"azure\" : {\n", " \"api_key\": os.environ[\"AZURE_API_KEY\"],\n", - " \"base_url\": \"https://mistral-large-2407.westus3.models.ai.azure.com/v1/\",\n", + " \"base_url\": \"https://aisuite-mistral-large-2407.westus3.models.ai.azure.com/v1/\",\n", "}});\n", - "azure_model = \"azure:mistral-large-2407\"\n", + "azure_model = \"azure:aisuite-mistral-large-2407\"\n", "response = client2.chat.completions.create(model=azure_model, messages=messages)\n", "print(response.choices[0].message.content)" ] @@ -146,6 +141,10 @@ "metadata": {}, "outputs": [], "source": [ + "# HuggingFace expects the model to be passed in the format of \"huggingface:\".\n", + "# The model name is the full name of the model in HuggingFace.\n", + "# In the exmaple below, the model is \"mistralai/Mistral-7B-Instruct-v0.3\".\n", + "# The model is deployed as serverless inference endpoint in HuggingFace.\n", "client3 = ai.Client()\n", "hf_model = \"huggingface:mistralai/Mistral-7B-Instruct-v0.3\"\n", "response = client3.chat.completions.create(model=hf_model, messages=messages)\n", @@ -159,11 +158,14 @@ "metadata": {}, "outputs": [], "source": [ + "\n", + "# Groq expects the model to be passed in the format of \"groq:\".\n", + "# The model name is the full name of the model in Groq.\n", + "# In the exmaple below, the model is \"llama3-8b-8192\".\n", "groq_llama3_8b = \"groq:llama3-8b-8192\"\n", "# groq_llama3_70b = \"groq:llama3-70b-8192\"\n", "\n", "response = client.chat.completions.create(model=groq_llama3_8b, messages=messages)\n", - "\n", "print(response.choices[0].message.content)" ] }, @@ -193,9 +195,7 @@ "outputs": [], "source": [ "mistral_7b = \"mistral:open-mistral-7b\"\n", - "\n", "response = client.chat.completions.create(model=mistral_7b, messages=messages, temperature=0.2)\n", - "\n", "print(response.choices[0].message.content)" ] }, @@ -207,9 +207,7 @@ "outputs": [], "source": [ "openai_gpt35 = \"openai:gpt-3.5-turbo\"\n", - "\n", "response = client.chat.completions.create(model=openai_gpt35, messages=messages, temperature=0.75)\n", - "\n", "print(response.choices[0].message.content)" ] } @@ -230,7 +228,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.4" + "version": "3.12.6" } }, "nbformat": 4, diff --git a/examples/llm_reasoning.ipynb b/examples/llm_reasoning.ipynb index 5aa834b1..42605374 100644 --- a/examples/llm_reasoning.ipynb +++ b/examples/llm_reasoning.ipynb @@ -7,37 +7,19 @@ "source": [ "# LLM Reasoning\n", "\n", - "This notebook compares how LLMs (mainly Llama 3 and 3.1, but other LLMs can be added easily) from different Generative AI providers perform on three recent examples that show issues with LLM reasoning:\n", + "This notebook compares how LLMs from different Generative AI providers perform on three examples that can show issues with LLM reasoning:\n", "\n", "* [The Reversal Curse](https://github.com/lukasberglund/reversal_curse) shows that LLMs trained on \"A is B\" fail to learn \"B is A\".\n", "* [How many r's in the word strawberry?](https://x.com/karpathy/status/1816637781659254908) shows \"the weirdness of LLM Tokenization\". \n", - "* [Which number is bigger, 9.11 or 9.9?](https://x.com/DrJimFan/status/1816521330298356181) shows that \"LLMs are alien beasts.\"\n" + "* [Which number is bigger, 9.11 or 9.9?](https://x.com/DrJimFan/status/1816521330298356181) shows that \"LLMs are alien beasts.\"" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "d2e413bd-983c-42a0-9580-96fedc7b1275", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ANTHROPIC_API_KEY=\"\"\n", - "FIREWORKS_API_KEY=\"\"\n", - "GROQ_API_KEY=\"\"\n", - "MISTRAL_API_KEY=\"\"\n", - "OPENAI_API_KEY=\"\"\n", - "OLLAMA_API_URL=\"http://localhost:11434\"\n", - "REPLICATE_API_KEY=\"\"\n", - "TOGETHER_API_KEY=\"\"\n", - "OCTO_API_KEY=\"\"\n", - "AWS_ACCESS_KEY_ID=\"\"\n", - "AWS_SECRET_ACCESS_KEY=\"\"" - ] - } - ], + "outputs": [], "source": [ "!cat ../.env.sample" ] @@ -52,21 +34,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "3c966895-1a63-4922-80b7-5a20e47f29de", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import sys\n", "sys.path.append('../../aisuite')\n", @@ -76,17 +47,6 @@ "load_dotenv(find_dotenv())" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "31e093e4-31b6-4df0-9121-d1dceaa39cb4", - "metadata": {}, - "outputs": [], - "source": [ - "#!pip install boto3\n", - "#!pip install fireworks-ai" - ] - }, { "cell_type": "markdown", "id": "09d5c5be-1085-4252-9d5e-80b50961484b", @@ -97,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "26c3d5ef-b1c9-48dd-9b89-30799fd4b698", "metadata": {}, "outputs": [], @@ -109,26 +69,20 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "886a904f-fef0-4f25-b3ed-41085bf0f2dd", "metadata": {}, "outputs": [], "source": [ "import time\n", "\n", - "llms = [\"aws:meta.llama3-8b-instruct-v1:0\",\n", + "llms = [\n", + " \"anthropic:claude-3-5-sonnet-20240620\",\n", + " \"aws:meta.llama3-1-8b-instruct-v1:0\",\n", " \"groq:llama3-8b-8192\",\n", - " \"fireworks:accounts/fireworks/models/llama-v3-8b-instruct\",\n", - " \"octo:meta-llama-3-8b-instruct\",\n", - " \"together:meta-llama/Llama-3-8b-chat-hf\",\n", + " \"groq:llama3-70b-8192\",\n", + " \"huggingface:mistralai/Mistral-7B-Instruct-v0.3\",\n", " \"openai:gpt-3.5-turbo\",\n", - " \"replicate:meta/meta-llama-3-8b-instruct\",\n", - "\n", - " \"aws:meta.llama3-1-8b-instruct-v1:0\",\n", - " \"groq:llama-3.1-8b-instant\",\n", - " \"fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct\",\n", - " \"together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo\",\n", - " \"octo:meta-llama-3.1-8b-instruct\",\n", " ]\n", "\n", "def compare_llm(messages):\n", @@ -155,29 +109,10 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "f3c4a8ef-e23b-4d4a-8561-3e5a2a866bd1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws:meta.llama3-8b-instruct-v1:0 - 2.38 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer).\n", - "groq:llama3-8b-8192 - 2.24 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer).\n", - "fireworks:accounts/fireworks/models/llama-v3-8b-instruct - 0.92 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer). She was a special education teacher and a social worker.\n", - "octo:meta-llama-3-8b-instruct - 1.82 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer). She was a special education teacher and a homemaker.\n", - "together:meta-llama/Llama-3-8b-chat-hf - 0.61 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer).\n", - "openai:gpt-3.5-turbo - 1.00 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n", - "replicate:meta/meta-llama-3-8b-instruct - 1.36 seconds: Tom Cruise's mother is Mary Lee South (née Pfeiffer).\n", - "aws:meta.llama3-1-8b-instruct-v1:0 - 0.45 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n", - "groq:llama-3.1-8b-instant - 0.84 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n", - "fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct - 0.37 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n", - "together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - 0.26 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n", - "octo:meta-llama-3.1-8b-instruct - 0.32 seconds: Tom Cruise's mother is Mary Lee Pfeiffer.\n" - ] - } - ], + "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Who is Tom Cruise's mother?\"},\n", @@ -188,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "769f7f42-2adb-4903-ab17-3143a5d950ce", "metadata": {}, "outputs": [], @@ -214,147 +149,20 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "d2359ad5-9f0b-4bd6-9838-54df91de0fb3", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 Provider:Model NameExecution TimeModel Response
1aws:meta.llama3-8b-instruct-v1:02.383425Tom Cruise's mother is Mary Lee South (née Pfeiffer).
2groq:llama3-8b-81922.241169Tom Cruise's mother is Mary Lee South (née Pfeiffer).
3fireworks:accounts/fireworks/models/llama-v3-8b-instruct0.916995Tom Cruise's mother is Mary Lee South (née Pfeiffer). She was a special education teacher and a social worker.
4octo:meta-llama-3-8b-instruct1.822360Tom Cruise's mother is Mary Lee South (née Pfeiffer). She was a special education teacher and a homemaker.
5together:meta-llama/Llama-3-8b-chat-hf0.607085Tom Cruise's mother is Mary Lee South (née Pfeiffer).
6openai:gpt-3.5-turbo1.002106Tom Cruise's mother is Mary Lee Pfeiffer.
7replicate:meta/meta-llama-3-8b-instruct1.362718Tom Cruise's mother is Mary Lee South (née Pfeiffer).
8aws:meta.llama3-1-8b-instruct-v1:00.454378Tom Cruise's mother is Mary Lee Pfeiffer.
9groq:llama-3.1-8b-instant0.835516Tom Cruise's mother is Mary Lee Pfeiffer.
10fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct0.371963Tom Cruise's mother is Mary Lee Pfeiffer.
11together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo0.264464Tom Cruise's mother is Mary Lee Pfeiffer.
12octo:meta-llama-3.1-8b-instruct0.315450Tom Cruise's mother is Mary Lee Pfeiffer.
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "display(llms, execution_times, responses)" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "399f6cca-7f34-4a91-aab0-070560640033", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws:meta.llama3-8b-instruct-v1:0 - 1.23 seconds: I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private individual and not a public figure, or that the name is not well-known. Can you provide more context or details about who Mary Lee Pfeiffer is or why you are looking for information about her son?\n", - "groq:llama3-8b-8192 - 0.39 seconds: I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?\n", - "fireworks:accounts/fireworks/models/llama-v3-8b-instruct - 0.44 seconds: According to my knowledge, Mary Lee Pfeiffer's son is John Pfeiffer.\n", - "octo:meta-llama-3-8b-instruct - 1.25 seconds: I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?\n", - "together:meta-llama/Llama-3-8b-chat-hf - 0.92 seconds: I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?\n", - "openai:gpt-3.5-turbo - 0.64 seconds: Mary Lee Pfeiffer's son is actor and filmmaker Joaquin Phoenix.\n", - "replicate:meta/meta-llama-3-8b-instruct - 1.38 seconds: According to my knowledge, Mary Lee Pfeiffer's son is John Pfeiffer.\n", - "aws:meta.llama3-1-8b-instruct-v1:0 - 0.64 seconds: I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?\n", - "groq:llama-3.1-8b-instant - 1.06 seconds: I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?\n", - "fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct - 0.39 seconds: I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?\n", - "together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - 0.79 seconds: I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private individual and not a public figure, or that the name is not well-known. If you could provide more context or clarify who Mary Lee Pfeiffer is, I may be able to help you better.\n", - "octo:meta-llama-3.1-8b-instruct - 0.76 seconds: I am unable to verify who Mary Lee Pfeiffer's son is.\n" - ] - } - ], + "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Who is Mary Lee Pfeiffer's son?\"},\n", @@ -365,118 +173,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "eee7704d-a187-41bc-b119-c94461d0ee74", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 Provider:Model NameExecution TimeModel Response
1aws:meta.llama3-8b-instruct-v1:01.225959I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private individual and not a public figure, or that the name is not well-known. Can you provide more context or details about who Mary Lee Pfeiffer is or why you are looking for information about her son?
2groq:llama3-8b-81920.391800I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?
3fireworks:accounts/fireworks/models/llama-v3-8b-instruct0.438607According to my knowledge, Mary Lee Pfeiffer's son is John Pfeiffer.
4octo:meta-llama-3-8b-instruct1.250298I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?
5together:meta-llama/Llama-3-8b-chat-hf0.924522I apologize, but I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private or personal matter, or that the person is not a public figure. Can you provide more context or clarify who Mary Lee Pfeiffer is?
6openai:gpt-3.5-turbo0.637278Mary Lee Pfeiffer's son is actor and filmmaker Joaquin Phoenix.
7replicate:meta/meta-llama-3-8b-instruct1.375630According to my knowledge, Mary Lee Pfeiffer's son is John Pfeiffer.
8aws:meta.llama3-1-8b-instruct-v1:00.639018I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?
9groq:llama-3.1-8b-instant1.059837I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?
10fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct0.387835I don't have information on Mary Lee Pfeiffer's son. Is there something else I can help you with?
11together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo0.788913I couldn't find any information on a person named Mary Lee Pfeiffer or her son. It's possible that this is a private individual and not a public figure, or that the name is not well-known. If you could provide more context or clarify who Mary Lee Pfeiffer is, I may be able to help you better.
12octo:meta-llama-3.1-8b-instruct0.763321I am unable to verify who Mary Lee Pfeiffer's son is.
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "display(llms, execution_times, responses)" ] @@ -491,31 +191,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "e537871e-68b6-44c3-886a-d3ebe7a692c1", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws:meta.llama3-8b-instruct-v1:0 - 0.48 seconds: There are 2 R's in the word \"strawberry\".\n", - "groq:llama3-8b-8192 - 0.16 seconds: There are 2 R's in the word \"strawberry\".\n", - "fireworks:accounts/fireworks/models/llama-v3-8b-instruct - 0.25 seconds: There are 2 R's in the word \"strawberry\".\n", - "octo:meta-llama-3-8b-instruct - 0.31 seconds: There are 2 R's in the word \"strawberry\".\n", - "together:meta-llama/Llama-3-8b-chat-hf - 0.25 seconds: There are 2 R's in the word \"strawberry\".\n", - "openai:gpt-3.5-turbo - 0.90 seconds: There are three r's in the word \"strawberry.\"\n", - "replicate:meta/meta-llama-3-8b-instruct - 1.33 seconds: Let me count them for you!\n", - "\n", - "There are 2 R's in the word \"strawberry\".\n", - "aws:meta.llama3-1-8b-instruct-v1:0 - 0.49 seconds: There are 3 r's in the word \"strawberry\".\n", - "groq:llama-3.1-8b-instant - 2.36 seconds: There are 3 r's in the word \"strawberry\".\n", - "fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct - 0.43 seconds: There are 3 r's in the word \"strawberry\".\n", - "together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - 0.33 seconds: There are 3 r's in the word \"strawberry\".\n", - "octo:meta-llama-3.1-8b-instruct - 0.59 seconds: There are 2 r's in the word \"strawberry\".\n" - ] - } - ], + "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"How many r's in the word strawberry?\"},\n", @@ -526,120 +205,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "5678e393-4967-49f1-9e0f-251471dc92b7", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 Provider:Model NameExecution TimeModel Response
1aws:meta.llama3-8b-instruct-v1:00.480391There are 2 R's in the word \"strawberry\".
2groq:llama3-8b-81920.159436There are 2 R's in the word \"strawberry\".
3fireworks:accounts/fireworks/models/llama-v3-8b-instruct0.254061There are 2 R's in the word \"strawberry\".
4octo:meta-llama-3-8b-instruct0.314966There are 2 R's in the word \"strawberry\".
5together:meta-llama/Llama-3-8b-chat-hf0.248981There are 2 R's in the word \"strawberry\".
6openai:gpt-3.5-turbo0.899374There are three r's in the word \"strawberry.\"
7replicate:meta/meta-llama-3-8b-instruct1.328329Let me count them for you!\n", - "\n", - "There are 2 R's in the word \"strawberry\".
8aws:meta.llama3-1-8b-instruct-v1:00.494379There are 3 r's in the word \"strawberry\".
9groq:llama-3.1-8b-instant2.364020There are 3 r's in the word \"strawberry\".
10fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct0.434086There are 3 r's in the word \"strawberry\".
11together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo0.327454There are 3 r's in the word \"strawberry\".
12octo:meta-llama-3.1-8b-instruct0.592822There are 2 r's in the word \"strawberry\".
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "display(llms, execution_times, responses)" ] @@ -654,35 +223,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "efdf2fd6-f63a-4f9b-af15-1df25590e4fc", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws:meta.llama3-8b-instruct-v1:0 - 0.49 seconds: 9.9 is bigger than 9.11.\n", - "groq:llama3-8b-8192 - 0.20 seconds: 9.11 is bigger than 9.9.\n", - "fireworks:accounts/fireworks/models/llama-v3-8b-instruct - 0.27 seconds: 9.9 is bigger than 9.11.\n", - "octo:meta-llama-3-8b-instruct - 0.29 seconds: 9.11 is bigger than 9.9.\n", - "together:meta-llama/Llama-3-8b-chat-hf - 0.70 seconds: 9.11 is bigger than 9.9.\n", - "openai:gpt-3.5-turbo - 1.05 seconds: 9.9\n", - "replicate:meta/meta-llama-3-8b-instruct - 1.58 seconds: Let me help you with that!\n", - "\n", - "9.11 is bigger than 9.9.\n", - "aws:meta.llama3-1-8b-instruct-v1:0 - 0.83 seconds: To compare these two numbers, we need to look at the decimal part. Since 9.11 has a larger decimal part (0.11) than 9.9 (0.9), 9.11 is bigger.\n", - "groq:llama-3.1-8b-instant - 0.23 seconds: 9.9 is bigger than 9.11.\n", - "fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct - 0.19 seconds: 9.9 is bigger than 9.11.\n", - "together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - 0.36 seconds: 9.9 is bigger than 9.11.\n", - "octo:meta-llama-3.1-8b-instruct - 0.91 seconds: To compare these two numbers, we need to look at the decimal part. \n", - "\n", - "9.11 has a decimal part of 0.11, and 9.9 has a decimal part of 0.9. \n", - "\n", - "Since 0.11 is greater than 0.9, 9.11 is bigger than 9.9.\n" - ] - } - ], + "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Which number is bigger, 9.11 or 9.9?\"},\n", @@ -693,230 +237,20 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "eaa14ed1-c83b-4c8f-bb14-d318bf0c9a60", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 Provider:Model NameExecution TimeModel Response
1aws:meta.llama3-8b-instruct-v1:00.4892799.9 is bigger than 9.11.
2groq:llama3-8b-81920.2008649.11 is bigger than 9.9.
3fireworks:accounts/fireworks/models/llama-v3-8b-instruct0.2716259.9 is bigger than 9.11.
4octo:meta-llama-3-8b-instruct0.2949589.11 is bigger than 9.9.
5together:meta-llama/Llama-3-8b-chat-hf0.6956579.11 is bigger than 9.9.
6openai:gpt-3.5-turbo1.0515959.9
7replicate:meta/meta-llama-3-8b-instruct1.580146Let me help you with that!\n", - "\n", - "9.11 is bigger than 9.9.
8aws:meta.llama3-1-8b-instruct-v1:00.828657To compare these two numbers, we need to look at the decimal part. Since 9.11 has a larger decimal part (0.11) than 9.9 (0.9), 9.11 is bigger.
9groq:llama-3.1-8b-instant0.2323539.9 is bigger than 9.11.
10fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct0.1929789.9 is bigger than 9.11.
11together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo0.3646799.9 is bigger than 9.11.
12octo:meta-llama-3.1-8b-instruct0.911637To compare these two numbers, we need to look at the decimal part. \n", - "\n", - "9.11 has a decimal part of 0.11, and 9.9 has a decimal part of 0.9. \n", - "\n", - "Since 0.11 is greater than 0.9, 9.11 is bigger than 9.9.
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "display(llms, execution_times, responses)" ] }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "198b213a-b7bf-4cce-8c30-a8408454370b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "aws:meta.llama3-8b-instruct-v1:0 - 1.83 seconds: Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.\n", - "groq:llama3-8b-8192 - 0.31 seconds: Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.\n", - "fireworks:accounts/fireworks/models/llama-v3-8b-instruct - 0.72 seconds: Let's break it down step by step!\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the correct answer is: 9.9 is bigger than 9.11!\n", - "octo:meta-llama-3-8b-instruct - 1.40 seconds: Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.\n", - "together:meta-llama/Llama-3-8b-chat-hf - 0.76 seconds: Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.\n", - "openai:gpt-3.5-turbo - 1.26 seconds: To determine which number is bigger, we can compare the whole numbers first. In this case, both numbers have a whole number of 9. Since they are the same, we need to look at the decimal portion.\n", - "\n", - "9.11 is bigger than 9.9 because 0.11 is greater than 0.9.\n", - "replicate:meta/meta-llama-3-8b-instruct - 2.31 seconds: Let's break it down step by step!\n", - "\n", - "1. Both numbers start with the same digits: 9.1\n", - "2. The next digit is also the same: 1\n", - "3. The last digit is different: 9.11 has a 1, while 9.9 has a 9\n", - "\n", - "Since 9 is greater than 1, the correct answer is:\n", - "\n", - "9.9 is bigger than 9.11\n", - "aws:meta.llama3-1-8b-instruct-v1:0 - 1.37 seconds: Let's compare the two numbers step by step.\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9. Which one is bigger?\n", - "3. Since 0.9 is bigger than 0.11, the whole number 9.9 is bigger than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.\n", - "groq:llama-3.1-8b-instant - 0.46 seconds: Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9.\n", - "3. 0.9 is bigger than 0.11.\n", - "4. Since 9 is the same in both numbers, the overall number with 0.9 is bigger than the number with 0.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.\n", - "fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct - 0.58 seconds: Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9.\n", - "3. 0.9 is bigger than 0.11.\n", - "4. Since 9 is the same in both numbers, the overall number with 0.9 is bigger than the number with 0.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.\n", - "together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo - 0.99 seconds: Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so they are equal in the first digit.\n", - "2. The next digit is 1 in 9.11 and 9 in 9.9. Since 9 is greater than 1, 9.9 is bigger than 9.11 in the second digit.\n", - "3. Since the first two digits are the same in both numbers (9.1), we need to look at the third digit to break the tie. In 9.11, the third digit is 1, and in 9.9, the third digit is 9. Since 9 is greater than 1, 9.9 is bigger than 9.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.\n", - "octo:meta-llama-3.1-8b-instruct - 1.55 seconds: To determine which number is bigger, let's compare them step by step:\n", - "\n", - "1. Both numbers start with 9, so they are equal in the first digit.\n", - "2. Now, let's look at the second digit. 9.11 has a 1, while 9.9 has a 9.\n", - "3. Since 9 is greater than 1, 9.9 is bigger than 9.11.\n", - "\n", - "So, the number 9.9 is bigger than 9.11.\n" - ] - } - ], + "outputs": [], "source": [ "messages = [\n", " {\"role\": \"user\", \"content\": \"Which number is bigger, 9.11 or 9.9? Think step by step.\"},\n", @@ -927,195 +261,10 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "4a3fb8fc-a7a2-47d3-9db2-792f03cc47c2", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 Provider:Model NameExecution TimeModel Response
1aws:meta.llama3-8b-instruct-v1:01.830247Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.
2groq:llama3-8b-81920.307869Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.
3fireworks:accounts/fireworks/models/llama-v3-8b-instruct0.716259Let's break it down step by step!\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the correct answer is: 9.9 is bigger than 9.11!
4octo:meta-llama-3-8b-instruct1.397936Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.
5together:meta-llama/Llama-3-8b-chat-hf0.764065Let's break it down step by step:\n", - "\n", - "1. Both numbers have the same first digit, which is 9.\n", - "2. The second digit of 9.11 is 1, and the second digit of 9.9 is 9.\n", - "3. Since 9 is greater than 1, the second digit of 9.9 is larger than the second digit of 9.11.\n", - "4. Therefore, 9.9 is greater than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.
6openai:gpt-3.5-turbo1.262903To determine which number is bigger, we can compare the whole numbers first. In this case, both numbers have a whole number of 9. Since they are the same, we need to look at the decimal portion.\n", - "\n", - "9.11 is bigger than 9.9 because 0.11 is greater than 0.9.
7replicate:meta/meta-llama-3-8b-instruct2.307123Let's break it down step by step!\n", - "\n", - "1. Both numbers start with the same digits: 9.1\n", - "2. The next digit is also the same: 1\n", - "3. The last digit is different: 9.11 has a 1, while 9.9 has a 9\n", - "\n", - "Since 9 is greater than 1, the correct answer is:\n", - "\n", - "9.9 is bigger than 9.11
8aws:meta.llama3-1-8b-instruct-v1:01.374292Let's compare the two numbers step by step.\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9. Which one is bigger?\n", - "3. Since 0.9 is bigger than 0.11, the whole number 9.9 is bigger than 9.11.\n", - "\n", - "So, the answer is: 9.9 is bigger than 9.11.
9groq:llama-3.1-8b-instant0.463357Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9.\n", - "3. 0.9 is bigger than 0.11.\n", - "4. Since 9 is the same in both numbers, the overall number with 0.9 is bigger than the number with 0.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.
10fireworks:accounts/fireworks/models/llama-v3p1-8b-instruct0.584036Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so we can ignore the 9 for now.\n", - "2. We're left with 0.11 and 0.9.\n", - "3. 0.9 is bigger than 0.11.\n", - "4. Since 9 is the same in both numbers, the overall number with 0.9 is bigger than the number with 0.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.
11together:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo0.994566Let's compare the two numbers step by step:\n", - "\n", - "1. Both numbers start with 9, so they are equal in the first digit.\n", - "2. The next digit is 1 in 9.11 and 9 in 9.9. Since 9 is greater than 1, 9.9 is bigger than 9.11 in the second digit.\n", - "3. Since the first two digits are the same in both numbers (9.1), we need to look at the third digit to break the tie. In 9.11, the third digit is 1, and in 9.9, the third digit is 9. Since 9 is greater than 1, 9.9 is bigger than 9.11.\n", - "\n", - "Therefore, 9.9 is bigger than 9.11.
12octo:meta-llama-3.1-8b-instruct1.551550To determine which number is bigger, let's compare them step by step:\n", - "\n", - "1. Both numbers start with 9, so they are equal in the first digit.\n", - "2. Now, let's look at the second digit. 9.11 has a 1, while 9.9 has a 9.\n", - "3. Since 9 is greater than 1, 9.9 is bigger than 9.11.\n", - "\n", - "So, the number 9.9 is bigger than 9.11.
\n" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 15, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "display(llms, execution_times, responses)" ] @@ -1157,7 +306,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.14" + "version": "3.12.6" } }, "nbformat": 4,