adding support for Claude, Cohere and PaLM + Model Fallback Strategy (prevent failed requests)

agent/llm_utils.py

@@ -4,7 +4,7 @@
 
 from fastapi import WebSocket
 import time
-
+from litellm import completion
 import openai
 from colorama import Fore, Style
 from openai.error import APIError, RateLimitError
@@ -69,13 +69,19 @@ def send_chat_completion_request(
     messages, model, temperature, max_tokens, stream, websocket
 ):
     if not stream:
-        result = openai.ChatCompletion.create(
-            model=model,
-            messages=messages,
-            temperature=temperature,
-            max_tokens=max_tokens,
-        )
-        return result.choices[0].message["content"]
+        model_fallback_list = ["claude-instant-1", "gpt-3.5-turbo", "gpt-3.5-turbo-16k"]
+        model_fallback_list = [model] + model_fallback_list
+        for model in model_fallback_list:
+            try:
+                result = completion(
+                    model=model,
+                    messages=messages,
+                    temperature=temperature,
+                    max_tokens=max_tokens,
+                )
+                return result.choices[0].message["content"]
+            except:
+                pass
     else:
         return stream_response(model, messages, temperature, max_tokens, websocket)
 
@@ -114,7 +120,7 @@ def choose_agent(task: str) -> str:
     try:
         configuration = choose_agent_configuration()
 
-        response = openai.ChatCompletion.create(
+        response = completion(
             model=CFG.smart_llm_model,
             messages=[
                 {"role": "user", "content": f"{task}"}],

requirements.txt

@@ -16,3 +16,4 @@ pydantic
 fastapi
 python-multipart
 markdown
+litellm==0.1.2291