From b9589631b16d806c9468cf507d915ff4aad144cc Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Fri, 23 Feb 2024 14:42:36 +0530 Subject: [PATCH 1/6] Added openrouter in e2e --- e2e.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e.py b/e2e.py index b7292fe..a12bb4e 100644 --- a/e2e.py +++ b/e2e.py @@ -14,6 +14,7 @@ mistral, octoai, openai, + openrouter, prem, replicate, together, @@ -72,6 +73,8 @@ def main(): ) elif connector["provider"] == "prem": connector_object = prem.PremConnector(api_key=os.environ["PREMAI_BEARER_TOKEN"]) + elif connector["provider"] == "openrouter": + connector_object = openrouter.OpenRouterConnector(api_key=os.environ["OPENROUTER_API_KEY"]) else: print(f"No connector for {connector['provider']}") From 859bd4391f318356c046b67d0ab32516ebb5eccd Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Fri, 23 Feb 2024 14:42:58 +0530 Subject: [PATCH 2/6] Added openrouter providers in models.json --- prem_utils/models.json | 125 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/prem_utils/models.json b/prem_utils/models.json index 6213069..7e69c33 100644 --- a/prem_utils/models.json +++ b/prem_utils/models.json @@ -593,6 +593,131 @@ "coming_soon": true } ] + }, + { + "provider": "openrouter", + "models": [ + { + "slug": "openrouter/auto", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "nousresearch/nous-capybara-7b:free", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "mistralai/mistral-7b-instruct:free", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "gryphe/mythomist-7b:free", + "model_type": "text2text", + "context_tokens": 32768 + }, + { + "slug": "undi95/toppy-m-7b:free", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "openrouter/cinematika-7b:free", + "model_type": "text2text", + "context_tokens": 8000 + }, + { + "slug": "google/gemma-7b-it:free", + "model_type": "text2text", + "context_tokens": 8000 + }, + { + "slug": "jondurbin/bagel-34b", + "model_type": "text2text", + "context_tokens": 8000 + }, + { + "slug": "jebcarter/psyfighter-13b", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "koboldai/psyfighter-13b-2", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "neversleep/noromaid-mixtral-8x7b-instruct", + "model_type": "text2text", + "context_tokens": 8000 + }, + { + "slug": "nousresearch/nous-hermes-llama2-13b", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "meta-llama/codellama-34b-instruct", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "phind/phind-codellama-34b", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "intel/neural-chat-7b", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "openai/creativity-unlocked-prompts-dpr-32b", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "piiezzo/chatcoder-13b-instruct", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "alephai/decoder-40b-0.125", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "alephai/generator-40b-0.125", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "genestephen/mythwriter-10b-instruct", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "gothinker/cinima-9b-instruct", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "hanghangq/nr2-2048-8x8", + "model_type": "text2text", + "context_tokens": 4096 + }, + { + "slug": "maxschatzow/parrot-40b-instruct", + "model_type": "text2text", + "context_tokens": 8192 + }, + { + "slug": "tworivers/pegasus-24b-1", + "model_type": "text2text", + "context_tokens": 8192 + } + ] } ] } From 755c20a2801a9018d7fea0b0caa9a74cecfbf8a7 Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Fri, 23 Feb 2024 14:43:40 +0530 Subject: [PATCH 3/6] Feat: Added openrouter using OpenAIBaseConnector for chat_completions --- prem_utils/connectors/openrouter.py | 70 +++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 prem_utils/connectors/openrouter.py diff --git a/prem_utils/connectors/openrouter.py b/prem_utils/connectors/openrouter.py new file mode 100644 index 0000000..d215d50 --- /dev/null +++ b/prem_utils/connectors/openrouter.py @@ -0,0 +1,70 @@ +from collections.abc import Sequence +from typing import Any + +from prem_utils.connectors.openai import OpenAIConnector + + +class OpenRouterConnector(OpenAIConnector): + def __init__( + self, api_key: str, base_url: str = "https://openrouter.ai/api/v1/", prompt_template: str = None + ) -> None: + super().__init__(prompt_template=prompt_template, base_url=base_url, api_key=api_key) + + def chat_completion( + self, + model: str, + messages: list[dict[str, Any]], + max_tokens: int = 512, + frequency_penalty: float = 0.1, + presence_penalty: float = 0, + seed: int | None = None, + stop: str | list[str] = None, + stream: bool = False, + temperature: float = 1, + top_p: float = 1, + tools: list[dict[str, Any]] = None, + tool_choice: dict = None, + ): + return super().chat_completion( + model=model, + messages=messages, + tools=tools, + tool_choice=tool_choice, + stream=stream, + max_tokens=max_tokens, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + seed=seed, + stop=stop, + temperature=temperature, + top_p=top_p, + ) + + def embeddings( + self, + model: str, + input: str | Sequence[str] | Sequence[int] | Sequence[Sequence[int]], + encoding_format: str = "float", + user: str = None, + ): + raise NotImplementedError + + def finetuning( + self, model: str, training_data: list[dict], validation_data: list[dict] | None = None, num_epochs: int = 3 + ) -> str: + raise NotImplementedError + + def get_finetuning_job(self, job_id) -> dict[str, Any]: + raise NotImplementedError + + def generate_image( + self, + model: str, + prompt: str, + size: str = "1024x1024", + n: int = 1, + quality: str = "standard", + style: str = "vivid", + response_format: str = "url", + ): + raise NotImplementedError From ff8505b6a9ee1f59d6ef25c3c152587ea55ea41b Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Mon, 26 Feb 2024 23:22:51 +0530 Subject: [PATCH 4/6] Added logic for handelling edge cases around mdoel naming --- prem_utils/connectors/openrouter.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/prem_utils/connectors/openrouter.py b/prem_utils/connectors/openrouter.py index d215d50..fa3ef57 100644 --- a/prem_utils/connectors/openrouter.py +++ b/prem_utils/connectors/openrouter.py @@ -25,6 +25,13 @@ def chat_completion( tools: list[dict[str, Any]] = None, tool_choice: dict = None, ): + if "openrouter" in model: + if len(model.split("openrouter/")) == 3: + model = model.replace("openrouter/", "", 1) + else: + if len(model.split("/")) == 3: + model = model.replace("openrouter/", "", 1) + return super().chat_completion( model=model, messages=messages, From 56559d555c07d826c2a216f850410ab4aa9d7c59 Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Mon, 26 Feb 2024 23:29:38 +0530 Subject: [PATCH 5/6] Added supported and working models from openrouter in models.json --- prem_utils/models.json | 350 +++++++++++++++++++++++++++++++++++------ 1 file changed, 303 insertions(+), 47 deletions(-) diff --git a/prem_utils/models.json b/prem_utils/models.json index 5927658..2ad70cc 100644 --- a/prem_utils/models.json +++ b/prem_utils/models.json @@ -628,124 +628,380 @@ "provider": "openrouter", "models": [ { - "slug": "openrouter/auto", + "slug": "openrouter/openrouter/auto", "model_type": "text2text", "context_tokens": 128000 }, { - "slug": "nousresearch/nous-capybara-7b:free", + "slug": "openrouter/jondurbin/bagel-34b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 + }, + { + "slug": "openrouter/jebcarter/psyfighter-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/koboldai/psyfighter-13b-2", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/neversleep/noromaid-mixtral-8x7b-instruct", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-llama2-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/meta-llama/codellama-34b-instruct", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/phind/phind-codellama-34b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/intel/neural-chat-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mistralai/mixtral-8x7b-instruct", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-dpo", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-2-mixtral-8x7b-sft", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/haotian-liu/llava-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-2-vision-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/meta-llama/llama-2-13b-chat", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/migtissera/synthia-70b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/pygmalionai/mythalion-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/undi95/remm-slerp-l2-13b-6k", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/gryphe/mythomax-l2-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/xwin-lm/xwin-lm-70b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/gryphe/mythomax-l2-13b-8k", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/alpindale/goliath-120b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/neversleep/noromaid-20b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/gryphe/mythomist-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mancer/weaver", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-capybara-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/codellama/codellama-70b-instruct", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/teknium/openhermes-2-mistral-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/teknium/openhermes-2.5-mistral-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/undi95/remm-slerp-l2-13b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/undi95/toppy-m-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/openrouter/cinematika-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/01-ai/yi-34b-chat", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/01-ai/yi-34b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/01-ai/yi-6b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/togethercomputer/stripedhyena-nous-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/togethercomputer/stripedhyena-hessian-7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mistralai/mixtral-8x7b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-yi-34b", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/nousresearch/nous-hermes-2-mistral-7b-dpo", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/open-orca/mistral-7b-openorca", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/huggingfaceh4/zephyr-7b-beta", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/google/palm-2-chat-bison", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/google/palm-2-codechat-bison", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/google/palm-2-chat-bison-32k", + "model_type": "text2text", + "context_tokens": 128000 }, { - "slug": "mistralai/mistral-7b-instruct:free", + "slug": "openrouter/google/palm-2-codechat-bison-32k", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 }, { - "slug": "gryphe/mythomist-7b:free", + "slug": "openrouter/google/gemini-pro", "model_type": "text2text", - "context_tokens": 32768 + "context_tokens": 128000 }, + { - "slug": "undi95/toppy-m-7b:free", + "slug": "openrouter/perplexity/pplx-70b-online", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "openrouter/cinematika-7b:free", + "slug": "openrouter/perplexity/pplx-7b-online", "model_type": "text2text", - "context_tokens": 8000 + "context_tokens": 128000 }, { - "slug": "google/gemma-7b-it:free", + "slug": "openrouter/perplexity/pplx-7b-chat", "model_type": "text2text", - "context_tokens": 8000 + "context_tokens": 128000 }, { - "slug": "jondurbin/bagel-34b", + "slug": "openrouter/perplexity/pplx-70b-chat", "model_type": "text2text", - "context_tokens": 8000 + "context_tokens": 128000 }, { - "slug": "jebcarter/psyfighter-13b", + "slug": "openrouter/meta-llama/llama-2-70b-chat", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "koboldai/psyfighter-13b-2", + "slug": "openrouter/nousresearch/nous-capybara-34b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "neversleep/noromaid-mixtral-8x7b-instruct", + "slug": "openrouter/jondurbin/airoboros-l2-70b", "model_type": "text2text", - "context_tokens": 8000 + "context_tokens": 128000 }, { - "slug": "nousresearch/nous-hermes-llama2-13b", + "slug": "openrouter/austism/chronos-hermes-13b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "meta-llama/codellama-34b-instruct", + "slug": "openrouter/mistralai/mistral-7b-instruct", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 }, { - "slug": "phind/phind-codellama-34b", + "slug": "openrouter/openchat/openchat-7b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "intel/neural-chat-7b", + "slug": "openrouter/lizpreciatior/lzlv-70b-fp16-hf", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "openai/creativity-unlocked-prompts-dpr-32b", + "slug": "openrouter/cognitivecomputations/dolphin-mixtral-8x7b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "piiezzo/chatcoder-13b-instruct", + "slug": "openrouter/rwkv/rwkv-5-world-3b", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 + }, + { + "slug": "openrouter/recursal/rwkv-5-3b-ai-town", + "model_type": "text2text", + "context_tokens": 128000 }, { - "slug": "alephai/decoder-40b-0.125", + "slug": "openrouter/recursal/eagle-7b", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "alephai/generator-40b-0.125", + "slug": "openrouter/google/gemma-7b-it", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "genestephen/mythwriter-10b-instruct", + "slug": "openrouter/anthropic/claude-2", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 }, { - "slug": "gothinker/cinima-9b-instruct", + "slug": "openrouter/anthropic/claude-2.1", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 }, { - "slug": "hanghangq/nr2-2048-8x8", + "slug": "openrouter/anthropic/claude-2.0", "model_type": "text2text", - "context_tokens": 4096 + "context_tokens": 128000 }, { - "slug": "maxschatzow/parrot-40b-instruct", + "slug": "openrouter/anthropic/claude-instant-1", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 }, { - "slug": "tworivers/pegasus-24b-1", + "slug": "openrouter/anthropic/claude-instant-1.2", "model_type": "text2text", - "context_tokens": 8192 + "context_tokens": 128000 + }, + { + "slug": "openrouter/anthropic/claude-2:beta", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/anthropic/claude-2.1:beta", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/anthropic/claude-2.0:beta", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/anthropic/claude-instant-1:beta", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mistralai/mistral-tiny", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mistralai/mistral-small", + "model_type": "text2text", + "context_tokens": 128000 + }, + { + "slug": "openrouter/mistralai/mistral-medium", + "model_type": "text2text", + "context_tokens": 128000 } ] }, From ae18dcbcae605d88c74bec106de650255f39481a Mon Sep 17 00:00:00 2001 From: Anindyadeep Date: Mon, 26 Feb 2024 23:42:35 +0530 Subject: [PATCH 6/6] removed the logic for edge case model name handling scenarios --- prem_utils/connectors/openrouter.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/prem_utils/connectors/openrouter.py b/prem_utils/connectors/openrouter.py index fa3ef57..089f18c 100644 --- a/prem_utils/connectors/openrouter.py +++ b/prem_utils/connectors/openrouter.py @@ -25,12 +25,7 @@ def chat_completion( tools: list[dict[str, Any]] = None, tool_choice: dict = None, ): - if "openrouter" in model: - if len(model.split("openrouter/")) == 3: - model = model.replace("openrouter/", "", 1) - else: - if len(model.split("/")) == 3: - model = model.replace("openrouter/", "", 1) + model = model.replace("openrouter/", "", 1) return super().chat_completion( model=model,