From bbc16577518f93c8304366a9f57fa727786fa4a0 Mon Sep 17 00:00:00 2001 From: ZiTao-Li <135263265+ZiTao-Li@users.noreply.github.com> Date: Wed, 22 May 2024 17:53:39 +0800 Subject: [PATCH] Add post API for embedding model and fix embedding model wrappers (#186) * add post model for embedding mdoel * fix bugs in other embedding models * update documents * update as comment * update documents * make ollama consistent * fix typo * fix typo in markdown files * merge --- .../en/source/tutorial/203-model.md | 91 +++++++++++++++---- .../zh_CN/source/tutorial/203-model.md | 87 ++++++++++++++---- .../postapi_model_config_template.json | 32 +++++++ src/agentscope/models/dashscope_model.py | 16 +--- src/agentscope/models/ollama_model.py | 2 +- src/agentscope/models/openai_model.py | 14 +-- src/agentscope/models/post_model.py | 60 ++++++++++++ 7 files changed, 244 insertions(+), 58 deletions(-) diff --git a/docs/sphinx_doc/en/source/tutorial/203-model.md b/docs/sphinx_doc/en/source/tutorial/203-model.md index d6e153d0f..d5565b617 100644 --- a/docs/sphinx_doc/en/source/tutorial/203-model.md +++ b/docs/sphinx_doc/en/source/tutorial/203-model.md @@ -90,7 +90,9 @@ In the current AgentScope, the supported `model_type` types, the corresponding | | Generation | [`OllamaGenerationWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py) | `"ollama_generate"` | llama2, ... | | LiteLLM API | Chat | [`LiteLLMChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py) | `"litellm_chat"` | - | | Post Request based API | - | [`PostAPIModelWrapperBase`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api"` | - | -| | Chat | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... | +| | Chat | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... | +| | Image Synthesis | [`PostAPIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `post_api_dall_e` | - | | +| | Embedding | [`PostAPIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `post_api_embedding` | - | #### Detailed Parameters @@ -417,15 +419,33 @@ Here we provide example configurations for different model wrappers.
+#### LiteLLM Chat API + +
+LiteLLM Chat API (agentscope.models.LiteLLMChatModelWrapper) + +```python +{ + "config_name": "lite_llm_openai_chat_gpt-3.5-turbo", + "model_type": "litellm_chat", + "model_name": "gpt-3.5-turbo" # You should note that for different models, you should set the corresponding environment variables, such as OPENAI_API_KEY, etc. You may refer to https://docs.litellm.ai/docs/ for this. +}, +``` + +
+ +
+ #### Post Request API
-Post request API (agentscope.models.PostAPIModelWrapperBase) +Post Request Chat API (agentscope.models.PostAPIChatWrapper) ```python { - "config_name": "my_postapiwrapper_config", - "model_type": "post_api", + "config_name": "my_postapichatwrapper_config", + "model_type": "post_api_chat", # Required parameters "api_url": "https://xxx.xxx", @@ -437,41 +457,69 @@ Here we provide example configurations for different model wrappers. "messages_key": "messages", } ``` +> ⚠️ The Post Request Chat model wrapper (`PostAPIChatWrapper`) has the following properties: +> 1) The `.format()` function makes sure the input messages become a list of dicts. +> 2) The `._parse_response()` function assumes the generated text will be in `response["data"]["response"]["choices"][0]["message"]["content"]`
-
- -#### LiteLLM Chat API
-LiteLLM Chat API (agentscope.models.LiteLLMChatModelWrapper) +Post Request Image Synthesis API (agentscope.models.PostAPIDALLEWrapper) ```python { - "config_name": "lite_llm_openai_chat_gpt-3.5-turbo", - "model_type": "litellm_chat", - "model_name": "gpt-3.5-turbo" # You should note that for different models, you should set the corresponding environment variables, such as OPENAI_API_KEY, etc. You may refer to https://docs.litellm.ai/docs/ for this. -}, + "config_name": "my_postapiwrapper_config", + "model_type": "post_api_dall_e", + + # Required parameters + "api_url": "https://xxx.xxx", + "headers": { + # e.g. "Authorization": "Bearer xxx", + }, + + # Optional parameters + "messages_key": "messages", +} ``` +> ⚠️ The Post Request Image Synthesis model wrapper (`PostAPIDALLEWrapper`) has the following properties: +> 1) The `._parse_response()` function assumes the generated image will be presented as urls in `response["data"]["response"]["data"][i]["url"]`
-
+
+Post Request Embedding API (agentscope.models.PostAPIEmbeddingWrapper) + +```python +{ + "config_name": "my_postapiwrapper_config", + "model_type": "post_api_embedding", + + # Required parameters + "api_url": "https://xxx.xxx", + "headers": { + # e.g. "Authorization": "Bearer xxx", + }, + + # Optional parameters + "messages_key": "messages", +} +``` +> ⚠️ The Post Request Embedding model wrapper (`PostAPIEmbeddingWrapper`) has the following properties: +> 1) The `._parse_response()` function assumes the generated embeddings will be in `response["data"]["response"]["data"][i]["embedding"]` + +
-#### Post Request Chat API
-Post request Chat API (agentscope.models.PostAPIChatModelWrapper) +Post Request API (agentscope.models.PostAPIModelWrapperBase) ```python { - "config_name": "my_postapichatwrapper_config", - "model_type": "post_api_chat", + "config_name": "my_postapiwrapper_config", + "model_type": "post_api", # Required parameters "api_url": "https://xxx.xxx", @@ -483,9 +531,14 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentsc "messages_key": "messages", } ``` +> ⚠️ Post Request model wrapper (`PostAPIModelWrapperBase`) returns raw HTTP responses from the API in ModelResponse, and the `.format()` is not implemented. It is recommended to use `Post Request Chat API` when running examples with chats. +> `PostAPIModelWrapperBase` can be used when +> 1) only the raw HTTP response is wanted and `.format()` is not called; +> 2) Or, the developers want to overwrite the `.format()` and/or `._parse_response()` functions.
+
## Build Model Service from Scratch diff --git a/docs/sphinx_doc/zh_CN/source/tutorial/203-model.md b/docs/sphinx_doc/zh_CN/source/tutorial/203-model.md index 7b912cbf2..78dc90bad 100644 --- a/docs/sphinx_doc/zh_CN/source/tutorial/203-model.md +++ b/docs/sphinx_doc/zh_CN/source/tutorial/203-model.md @@ -94,7 +94,7 @@ API如下： | API | Task | Model Wrapper | `model_type` | Some Supported Models | |------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------|-------------------------------|--------------------------------------------------| -| OpenAI API | Chat | [`OpenAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py) | `"openai_chat"` | gpt-4, gpt-3.5-turbo, ... | +| OpenAI API | Chat | [`OpenAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py) | `"openai_chat"` | gpt-4, gpt-3.5-turbo, ... | | | Embedding | [`OpenAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py) | `"openai_embedding"` | text-embedding-ada-002, ... | | | DALL·E | [`OpenAIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py) | `"openai_dall_e"` | dall-e-2, dall-e-3 | | DashScope API | Chat | [`DashScopeChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/dashscope_model.py) | `"dashscope_chat"` | qwen-plus, qwen-max, ... | @@ -103,14 +103,16 @@ API如下： | | Multimodal | [`DashScopeMultiModalWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/dashscope_model.py) | `"dashscope_multimodal"` | qwen-vl-plus, qwen-vl-max, qwen-audio-turbo, ... | | Gemini API | Chat | [`GeminiChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/gemini_model.py) | `"gemini_chat"` | gemini-pro, ... | | | Embedding | [`GeminiEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/gemini_model.py) | `"gemini_embedding"` | models/embedding-001, ... | -| ZhipuAI API | Chat | [`ZhipuAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py) | `"zhipuai_chat"` | glm-4, ... | -| | Embedding | [`ZhipuAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py) | `"zhipuai_embedding"` | embedding-2, ... | +| ZhipuAI API | Chat | [`ZhipuAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py) | `"zhipuai_chat"` | glm-4, ... | +| | Embedding | [`ZhipuAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py) | `"zhipuai_embedding"` | embedding-2, ... | | ollama | Chat | [`OllamaChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py) | `"ollama_chat"` | llama2, ... | | | Embedding | [`OllamaEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py) | `"ollama_embedding"` | llama2, ... | | | Generation | [`OllamaGenerationWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py) | `"ollama_generate"` | llama2, ... | | LiteLLM API | Chat | [`LiteLLMChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py) | `"litellm_chat"` | - | | Post Request based API | - | [`PostAPIModelWrapperBase`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api"` | - | -| | Chat | [`PostAPIChatModelWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... | +| | Chat | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... | +| | Image Synthesis | [`PostAPIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `post_api_dall_e` | - | | +| | Embedding | [`PostAPIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `post_api_embedding` | - | #### 详细参数 @@ -460,38 +462,85 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py">agen #### Post Request API
-Post request API (agentscope.models.PostAPIModelWrapperBase) +Post Request Chat API (agentscope.models.PostAPIChatWrapper) ```python { "config_name": "my_postapiwrapper_config", - "model_type": "post_api", + "model_type": "post_api_chat", - # 必要参数 + # Required parameters "api_url": "https://xxx.xxx", "headers": { - # 例如："Authorization": "Bearer xxx", + # e.g. "Authorization": "Bearer xxx", }, - # 可选参数 + # Optional parameters "messages_key": "messages", } ``` +> ⚠️ Post Request Chat model wrapper (`PostAPIChatWrapper`) 有以下特性： +> 1) 它的 `.format()` 方法会确保输入的信息（messages）会被转换成字典列表（a list of dict）. +> 2) 它的 `._parse_response()` 方法假设了生成的文字内容会在 `response["data"]["response"]["choices"][0]["message"]["content"]`
-
-#### Post Request Chat API +
+Post Request Image Synthesis API (agentscope.models.PostAPIDALLEWrapper) + +```python +{ + "config_name": "my_postapiwrapper_config", + "model_type": "post_api_dall_e", + + # Required parameters + "api_url": "https://xxx.xxx", + "headers": { + # e.g. "Authorization": "Bearer xxx", + }, + + # Optional parameters + "messages_key": "messages", +} +``` +> ⚠️ Post Request Image Synthesis model wrapper (`PostAPIDALLEWrapper`) 有以下特性: +> 1) 它的 `._parse_response()` 方法假设生成的图片都以url的形式表示在`response["data"]["response"]["data"][i]["url"]` + + +
+ +
+Post Request Embedding API (agentscope.models.PostAPIEmbeddingWrapper) + +```python +{ + "config_name": "my_postapiwrapper_config", + "model_type": "post_api_embedding", + + # Required parameters + "api_url": "https://xxx.xxx", + "headers": { + # e.g. "Authorization": "Bearer xxx", + }, + + # Optional parameters + "messages_key": "messages", +} +``` + +> ⚠️ Post Request Embedding model wrapper (`PostAPIEmbeddingWrapper`) 有以下特性: +> 1) 它的 `._parse_response()`方法假设生成的特征向量会存放在 `response["data"]["response"]["data"][i]["embedding"]` + +
-Post request Chat API (agentscope.models.PostAPIChatModelWrapper) +Post Request API (agentscope.models.PostAPIModelWrapperBase) ```python { - "config_name": "my_postapichatwrapper_config", - "model_type": "post_api_chat", + "config_name": "my_postapiwrapper_config", + "model_type": "post_api", # 必要参数 "api_url": "https://xxx.xxx", @@ -503,13 +552,19 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentsc "messages_key": "messages", } ``` +> ⚠️ Post request model wrapper (`PostAPIModelWrapperBase`) 返回原生的 HTTP 响应值， 且没有实现 `.format()`. 当运行样例时，推荐使用 `Post Request Chat API`. +> 使用`PostAPIModelWrapperBase`时，需要注意 +> 1) `.format()` 方法不能被调用； +> 2) 或开发者希望实现自己的`.format()`和/或`._parse_response()`
-
+ +
+ ## 从零搭建模型服务 针对需要自己搭建模型服务的开发者，AgentScope提供了一些脚本来帮助开发者快速搭建模型服务。您可以在[scripts](https://github.com/modelscope/agentscope/tree/main/scripts)目录下找到这些脚本以及说明。 diff --git a/examples/model_configs_template/postapi_model_config_template.json b/examples/model_configs_template/postapi_model_config_template.json index c5e512dab..3f4586621 100644 --- a/examples/model_configs_template/postapi_model_config_template.json +++ b/examples/model_configs_template/postapi_model_config_template.json @@ -45,5 +45,37 @@ "Authorization": "Bearer {YOUR_API_TOKEN}" }, "api_url": "https://api-inference.huggingface.co/models/gpt2" +}, +{ + "config_name": "post_api_img_syn_config", + "model_type": "post_api_dall_e", + "api_url": "http://xxx.xxx.xxx.xxx:xxxx/xxx", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {YOUR_API_TOKEN}" + }, + "messages_key": "prompt", + "json_args": { + "model": "{YOUR_MODEL}", + "n":1, + "quality":"standard", + "response_format":"url", + "size":"512x512", + "style":"natural" + } +}, +{ + "config_name": "post_api_embedding_config", + "model_type": "post_api_embedding", + "api_url": "http://xxx.xxx.xxx.xxx:xxxx/xxx", + "headers": { + "Content-Type": "application/json", + "Authorization": "Bearer {YOUR_API_TOKEN}" + }, + "messages_key": "input", + "json_args": { + "model": "{YOUR_MODEL}", + "encoding_format": "float" + } } ] \ No newline at end of file diff --git a/src/agentscope/models/dashscope_model.py b/src/agentscope/models/dashscope_model.py index c4183aa85..4abc6b4ff 100644 --- a/src/agentscope/models/dashscope_model.py +++ b/src/agentscope/models/dashscope_model.py @@ -505,18 +505,10 @@ def __call__( ) # step5: return response - if len(response.output["embeddings"]) == 0: - return ModelResponse( - embedding=response.output["embedding"][0], - raw=response, - ) - else: - return ModelResponse( - embedding=[ - _["embedding"] for _ in response.output["embeddings"] - ], - raw=response, - ) + return ModelResponse( + embedding=[_["embedding"] for _ in response.output["embeddings"]], + raw=response, + ) class DashScopeMultiModalWrapper(DashScopeWrapperBase): diff --git a/src/agentscope/models/ollama_model.py b/src/agentscope/models/ollama_model.py index b4157c586..9bdc07869 100644 --- a/src/agentscope/models/ollama_model.py +++ b/src/agentscope/models/ollama_model.py @@ -341,7 +341,7 @@ def __call__( # step5: return response return ModelResponse( - embedding=response["embedding"], + embedding=[response["embedding"]], raw=response, ) diff --git a/src/agentscope/models/openai_model.py b/src/agentscope/models/openai_model.py index 99542582b..9bbdc59c1 100644 --- a/src/agentscope/models/openai_model.py +++ b/src/agentscope/models/openai_model.py @@ -522,13 +522,7 @@ def __call__( # step5: return response response_json = response.model_dump() - if len(response_json["data"]) == 0: - return ModelResponse( - embedding=response_json["data"]["embedding"][0], - raw=response_json, - ) - else: - return ModelResponse( - embedding=[_["embedding"] for _ in response_json["data"]], - raw=response_json, - ) + return ModelResponse( + embedding=[_["embedding"] for _ in response_json["data"]], + raw=response_json, + ) diff --git a/src/agentscope/models/post_model.py b/src/agentscope/models/post_model.py index c60fb2632..fe61be221 100644 --- a/src/agentscope/models/post_model.py +++ b/src/agentscope/models/post_model.py @@ -240,3 +240,63 @@ def format( f"need to format the input. Please try to use the " f"model wrapper directly.", ) + + +class PostAPIEmbeddingWrapper(PostAPIModelWrapperBase): + """ + A post api model wrapper for embedding model + """ + + model_type: str = "post_api_embedding" + + def _parse_response(self, response: dict) -> ModelResponse: + """ + Parse the response json data into ModelResponse with embedding. + Args: + response (`dict`): + The response obtained from the API. This parsing assume the + structure of the response is as following: + { + "code": 200, + "data": { + ... + "response": { + "data": [ + { + "embedding": [ + 0.001, + ... + ], + ... + } + ], + "model": "xxxx", + ... + }, + }, + } + """ + if "data" not in response["data"]["response"]: + if "error" in response["data"]["response"]: + error_msg = response["data"]["response"]["error"]["message"] + else: + error_msg = response["data"]["response"] + logger.error(f"Error in embedding API call:\n{error_msg}") + raise ValueError(f"Error in embedding API call:\n{error_msg}") + embeddings = [ + data["embedding"] for data in response["data"]["response"]["data"] + ] + return ModelResponse( + embedding=embeddings, + raw=response, + ) + + def format( + self, + *args: Union[MessageBase, Sequence[MessageBase]], + ) -> Union[List[dict], str]: + raise RuntimeError( + f"Model Wrapper [{type(self).__name__}] doesn't " + f"need to format the input. Please try to use the " + f"model wrapper directly.", + )