Add post API for embedding model and fix embedding model wrappers (#186

)

* add post model for embedding mdoel

* fix bugs in other embedding models

* update documents

* update as comment

* update documents

* make ollama consistent

* fix typo

* fix typo in markdown files

* merge

docs/sphinx_doc/en/source/tutorial/203-model.md

@@ -90,7 +90,9 @@ In the current AgentScope, the supported `model_type` types, the corresponding
 |                        | Generation      | [`OllamaGenerationWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py)           | `"ollama_generate"`           | llama2, ...                                      |
 | LiteLLM API | Chat               | [`LiteLLMChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py)             | `"litellm_chat"`                  | -                                                |
 | Post Request based API | -               | [`PostAPIModelWrapperBase`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)             | `"post_api"`                  | -                                                |
-|  | Chat | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... |
+|                        | Chat            | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)                  | `"post_api_chat"`             | meta-llama/Meta-Llama-3-8B-Instruct, ...         |
+|                        | Image Synthesis | [`PostAPIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)                 | `post_api_dall_e`             | -                                                |                                                  |
+|                        | Embedding       | [`PostAPIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)             | `post_api_embedding`          | -                                                |
 
 #### Detailed Parameters
 
@@ -417,15 +419,33 @@ Here we provide example configurations for different model wrappers.
 
 <br/>
 
+#### LiteLLM Chat API
+
+<details>
+<summary>LiteLLM Chat API (<code><a href="https://github.
+com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py">agentscope.models.LiteLLMChatModelWrapper</a></code>)</summary>
+
+```python
+{
+    "config_name": "lite_llm_openai_chat_gpt-3.5-turbo",
+    "model_type": "litellm_chat",
+    "model_name": "gpt-3.5-turbo" # You should note that for different models, you should set the corresponding environment variables, such as OPENAI_API_KEY, etc. You may refer to https://docs.litellm.ai/docs/ for this.
+},
+```
+
+</details>
+
+<br/>
+
 #### Post Request API
 
 <details>
-<summary>Post request API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIModelWrapperBase</a></code>)</summary>
+<summary>Post Request Chat API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIChatWrapper</a></code>)</summary>
 
 ```python
 {
-    "config_name": "my_postapiwrapper_config",
-    "model_type": "post_api",
+    "config_name": "my_postapichatwrapper_config",
+    "model_type": "post_api_chat",
 
     # Required parameters
     "api_url": "https://xxx.xxx",
@@ -437,41 +457,69 @@ Here we provide example configurations for different model wrappers.
     "messages_key": "messages",
 }
 ```
+> ⚠️ The Post Request Chat model wrapper (`PostAPIChatWrapper`) has the following properties:
+> 1) The `.format()` function makes sure the input messages become a list of dicts.
+> 2) The `._parse_response()` function assumes the generated text will be in `response["data"]["response"]["choices"][0]["message"]["content"]`
 
 </details>
 
-<br/>
-
 
-#### LiteLLM Chat API
 
 <details>
-<summary>LiteLLM Chat API (<code><a href="https://github.
-com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py">agentscope.models.LiteLLMChatModelWrapper</a></code>)</summary>
+<summary>Post Request Image Synthesis API  (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIDALLEWrapper</a></code>)</summary>
 
 ```python
 {
-    "config_name": "lite_llm_openai_chat_gpt-3.5-turbo",
-    "model_type": "litellm_chat",
-    "model_name": "gpt-3.5-turbo" # You should note that for different models, you should set the corresponding environment variables, such as OPENAI_API_KEY, etc. You may refer to https://docs.litellm.ai/docs/ for this.
-},
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api_dall_e",
+
+    # Required parameters
+    "api_url": "https://xxx.xxx",
+    "headers": {
+        # e.g. "Authorization": "Bearer xxx",
+    },
+
+    # Optional parameters
+    "messages_key": "messages",
+}
 ```
+> ⚠️ The Post Request Image Synthesis model wrapper (`PostAPIDALLEWrapper`) has the following properties:
+> 1) The `._parse_response()` function assumes the generated image will be presented as urls in `response["data"]["response"]["data"][i]["url"]`
 
 </details>
 
-<br/>
 
+<details>
+<summary>Post Request Embedding API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIEmbeddingWrapper</a></code>)</summary>
+
+```python
+{
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api_embedding",
+
+    # Required parameters
+    "api_url": "https://xxx.xxx",
+    "headers": {
+        # e.g. "Authorization": "Bearer xxx",
+    },
+
+    # Optional parameters
+    "messages_key": "messages",
+}
+```
+> ⚠️ The Post Request Embedding model wrapper (`PostAPIEmbeddingWrapper`) has the following properties:
+> 1) The `._parse_response()` function assumes the generated embeddings will be in `response["data"]["response"]["data"][i]["embedding"]`
+
+</details>
 
-#### Post Request Chat API
 
 <details>
-<summary>Post request Chat API (<code><a href="https://github.
-com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIChatModelWrapper</a></code>)</summary>
+<summary>Post Request API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIModelWrapperBase</a></code>)</summary>
 
 ```python
 {
-    "config_name": "my_postapichatwrapper_config",
-    "model_type": "post_api_chat",
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api",
 
     # Required parameters
     "api_url": "https://xxx.xxx",
@@ -483,9 +531,14 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentsc
     "messages_key": "messages",
 }
 ```
+> ⚠️ Post Request model wrapper (`PostAPIModelWrapperBase`) returns raw HTTP responses from the API in ModelResponse, and the `.format()` is not implemented. It is recommended to use `Post Request Chat API` when running examples with chats.
+> `PostAPIModelWrapperBase` can be used when
+> 1) only the raw HTTP response is wanted and `.format()` is not called;
+> 2) Or, the developers want to overwrite the `.format()` and/or `._parse_response()` functions.
 
 </details>
 
+
 <br/>
 
 ## Build Model Service from Scratch

docs/sphinx_doc/zh_CN/source/tutorial/203-model.md

@@ -94,7 +94,7 @@ API如下：
 
 | API                    | Task            | Model Wrapper                                                                                                                   | `model_type`                  | Some Supported Models                            |
 |------------------------|-----------------|---------------------------------------------------------------------------------------------------------------------------------|-------------------------------|--------------------------------------------------|
-| OpenAI API             | Chat            | [`OpenAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py)                 | `"openai_chat"`                    | gpt-4, gpt-3.5-turbo, ...                        |
+| OpenAI API             | Chat            | [`OpenAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py)                 | `"openai_chat"`               | gpt-4, gpt-3.5-turbo, ...                        |
 |                        | Embedding       | [`OpenAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py)            | `"openai_embedding"`          | text-embedding-ada-002, ...                      |
 |                        | DALL·E          | [`OpenAIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/openai_model.py)                | `"openai_dall_e"`             | dall-e-2, dall-e-3                               |
 | DashScope API          | Chat            | [`DashScopeChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/dashscope_model.py)           | `"dashscope_chat"`            | qwen-plus, qwen-max, ...                         |
@@ -103,14 +103,16 @@ API如下：
 |                        | Multimodal      | [`DashScopeMultiModalWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/dashscope_model.py)     | `"dashscope_multimodal"`      | qwen-vl-plus, qwen-vl-max, qwen-audio-turbo, ... |
 | Gemini API             | Chat            | [`GeminiChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/gemini_model.py)                 | `"gemini_chat"`               | gemini-pro, ...                                  |
 |                        | Embedding       | [`GeminiEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/gemini_model.py)            | `"gemini_embedding"`          | models/embedding-001, ...                        |
-| ZhipuAI API             | Chat            | [`ZhipuAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py)                 | `"zhipuai_chat"`               | glm-4, ...                                  |
-|                        | Embedding       | [`ZhipuAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py)            | `"zhipuai_embedding"`          | embedding-2, ...                        |
+| ZhipuAI API            | Chat            | [`ZhipuAIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py)                 | `"zhipuai_chat"`              | glm-4, ...                                       |
+|                        | Embedding       | [`ZhipuAIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/zhipu_model.py)            | `"zhipuai_embedding"`         | embedding-2, ...                                 |
 | ollama                 | Chat            | [`OllamaChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py)                 | `"ollama_chat"`               | llama2, ...                                      |
 |                        | Embedding       | [`OllamaEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py)            | `"ollama_embedding"`          | llama2, ...                                      |
 |                        | Generation      | [`OllamaGenerationWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/ollama_model.py)           | `"ollama_generate"`           | llama2, ...                                      |
 | LiteLLM API | Chat               | [`LiteLLMChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py)             | `"litellm_chat"`                  | -                                                |
 | Post Request based API | -               | [`PostAPIModelWrapperBase`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)             | `"post_api"`                  | -                                                |
-|  | Chat | [`PostAPIChatModelWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py) | `"post_api_chat"` | meta-llama/Meta-Llama-3-8B-Instruct, ... |
+|                        | Chat            | [`PostAPIChatWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)                  | `"post_api_chat"`             | meta-llama/Meta-Llama-3-8B-Instruct, ...         |
+|                        | Image Synthesis | [`PostAPIDALLEWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)                 | `post_api_dall_e`             | -                                                |                                                  |
+|                        | Embedding       | [`PostAPIEmbeddingWrapper`](https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py)             | `post_api_embedding`          | -                                                |
 
 #### 详细参数
 
@@ -460,38 +462,85 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/litellm_model.py">agen
 #### Post Request API
 
 <details>
-<summary>Post request API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIModelWrapperBase</a></code>)</summary>
+<summary>Post Request Chat API  (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIChatWrapper</a></code>)</summary>
 
 ```python
 {
     "config_name": "my_postapiwrapper_config",
-    "model_type": "post_api",
+    "model_type": "post_api_chat",
 
-    # 必要参数
+    # Required parameters
     "api_url": "https://xxx.xxx",
     "headers": {
-        # 例如："Authorization": "Bearer xxx",
+        # e.g. "Authorization": "Bearer xxx",
     },
 
-    # 可选参数
+    # Optional parameters
     "messages_key": "messages",
 }
 ```
+> ⚠️ Post Request Chat model wrapper (`PostAPIChatWrapper`) 有以下特性：
+> 1) 它的 `.format()` 方法会确保输入的信息（messages）会被转换成字典列表（a list of dict）.
+> 2) 它的 `._parse_response()` 方法假设了生成的文字内容会在 `response["data"]["response"]["choices"][0]["message"]["content"]`
 
 </details>
 
-<br/>
 
-#### Post Request Chat API
+<details>
+<summary>Post Request Image Synthesis API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIDALLEWrapper</a></code>)</summary>
+
+```python
+{
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api_dall_e",
+
+    # Required parameters
+    "api_url": "https://xxx.xxx",
+    "headers": {
+        # e.g. "Authorization": "Bearer xxx",
+    },
+
+    # Optional parameters
+    "messages_key": "messages",
+}
+```
+> ⚠️  Post Request Image Synthesis model wrapper (`PostAPIDALLEWrapper`) 有以下特性:
+> 1) 它的 `._parse_response()` 方法假设生成的图片都以url的形式表示在`response["data"]["response"]["data"][i]["url"]`
+
+
+</details>
+
+<details>
+<summary>Post Request Embedding API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIEmbeddingWrapper</a></code>)</summary>
+
+```python
+{
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api_embedding",
+
+    # Required parameters
+    "api_url": "https://xxx.xxx",
+    "headers": {
+        # e.g. "Authorization": "Bearer xxx",
+    },
+
+    # Optional parameters
+    "messages_key": "messages",
+}
+```
+
+> ⚠️ Post Request Embedding model wrapper (`PostAPIEmbeddingWrapper`) 有以下特性:
+> 1) 它的 `._parse_response()`方法假设生成的特征向量会存放在 `response["data"]["response"]["data"][i]["embedding"]`
+
+</details>
 
 <details>
-<summary>Post request Chat API (<code><a href="https://github.
-com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIChatModelWrapper</a></code>)</summary>
+<summary>Post Request API (<code><a href="https://github.com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentscope.models.PostAPIModelWrapperBase</a></code>)</summary>
 
 ```python
 {
-    "config_name": "my_postapichatwrapper_config",
-    "model_type": "post_api_chat",
+    "config_name": "my_postapiwrapper_config",
+    "model_type": "post_api",
 
     # 必要参数
     "api_url": "https://xxx.xxx",
@@ -503,13 +552,19 @@ com/modelscope/agentscope/blob/main/src/agentscope/models/post_model.py">agentsc
     "messages_key": "messages",
 }
 ```
+> ⚠️ Post request model wrapper (`PostAPIModelWrapperBase`) 返回原生的 HTTP 响应值， 且没有实现 `.format()`. 当运行样例时，推荐使用 `Post Request Chat API`.
+> 使用`PostAPIModelWrapperBase`时，需要注意
+> 1) `.format()` 方法不能被调用；
+> 2) 或开发者希望实现自己的`.format()`和/或`._parse_response()`
 
 </details>
 
-<br/>
 
 
 
+
+<br/>
+
 ## 从零搭建模型服务
 
 针对需要自己搭建模型服务的开发者，AgentScope提供了一些脚本来帮助开发者快速搭建模型服务。您可以在[scripts](https://github.com/modelscope/agentscope/tree/main/scripts)目录下找到这些脚本以及说明。

examples/model_configs_template/postapi_model_config_template.json

@@ -45,5 +45,37 @@
         "Authorization": "Bearer {YOUR_API_TOKEN}"
     },
     "api_url": "https://api-inference.huggingface.co/models/gpt2"
+},
+{
+    "config_name": "post_api_img_syn_config",
+    "model_type": "post_api_dall_e",
+    "api_url": "http://xxx.xxx.xxx.xxx:xxxx/xxx",
+    "headers": {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer {YOUR_API_TOKEN}"
+    },
+    "messages_key": "prompt",
+    "json_args": {
+        "model": "{YOUR_MODEL}",
+        "n":1,
+        "quality":"standard",
+        "response_format":"url",
+        "size":"512x512",
+        "style":"natural"
+    }
+},
+{
+    "config_name": "post_api_embedding_config",
+    "model_type": "post_api_embedding",
+    "api_url": "http://xxx.xxx.xxx.xxx:xxxx/xxx",
+    "headers": {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer {YOUR_API_TOKEN}"
+    },
+    "messages_key": "input",
+    "json_args": {
+        "model": "{YOUR_MODEL}",
+        "encoding_format": "float"
+    }
 }
 ]

src/agentscope/models/dashscope_model.py

@@ -505,18 +505,10 @@ def __call__(
         )
 
         # step5: return response
-        if len(response.output["embeddings"]) == 0:
-            return ModelResponse(
-                embedding=response.output["embedding"][0],
-                raw=response,
-            )
-        else:
-            return ModelResponse(
-                embedding=[
-                    _["embedding"] for _ in response.output["embeddings"]
-                ],
-                raw=response,
-            )
+        return ModelResponse(
+            embedding=[_["embedding"] for _ in response.output["embeddings"]],
+            raw=response,
+        )
 
 
 class DashScopeMultiModalWrapper(DashScopeWrapperBase):

src/agentscope/models/ollama_model.py

@@ -341,7 +341,7 @@ def __call__(
 
         # step5: return response
         return ModelResponse(
-            embedding=response["embedding"],
+            embedding=[response["embedding"]],
             raw=response,
         )
 

src/agentscope/models/openai_model.py

@@ -522,13 +522,7 @@ def __call__(
 
         # step5: return response
         response_json = response.model_dump()
-        if len(response_json["data"]) == 0:
-            return ModelResponse(
-                embedding=response_json["data"]["embedding"][0],
-                raw=response_json,
-            )
-        else:
-            return ModelResponse(
-                embedding=[_["embedding"] for _ in response_json["data"]],
-                raw=response_json,
-            )
+        return ModelResponse(
+            embedding=[_["embedding"] for _ in response_json["data"]],
+            raw=response_json,
+        )