server : fill usage info in embeddings response

ggerganov · Dec 16, 2024 · 82959be · 82959be
1 parent 08ea539
commit 82959be
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 5 deletions.
diff --git a/examples/server/server.cpp b/examples/server/server.cpp
@@ -719,14 +719,17 @@ struct server_task_result_embd : server_task_result {
     int index = 0;
     std::vector<float> embedding;
 
+    int32_t n_prompt_tokens;
+
     virtual int get_index() override {
         return index;
     }
 
     virtual json to_json() override {
         return json {
-            {"index",     index},
-            {"embedding", embedding},
+            {"index",            index},
+            {"embedding",        embedding},
+            {"tokens_evaluated", n_prompt_tokens},
         };
     }
 };
@@ -1995,6 +1998,7 @@ struct server_context {
         auto res = std::make_unique<server_task_result_embd>();
         res->id    = slot.id_task;
         res->index = slot.index;
+        res->n_prompt_tokens = slot.n_prompt_tokens;
 
         const int n_embd = llama_n_embd(model);
 

diff --git a/examples/server/utils.hpp b/examples/server/utils.hpp
@@ -560,21 +560,24 @@ static json oaicompat_completion_params_parse(
 
 static json format_embeddings_response_oaicompat(const json & request, const json & embeddings) {
     json data = json::array();
+    int32_t n_prompt_tokens = 0;
     int i = 0;
     for (const auto & elem : embeddings) {
         data.push_back(json{
             {"embedding", json_value(elem, "embedding", json::array())},
             {"index",     i++},
             {"object",    "embedding"}
         });
+
+        n_prompt_tokens += json_value(elem, "tokens_evaluated", 0);
     }
 
     json res = json {
         {"model", json_value(request, "model", std::string(DEFAULT_OAICOMPAT_MODEL))},
         {"object", "list"},
-        {"usage", json { // TODO: fill
-            {"prompt_tokens", 0},
-            {"total_tokens", 0}
+        {"usage", json {
+            {"prompt_tokens", n_prompt_tokens},
+            {"total_tokens", n_prompt_tokens}
         }},
         {"data", data}
     };