From 0b9740d8ea9e93163dd9154b32b04234fe08017d Mon Sep 17 00:00:00 2001
From: gtygo <gtydoit@gmail.com>
Date: Thu, 15 Aug 2024 15:40:12 +0800
Subject: [PATCH] retrieval : fix memory leak in retrieval query handling
 (#8955)

* retrieval

* Reuse querybatch to reduce frequent memory allocation

* delete unused white space
---
 examples/retrieval/retrieval.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/retrieval/retrieval.cpp b/examples/retrieval/retrieval.cpp
index 65b19ce71cbe3..aab9d81058af9 100644
--- a/examples/retrieval/retrieval.cpp
+++ b/examples/retrieval/retrieval.cpp
@@ -253,6 +253,8 @@ int main(int argc, char ** argv) {
         chunks[i].tokens.clear();
     }
 
+    struct llama_batch query_batch = llama_batch_init(n_batch, 0, 1);
+
     // start loop, receive query and return top k similar chunks based on cosine similarity
     std::string query;
     while (true) {
@@ -260,7 +262,6 @@ int main(int argc, char ** argv) {
         std::getline(std::cin, query);
         std::vector<int32_t> query_tokens = llama_tokenize(ctx, query, true);
 
-        struct llama_batch query_batch = llama_batch_init(n_batch, 0, 1);
         batch_add_seq(query_batch, query_tokens, 0);
 
         std::vector<float> query_emb(n_embd, 0);
@@ -293,6 +294,7 @@ int main(int argc, char ** argv) {
     }
 
     // clean up
+    llama_batch_free(query_batch);
     llama_print_timings(ctx);
     llama_free(ctx);
     llama_free_model(model);