From 4d9c86401ade0082868bd982e63de4239c6bae20 Mon Sep 17 00:00:00 2001 From: Guoqing Bao Date: Wed, 24 Jul 2024 17:12:18 +0800 Subject: [PATCH] Fix bug for non-streaming generation. --- src/openai/openai_server.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openai/openai_server.rs b/src/openai/openai_server.rs index 0561146..25c6a67 100644 --- a/src/openai/openai_server.rs +++ b/src/openai/openai_server.rs @@ -219,6 +219,7 @@ pub async fn chat_completions( ) } else { // wait until current response finished + tokio::time::sleep(Duration::from_millis(100)).await; //permits generation thread to work let model = data.model.lock().await; let choices = &model.completion_records[&request_id].0; let usage = &model.completion_records[&request_id].1;