server : throw an error when slot unavailable (ggerganov#4741)

danbev · Jan 3, 2024 · f2eb19b · f2eb19b
1 parent f3f62f0
commit f2eb19b
Showing 1 changed file with 9 additions and 0 deletions.
diff --git a/examples/server/public/completion.js b/examples/server/public/completion.js
@@ -95,6 +95,15 @@ export async function* llama(prompt, params = {}, config = {}) {
               break;
             }
           }
+          if (result.error) {
+            result.error = JSON.parse(result.error);
+            if (result.error.content.includes('slot unavailable')) {
+              // Throw an error to be caught by upstream callers
+              throw new Error('slot unavailable');
+            } else {
+              console.error(`llama.cpp error: ${result.error.content}`);
+            }
+          }
           if (result.error) {
             result.error = JSON.parse(result.error);
             console.error(`llama.cpp error: ${result.error.content}`);