feat: prepare support for loading and error states in speech synthesis (

#726)
Yonom · Sep 3, 2024 · e52f337 · e52f337
1 parent 56fda5a
commit e52f337
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 19 deletions.
diff --git a/packages/react/src/context/stores/MessageUtils.ts b/packages/react/src/context/stores/MessageUtils.ts
@@ -26,7 +26,7 @@ export const makeMessageUtilsStore = () =>
       },
       isSpeaking: false,
       stopSpeaking: () => {
-        utterance?.stop();
+        utterance?.cancel();
       },
       addUtterance: (utt) => {
         utterance = utt;

diff --git a/packages/react/src/runtimes/local/LocalThreadRuntime.tsx b/packages/react/src/runtimes/local/LocalThreadRuntime.tsx
@@ -300,7 +300,7 @@ export class LocalThreadRuntime implements ThreadRuntime {
     const { message } = this.repository.getMessage(messageId);
 
     if (this._utterance) {
-      this._utterance.stop();
+      this._utterance.cancel();
       this._utterance = undefined;
     }
 

diff --git a/packages/react/src/runtimes/speech/SpeechAdapterTypes.ts b/packages/react/src/runtimes/speech/SpeechAdapterTypes.ts
@@ -1,8 +1,19 @@
 import { ThreadMessage, Unsubscribe } from "../../types";
 
 export namespace SpeechSynthesisAdapter {
+  export type Status =
+    | {
+        type: "starting" | "running";
+      }
+    | {
+        type: "ended";
+        reason: "finished" | "cancelled" | "error";
+        error?: unknown;
+      };
+
   export type Utterance = {
-    stop: () => void;
+    status: Status;
+    cancel: () => void;
     onEnd: (callback: () => void) => Unsubscribe;
   };
 }
@@ -12,13 +23,29 @@ export type SpeechSynthesisAdapter = {
 };
 
 export namespace SpeechRecognitionAdapter {
-  export type Status = {
-    type: "stopped" | "starting" | "running";
+  export type Status =
+    | {
+        type: "starting" | "running";
+      }
+    | {
+        type: "ended";
+        reason: "stopped" | "cancelled" | "error";
+      };
+
+  export type Result = {
+    transcript: string;
+  };
+
+  export type Session = {
+    status: Status;
+    stop: () => Promise<void>;
+    cancel: () => void;
+    onSpeechStart: (callback: () => void) => Unsubscribe;
+    onSpeechEnd: (callback: (result: Result) => void) => Unsubscribe;
+    onSpeech: (callback: (result: Result) => void) => Unsubscribe;
   };
 }
+
 export type SpeechRecognitionAdapter = {
-  status: SpeechRecognitionAdapter.Status;
-  start: () => Promise<void>;
-  stop: () => Promise<void>;
-  onMessage: (callback: (message: ThreadMessage) => void) => Unsubscribe;
+  listen: () => SpeechRecognitionAdapter.Session;
 };
diff --git a/packages/react/src/runtimes/speech/WebSpeechSynthesisAdapter.ts b/packages/react/src/runtimes/speech/WebSpeechSynthesisAdapter.ts
@@ -7,27 +7,30 @@ export class WebSpeechSynthesisAdapter implements SpeechSynthesisAdapter {
     const text = getThreadMessageText(message);
     const utterance = new SpeechSynthesisUtterance(text);
 
-    let ended = false;
     const endHandlers = new Set<() => void>();
-    const handleEnd = () => {
-      if (ended) return;
+    const handleEnd = (
+      reason: "finished" | "error" | "cancelled",
+      error?: unknown,
+    ) => {
+      if (res.status.type === "ended") return;
 
-      ended = true;
+      res.status = { type: "ended", reason, error };
       endHandlers.forEach((handler) => handler());
     };
 
-    utterance.addEventListener("end", handleEnd);
-    utterance.addEventListener("error", handleEnd);
+    utterance.addEventListener("end", () => handleEnd("finished"));
+    utterance.addEventListener("error", (e) => handleEnd("error", e.error));
 
     window.speechSynthesis.speak(utterance);
 
-    return {
-      stop: () => {
+    const res: SpeechSynthesisAdapter.Utterance = {
+      status: { type: "running" },
+      cancel: () => {
         window.speechSynthesis.cancel();
-        handleEnd();
+        handleEnd("cancelled");
       },
       onEnd: (callback) => {
-        if (ended) {
+        if (res.status.type === "ended") {
           let cancelled = false;
           queueMicrotask(() => {
             if (!cancelled) callback();
@@ -43,5 +46,6 @@ export class WebSpeechSynthesisAdapter implements SpeechSynthesisAdapter {
         }
       },
     };
+    return res;
   }
 }