Add custom delay (#31)

* Add custom delay

* Configure build scripts

* Support default delay

* Remove Langauge export

* Update tests

* Update vitest

* Add pulse audio

.github/workflows/test.yml

@@ -33,6 +33,13 @@ jobs:
       - name: Install dependencies
         run: pnpm i
 
+      - name: Install pulseaudio
+        run: |
+          apt-get update
+          apt-get install -y pulseaudio
+          apt-get install sudo
+          sudo pulseaudio --start
+
       - name: Run tests
         run: pnpm run test
         env:

package.json

@@ -17,5 +17,11 @@
     "fs-extra": "^11.2.0",
     "husky": "^9.1.3",
     "prettier": "^3.3.3"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "esbuild",
+      "msw"
+    ]
   }
 }

packages/client/README.md

@@ -169,6 +169,22 @@ const conversation = await Conversation.startSession({
 });
 ```
 
+#### Connection delay
+
+You can configure additional delay between when the microphone is activated and when the connection is established.
+On Android, the delay is set to 3 seconds by default to make sure the device has time to switch to the correct audio mode.
+Without it, you may experience issues with the beginning of the first message being cut off.
+
+```ts
+const conversation = await Conversation.startSession({
+  connectionDelay: {
+    android: 3_000,
+    ios: 0,
+    default: 0,
+  },
+});
+```
+
 #### Return value
 
 `startSession` returns a `Conversation` instance that can be used to control the session. The method will throw an error if the session cannot be established. This can happen if the user denies microphone access, or if the websocket connection

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@11labs/client",
-  "version": "0.0.6",
+  "version": "0.0.7-beta.1",
   "description": "ElevenLabs JavaScript Client Library",
   "main": "./dist/lib.umd.js",
   "module": "./dist/lib.module.js",
@@ -31,14 +31,14 @@
   "license": "MIT",
   "devDependencies": {
     "@types/node-wav": "^0.0.3",
-    "@vitest/browser": "^2.0.5",
+    "@vitest/browser": "^3.0.5",
     "eslint": "^9.8.0",
     "microbundle": "^0.15.1",
     "mock-socket": "^9.3.1",
     "node-wav": "^0.0.2",
     "playwright": "^1.46.1",
     "typescript": "^5.5.4",
-    "vitest": "^2.0.5"
+    "vitest": "^3.0.5"
   },
   "repository": {
     "type": "git",

packages/client/src/index.test.ts

@@ -55,6 +55,7 @@ describe("Conversation", () => {
         status = value.status;
       },
       onUnhandledClientToolCall,
+      connectionDelay: { default: 0 },
     });
     const client = await clientPromise;
 
@@ -190,6 +191,7 @@ describe("Conversation", () => {
     await expect(async () => {
       await Conversation.startSession({
         signedUrl: "wss://api.elevenlabs.io/2",
+        connectionDelay: { default: 0 },
       });
       await clientPromise;
     }).rejects.toThrowError(
@@ -212,6 +214,7 @@ describe("Conversation", () => {
       Conversation.startSession({
         signedUrl: "wss://api.elevenlabs.io/3",
         onDisconnect: resolve,
+        connectionDelay: { default: 0 },
       });
       setTimeout(() => reject(new Error("timeout")), 5000);
     });

packages/client/src/index.ts

@@ -8,6 +8,7 @@ import {
   SessionConfig,
 } from "./utils/connection";
 import { ClientToolCallEvent, IncomingSocketEvent } from "./utils/events";
+import { isAndroidDevice, isIosDevice } from "./utils/compatibility";
 
 export type { IncomingSocketEvent } from "./utils/events";
 export type { SessionConfig, DisconnectionDetails } from "./utils/connection";
@@ -78,14 +79,30 @@ export class Conversation {
     let input: Input | null = null;
     let connection: Connection | null = null;
     let output: Output | null = null;
+    let preliminaryInputStream: MediaStream | null = null;
 
     try {
       // some browsers won't allow calling getSupportedConstraints or enumerateDevices
       // before getting approval for microphone access
-      const preliminaryInputStream = await navigator.mediaDevices.getUserMedia({
+      preliminaryInputStream = await navigator.mediaDevices.getUserMedia({
         audio: true,
       });
-      preliminaryInputStream?.getTracks().forEach(track => track.stop());
+
+      const delayConfig = options.connectionDelay ?? {
+        default: 0,
+        // Give the Android AudioManager enough time to switch to the correct audio mode
+        android: 3_000,
+      };
+      let delay = delayConfig.default;
+      if (isAndroidDevice()) {
+        delay = delayConfig.android ?? delay;
+      } else if (isIosDevice()) {
+        delay = delayConfig.ios ?? delay;
+      }
+
+      if (delay > 0) {
+        await new Promise(resolve => setTimeout(resolve, delay));
+      }
 
       connection = await Connection.create(options);
       [input, output] = await Promise.all([
@@ -96,9 +113,13 @@ export class Conversation {
         Output.create(connection.outputFormat),
       ]);
 
+      preliminaryInputStream?.getTracks().forEach(track => track.stop());
+      preliminaryInputStream = null;
+
       return new Conversation(fullOptions, connection, input, output);
     } catch (error) {
       fullOptions.onStatusChange({ status: "disconnected" });
+      preliminaryInputStream?.getTracks().forEach(track => track.stop());
       connection?.close();
       await input?.close();
       await output?.close();

packages/client/src/utils/compatibility.ts

@@ -0,0 +1,18 @@
+export function isIosDevice() {
+  return (
+    [
+      "iPad Simulator",
+      "iPhone Simulator",
+      "iPod Simulator",
+      "iPad",
+      "iPhone",
+      "iPod",
+    ].includes(navigator.platform) ||
+    // iPad on iOS 13 detection
+    (navigator.userAgent.includes("Mac") && "ontouchend" in document)
+  );
+}
+
+export function isAndroidDevice() {
+  return /android/i.test(navigator.userAgent);
+}

packages/client/src/utils/connection.ts

@@ -55,6 +55,11 @@ export type SessionConfig = {
   };
   customLlmExtraBody?: any;
   dynamicVariables?: Record<string, string | number | boolean>;
+  connectionDelay?: {
+    default: number;
+    android?: number;
+    ios?: number;
+  };
 } & (
   | { signedUrl: string; agentId?: undefined }
   | { agentId: string; signedUrl?: undefined }

packages/client/src/utils/input.ts

@@ -1,5 +1,6 @@
 import { rawAudioProcessor } from "./rawAudioProcessor";
 import { FormatConfig } from "./connection";
+import { isIosDevice } from "./compatibility";
 
 export type InputConfig = {
   preferHeadphonesForIosDevices?: boolean;
@@ -8,21 +9,6 @@ export type InputConfig = {
 const LIBSAMPLERATE_JS =
   "https://cdn.jsdelivr.net/npm/@alexanderolsen/[email protected]/dist/libsamplerate.worklet.js";
 
-function isIosDevice() {
-  return (
-    [
-      "iPad Simulator",
-      "iPhone Simulator",
-      "iPod Simulator",
-      "iPad",
-      "iPhone",
-      "iPod",
-    ].includes(navigator.platform) ||
-    // iPad on iOS 13 detection
-    (navigator.userAgent.includes("Mac") && "ontouchend" in document)
-  );
-}
-
 export class Input {
   public static async create({
     sampleRate,
@@ -79,6 +65,8 @@ export class Input {
       source.connect(analyser);
       analyser.connect(worklet);
 
+      await context.resume();
+
       return new Input(context, analyser, worklet, inputStream);
     } catch (error) {
       inputStream?.getTracks().forEach(track => track.stop());

packages/client/src/utils/output.ts

@@ -18,6 +18,8 @@ export class Output {
       worklet.port.postMessage({ type: "setFormat", format });
       worklet.connect(gain);
 
+      await context.resume();
+
       return new Output(context, analyser, gain, worklet);
     } catch (error) {
       context?.close();

packages/client/vitest.workspace.ts

@@ -1,43 +1,39 @@
+/// <reference types="@vitest/browser/providers/playwright" />
+
 import { defineWorkspace } from "vitest/config";
 
 export default defineWorkspace([
   {
     test: {
-      name: "Chromium",
+      name: "Browser tests",
       browser: {
         provider: "playwright",
         enabled: true,
-        name: "chromium",
-        providerOptions: {
-          launch: {
-            args: [
-              "--use-fake-device-for-media-stream",
-              "--use-fake-ui-for-media-stream",
-            ],
-          },
-          context: {
-            permissions: ["microphone"],
+        instances: [
+          {
+            browser: "chromium",
+            launch: {
+              args: [
+                "--use-fake-device-for-media-stream",
+                "--use-fake-ui-for-media-stream",
+              ],
+            },
+            context: {
+              permissions: ["microphone"],
+            },
           },
-        },
-      },
-    },
-  },
-  {
-    test: {
-      name: "Firefox",
-      browser: {
-        provider: "playwright",
-        enabled: true,
-        name: "firefox",
-        providerOptions: {
-          launch: {
-            firefoxUserPrefs: {
-              "permissions.default.microphone": 1,
-              "media.navigator.streams.fake": true,
-              "media.navigator.permission.disabled": true,
+          {
+            browser: "firefox",
+            headless: true,
+            launch: {
+              firefoxUserPrefs: {
+                "permissions.default.microphone": 1,
+                "media.navigator.streams.fake": true,
+                "media.navigator.permission.disabled": true,
+              },
             },
           },
-        },
+        ],
       },
     },
   },

packages/react/README.md

@@ -119,6 +119,22 @@ const conversation = useConversation({
 });
 ```
 
+#### Connection delay
+
+You can configure additional delay between when the microphone is activated and when the connection is established.
+On Android, the delay is set to 3 seconds by default to make sure the device has time to switch to the correct audio mode.
+Without it, you may experience issues with the beginning of the first message being cut off.
+
+```ts
+const conversation = useConversation({
+  connectionDelay: {
+    android: 3_000,
+    ios: 0,
+    default: 0,
+  },
+});
+```
+
 #### Methods
 
 ##### startConversation

packages/react/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@11labs/react",
-  "version": "0.0.6",
+  "version": "0.0.7-beta.1",
   "description": "ElevenLabs React Library",
   "main": "./dist/lib.umd.js",
   "module": "./dist/lib.module.js",