feat(elevenlabs): add tts (wip)

examples/src/tts.ts

@@ -1,43 +1,49 @@
 // SPDX-FileCopyrightText: 2024 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { type JobContext, WorkerOptions, cli, defineAgent, log } from '@livekit/agents';
+import { type JobContext, WorkerOptions, cli, defineAgent } from '@livekit/agents';
 import { TTS } from '@livekit/agents-plugin-elevenlabs';
-import { AudioSource, LocalAudioTrack, TrackPublishOptions, TrackSource } from '@livekit/rtc-node';
-import { fileURLToPath } from 'url';
-
-if (process.argv[1] === fileURLToPath(import.meta.url)) {
-  cli.runApp(new WorkerOptions({ agent: import.meta.filename }));
-}
+import {
+  AudioSource,
+  LocalAudioTrack,
+  RoomEvent,
+  TrackPublishOptions,
+  TrackSource,
+} from '@livekit/rtc-node';
+import { fileURLToPath } from 'node:url';
 
 export default defineAgent({
   entry: async (ctx: JobContext) => {
     await ctx.connect();
-    log().info('starting TTS example agent');
 
-    const source = new AudioSource(24000, 1);
+    console.log('starting TTS example agent');
+
+    const source = new AudioSource(22050, 1);
     const track = LocalAudioTrack.createAudioTrack('agent-mic', source);
     const options = new TrackPublishOptions();
     options.source = TrackSource.SOURCE_MICROPHONE;
+
     await ctx.room.localParticipant?.publishTrack(track, options);
+    const stream = new TTS().stream();
+
+    ctx.room.on(RoomEvent.LocalTrackSubscribed, async () => {
+      console.log('speaking "Hello!"');
+      stream.pushText('Hello!');
+      stream.flush();
 
-    const tts = new TTS();
-    log().info('speaking "Hello!"');
-    await tts
-      .synthesize('Hello!')
-      .then((output) => output.collect())
-      .then((output) => {
-        source.captureFrame(output);
-      });
-
-    await new Promise((resolve) => setTimeout(resolve, 1000));
-
-    log().info('speaking "Goodbye."');
-    await tts
-      .synthesize('Goodbye.')
-      .then((output) => output.collect())
-      .then((output) => {
-        source.captureFrame(output);
-      });
+      await new Promise<void>((resolve) => setTimeout(resolve, 2000));
+
+      console.log('speaking "Goodbye!"');
+      stream.pushText('Goodbye!');
+      stream.flush();
+      // stream.endInput();
+    });
+
+    for await (const audio of stream) {
+      console.log(audio);
+      await source.captureFrame(audio.frame);
+    }
   },
 });
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/elevenlabs/package.json

@@ -4,10 +4,12 @@
   "description": "ElevenLabs plugin for LiveKit Node Agents",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
-  "author": "aoife cassidy <[email protected]>",
+  "author": "LiveKit",
   "type": "module",
   "scripts": {
     "build": "tsc",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
     "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
     "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
     "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
@@ -19,7 +21,7 @@
   },
   "dependencies": {
     "@livekit/agents": "workspace:*",
-    "@livekit/rtc-node": "^0.8.1",
+    "@livekit/rtc-node": "^0.10.2",
     "ws": "^8.16.0"
   }
 }

plugins/elevenlabs/src/models.ts

@@ -6,4 +6,16 @@ export type TTSModels =
   | 'eleven_monolingual_v1'
   | 'eleven_multilingual_v1'
   | 'eleven_multilingual_v2'
-  | 'eleven_turbo_v2';
+  | 'eleven_turbo_v2'
+  | 'eleven_turbo_v2_5';
+
+export type TTSEncoding =
+  | 'mp3_22050_32'
+  | 'mp3_44100_32'
+  | 'mp3_44100_64'
+  | 'mp3_44100_96'
+  | 'mp3_44100_128'
+  | 'mp3_44100_192'
+  | 'pcm_16000'
+  | 'pcm_22050'
+  | 'pcm_44100';