feat(elevenlabs): re-add (#56)

.changeset/breezy-books-behave.md

@@ -0,0 +1,6 @@
+---
+"@livekit/agents": patch
+"@livekit/agents-plugin-elevenlabs": minor
+---
+
+re-add ElevenLabs TTS plugin

agents/src/tokenize/basic/basic.ts

@@ -2,11 +2,11 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import * as tokenizer from '../index.js';
-import { BufferedSentenceStream } from '../token_stream.js';
+import { BufferedSentenceStream, BufferedWordStream } from '../token_stream.js';
 import { hyphenator } from './hyphenator.js';
 import { splitParagraphs } from './paragraph.js';
 import { splitSentences } from './sentence.js';
-import { splitWords } from './word';
+import { splitWords } from './word.js';
 
 interface TokenizerOptions {
   language: string;
@@ -41,7 +41,7 @@ export class SentenceTokenizer extends tokenizer.SentenceTokenizer {
   }
 }
 
-export class WordTokenizer extends tokenizer.SentenceTokenizer {
+export class WordTokenizer extends tokenizer.WordTokenizer {
   #ignorePunctuation: boolean;
 
   constructor(ignorePunctuation = true) {
@@ -55,8 +55,8 @@ export class WordTokenizer extends tokenizer.SentenceTokenizer {
   }
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  stream(language?: string): tokenizer.SentenceStream {
-    return new BufferedSentenceStream(
+  stream(language?: string): tokenizer.WordStream {
+    return new BufferedWordStream(
       (text: string) => splitWords(text, this.#ignorePunctuation),
       1,
       1,

agents/src/tokenize/index.ts

@@ -11,6 +11,6 @@ export {
   WordStream,
 } from './tokenizer.js';
 
-export { BufferedSentenceStream, BufferedTokenStream } from './token_stream.js';
+export { BufferedSentenceStream, BufferedTokenStream, BufferedWordStream } from './token_stream.js';
 
 export { basic };

agents/src/tokenize/token_stream.ts

@@ -39,14 +39,14 @@ export class BufferedTokenStream implements AsyncIterableIterator<TokenData> {
 
     while (true) {
       const tokens = this.#func(this.#inBuf);
-      if (tokens.length <= 1) break;
+      if (tokens.length === 0) break;
 
       if (this.#outBuf) this.#outBuf += ' ';
 
-      const tok = tokens.pop();
-      let tokText = tok! as string;
-      if (typeof tok! !== 'string') {
-        tokText = tok![0];
+      const tok = tokens.shift()!;
+      let tokText = tok as string;
+      if (tok.length > 1 && typeof tok[1] === 'number') {
+        tokText = tok[0];
       }
 
       this.#outBuf += tokText;
@@ -130,6 +130,11 @@ export class BufferedSentenceStream extends SentenceStream {
     this.#stream.pushText(text);
   }
 
+  close() {
+    super.close();
+    this.#stream.close();
+  }
+
   next(): Promise<IteratorResult<TokenData>> {
     return this.#stream.next();
   }
@@ -147,6 +152,11 @@ export class BufferedWordStream extends WordStream {
     this.#stream.pushText(text);
   }
 
+  close() {
+    super.close();
+    this.#stream.close();
+  }
+
   next(): Promise<IteratorResult<TokenData>> {
     return this.#stream.next();
   }

agents/src/tokenize/tokenizer.ts

@@ -28,14 +28,18 @@ export abstract class SentenceStream {
   protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
   protected input = new AsyncIterableQueue<string | typeof SentenceStream.FLUSH_SENTINEL>();
   protected queue = new AsyncIterableQueue<TokenData>();
-  protected closed = false;
+  #closed = false;
+
+  get closed(): boolean {
+    return this.#closed;
+  }
 
   /** Push a string of text to the tokenizer */
   pushText(text: string) {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.put(text);
@@ -46,7 +50,7 @@ export abstract class SentenceStream {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.put(SentenceStream.FLUSH_SENTINEL);
@@ -57,7 +61,7 @@ export abstract class SentenceStream {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.close();
@@ -71,7 +75,7 @@ export abstract class SentenceStream {
   close() {
     this.input.close();
     this.queue.close();
-    this.closed = true;
+    this.#closed = true;
   }
 
   [Symbol.asyncIterator](): SentenceStream {
@@ -92,14 +96,18 @@ export abstract class WordStream {
   protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
   protected input = new AsyncIterableQueue<string | typeof WordStream.FLUSH_SENTINEL>();
   protected queue = new AsyncIterableQueue<TokenData>();
-  protected closed = false;
+  #closed = false;
+
+  get closed(): boolean {
+    return this.#closed;
+  }
 
   /** Push a string of text to the tokenizer */
   pushText(text: string) {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.put(text);
@@ -110,7 +118,7 @@ export abstract class WordStream {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.put(WordStream.FLUSH_SENTINEL);
@@ -121,7 +129,7 @@ export abstract class WordStream {
     if (this.input.closed) {
       throw new Error('Input is closed');
     }
-    if (this.closed) {
+    if (this.#closed) {
       throw new Error('Stream is closed');
     }
     this.input.close();
@@ -135,7 +143,7 @@ export abstract class WordStream {
   close() {
     this.input.close();
     this.queue.close();
-    this.closed = true;
+    this.#closed = true;
   }
 
   [Symbol.asyncIterator](): WordStream {

agents/src/tts/tts.ts

@@ -13,7 +13,7 @@ export interface SynthesizedAudio {
   /** Synthesized audio frame */
   frame: AudioFrame;
   /** Current segment of the synthesized audio */
-  deltaText: string;
+  deltaText?: string;
 }
 
 /**

examples/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-elevenlabs": "workspace:*",
     "@livekit/agents-plugin-openai": "workspace:*",
     "@livekit/rtc-node": "^0.10.2",
     "zod": "^3.23.8"

examples/src/tts.ts

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type JobContext, WorkerOptions, cli, defineAgent } from '@livekit/agents';
+import { TTS } from '@livekit/agents-plugin-elevenlabs';
+import {
+  AudioSource,
+  LocalAudioTrack,
+  RoomEvent,
+  TrackPublishOptions,
+  TrackSource,
+} from '@livekit/rtc-node';
+import { fileURLToPath } from 'node:url';
+
+export default defineAgent({
+  entry: async (ctx: JobContext) => {
+    await ctx.connect();
+
+    console.log('starting TTS example agent');
+
+    const source = new AudioSource(22050, 1);
+    const track = LocalAudioTrack.createAudioTrack('agent-mic', source);
+    const options = new TrackPublishOptions();
+    options.source = TrackSource.SOURCE_MICROPHONE;
+
+    await ctx.room.localParticipant?.publishTrack(track, options);
+    const stream = new TTS().stream();
+
+    ctx.room.on(RoomEvent.LocalTrackSubscribed, async () => {
+      console.log('speaking "Hello!"');
+      stream.pushText('Hello!');
+      stream.flush();
+
+      await new Promise<void>((resolve) => setTimeout(resolve, 2000));
+
+      console.log('speaking "Goodbye!"');
+      stream.pushText('Goodbye!');
+      stream.flush();
+    });
+
+    for await (const audio of stream) {
+      await source.captureFrame(audio.frame);
+    }
+  },
+});
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/elevenlabs/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/elevenlabs/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@livekit/agents-plugin-elevenlabs",
+  "version": "0.1.0",
+  "description": "ElevenLabs plugin for LiveKit Node Agents",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "LiveKit",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "^0.10.2",
+    "ws": "^8.16.0"
+  }
+}

plugins/elevenlabs/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './tts.js';

plugins/elevenlabs/src/models.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type TTSModels =
+  | 'eleven_monolingual_v1'
+  | 'eleven_multilingual_v1'
+  | 'eleven_multilingual_v2'
+  | 'eleven_turbo_v2'
+  | 'eleven_turbo_v2_5';
+
+export type TTSEncoding =
+  // XXX(nbsp): MP3 is not yet supported
+  // | 'mp3_22050_32'
+  // | 'mp3_44100_32'
+  // | 'mp3_44100_64'
+  // | 'mp3_44100_96'
+  // | 'mp3_44100_128'
+  // | 'mp3_44100_192'
+  'pcm_16000' | 'pcm_22050' | 'pcm_44100';