From 53092d01560db72abd7ed71e9e93a5217cca4ebd Mon Sep 17 00:00:00 2001 From: aoife cassidy Date: Fri, 24 Jan 2025 12:36:57 +0200 Subject: [PATCH 1/3] fix(proc): clearer errors on crash (#271) --- .changeset/afraid-camels-listen.md | 5 +++++ agents/src/ipc/proc_job_executor.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .changeset/afraid-camels-listen.md diff --git a/.changeset/afraid-camels-listen.md b/.changeset/afraid-camels-listen.md new file mode 100644 index 00000000..8865f5cd --- /dev/null +++ b/.changeset/afraid-camels-listen.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +fix(proc): clearer errors on crash diff --git a/agents/src/ipc/proc_job_executor.ts b/agents/src/ipc/proc_job_executor.ts index 980a35b5..03f619b7 100644 --- a/agents/src/ipc/proc_job_executor.ts +++ b/agents/src/ipc/proc_job_executor.ts @@ -96,7 +96,7 @@ export class ProcJobExecutor extends JobExecutor { this.#proc!.on('message', listener); this.#proc!.on('error', (err) => { if (this.#closing) return; - this.#logger.child({ err }).warn('job process exited unexpectedly'); + this.#logger.child({ err }).warn('job process exited unexpectedly; this likely means the error above caused a crash'); clearTimeout(this.#pongTimeout); clearInterval(this.#pingInterval); this.#join.resolve(); From 827cf3e3662020f28cc6059a94045961dc342a53 Mon Sep 17 00:00:00 2001 From: aoife cassidy Date: Mon, 27 Jan 2025 12:29:38 +0200 Subject: [PATCH 2/3] fix(metrics): fix MultimodalLLMError typescript (#278) --- .changeset/clean-garlics-cross.md | 5 +++++ agents/src/ipc/proc_job_executor.ts | 4 +++- agents/src/metrics/base.ts | 17 +++++++---------- 3 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 .changeset/clean-garlics-cross.md diff --git a/.changeset/clean-garlics-cross.md b/.changeset/clean-garlics-cross.md new file mode 100644 index 00000000..0b2ace77 --- /dev/null +++ b/.changeset/clean-garlics-cross.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents": patch +--- + +fix(metrics): remove ErrorOptions from MultimodalError diff --git a/agents/src/ipc/proc_job_executor.ts b/agents/src/ipc/proc_job_executor.ts index 03f619b7..c4a34209 100644 --- a/agents/src/ipc/proc_job_executor.ts +++ b/agents/src/ipc/proc_job_executor.ts @@ -96,7 +96,9 @@ export class ProcJobExecutor extends JobExecutor { this.#proc!.on('message', listener); this.#proc!.on('error', (err) => { if (this.#closing) return; - this.#logger.child({ err }).warn('job process exited unexpectedly; this likely means the error above caused a crash'); + this.#logger + .child({ err }) + .warn('job process exited unexpectedly; this likely means the error above caused a crash'); clearTimeout(this.#pongTimeout); clearInterval(this.#pingInterval); this.#join.resolve(); diff --git a/agents/src/metrics/base.ts b/agents/src/metrics/base.ts index 3252027b..626f2306 100644 --- a/agents/src/metrics/base.ts +++ b/agents/src/metrics/base.ts @@ -86,16 +86,13 @@ export class MultimodalLLMError extends Error { type?: string; reason?: string; code?: string; - constructor( - { - type, - reason, - code, - message, - }: { type?: string; reason?: string; code?: string; message?: string } = {}, - options?: ErrorOptions, - ) { - super(message, options); + constructor({ + type, + reason, + code, + message, + }: { type?: string; reason?: string; code?: string; message?: string } = {}) { + super(message); this.type = type; this.reason = reason; this.code = code; From e0ae7ce2a5461ea54719d804ac4a1004807be14b Mon Sep 17 00:00:00 2001 From: aoife cassidy Date: Tue, 28 Jan 2025 12:46:30 +0200 Subject: [PATCH 3/3] fix(elevenlabs): fix errors capturing frames (#280) --- .changeset/lazy-seals-camp.md | 5 +++ .envrc | 2 -- flake.lock | 61 ----------------------------------- flake.nix | 24 -------------- plugins/elevenlabs/src/tts.ts | 8 ++++- 5 files changed, 12 insertions(+), 88 deletions(-) create mode 100644 .changeset/lazy-seals-camp.md delete mode 100644 .envrc delete mode 100644 flake.lock delete mode 100644 flake.nix diff --git a/.changeset/lazy-seals-camp.md b/.changeset/lazy-seals-camp.md new file mode 100644 index 00000000..627876e0 --- /dev/null +++ b/.changeset/lazy-seals-camp.md @@ -0,0 +1,5 @@ +--- +"@livekit/agents-plugin-elevenlabs": patch +--- + +fix errors capturing frames diff --git a/.envrc b/.envrc deleted file mode 100644 index a96880d7..00000000 --- a/.envrc +++ /dev/null @@ -1,2 +0,0 @@ -use flake -dotenv_if_exists diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 74140bbc..00000000 --- a/flake.lock +++ /dev/null @@ -1,61 +0,0 @@ -{ - "nodes": { - "nixpkgs": { - "locked": { - "lastModified": 1725194671, - "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs", - "utils": "utils" - } - }, - "systems": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } - }, - "utils": { - "inputs": { - "systems": "systems" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 57bf52f6..00000000 --- a/flake.nix +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: 2024 LiveKit, Inc. -# -# SPDX-License-Identifier: Apache-2.0 - -{ - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; - utils.url = "github:numtide/flake-utils"; - }; - - outputs = { self, utils, nixpkgs }: - utils.lib.eachDefaultSystem (system: - let pkgs = (import nixpkgs) { - inherit system; - }; - - in { - devShell = with pkgs; mkShell { - nativeBuildInputs = [ nodejs corepack reuse turbo ]; - LD_LIBRARY_PATH = "${pkgs.stdenv.cc.cc.lib}/lib/"; - }; - } - ); -} diff --git a/plugins/elevenlabs/src/tts.ts b/plugins/elevenlabs/src/tts.ts index ae6799da..eb80664c 100644 --- a/plugins/elevenlabs/src/tts.ts +++ b/plugins/elevenlabs/src/tts.ts @@ -267,7 +267,7 @@ export class SynthesizeStream extends tts.SynthesizeStream { }).then((msg) => { const json = JSON.parse(msg.toString()); if ('audio' in json) { - const data = new Int8Array(Buffer.from(json.audio, 'base64').buffer); + const data = new Int8Array(Buffer.from(json.audio, 'base64')); for (const frame of bstream.write(data)) { sendLastFrame(segmentId, false); lastFrame = frame; @@ -278,6 +278,12 @@ export class SynthesizeStream extends tts.SynthesizeStream { lastFrame = frame; } sendLastFrame(segmentId, true); + this.queue.put(SynthesizeStream.END_OF_STREAM); + + if (segmentId === requestId) { + ws.close(); + return; + } } }); } catch {