diff --git a/screenpipe-actions/ai-proxy/src/index.ts b/screenpipe-actions/ai-proxy/src/index.ts index e77058b12..b7d17ae61 100644 --- a/screenpipe-actions/ai-proxy/src/index.ts +++ b/screenpipe-actions/ai-proxy/src/index.ts @@ -208,11 +208,11 @@ async function handleWebSocketUpgrade(request: Request, env: Env): Promise { + deepgramSocket.on(LiveTranscriptionEvents.Open, () => { server.send( JSON.stringify({ - type: 'message', - data: JSON.stringify(data), + type: 'connected', + message: 'WebSocket connection established', }) ); }); @@ -323,6 +323,12 @@ export default Sentry.withSentry( const url = new URL(request.url); const path = url.pathname; + const upgradeHeader = request.headers.get('upgrade')?.toLowerCase(); + if (path === '/v1/listen' && upgradeHeader === 'websocket') { + console.log('websocket request to /v1/listen detected, bypassing auth'); + return await handleWebSocketUpgrade(request, env); + } + // Add auth check for protected routes if (path !== '/test') { const authHeader = request.headers.get('Authorization'); @@ -484,11 +490,6 @@ export default Sentry.withSentry( } } - if (path === '/v1/listen' && request.headers.get('Upgrade') === 'websocket') { - console.log('websocket request'); - return await handleWebSocketUpgrade(request, env); - } - const response = new Response('not found', { status: 404, headers: { diff --git a/screenpipe-actions/ai-proxy/wrangler.toml b/screenpipe-actions/ai-proxy/wrangler.toml index 6f04bae04..91cee9176 100644 --- a/screenpipe-actions/ai-proxy/wrangler.toml +++ b/screenpipe-actions/ai-proxy/wrangler.toml @@ -5,6 +5,8 @@ compatibility_date = "2024-10-11" compatibility_flags = ["nodejs_compat"] upload_source_maps = true +tail_consumers = [{service = "ai-proxy-tail"}] + # Workers Logs # Docs: https://developers.cloudflare.com/workers/observability/logs/workers-logs/ # Configuration: https://developers.cloudflare.com/workers/observability/logs/workers-logs/#enable-workers-logs diff --git a/screenpipe-app-tauri/src-tauri/Cargo.lock b/screenpipe-app-tauri/src-tauri/Cargo.lock index 56d18fba9..3ba027ae1 100755 --- a/screenpipe-app-tauri/src-tauri/Cargo.lock +++ b/screenpipe-app-tauri/src-tauri/Cargo.lock @@ -2787,12 +2787,12 @@ dependencies = [ [[package]] name = "image" -version = "0.25.1" +version = "0.25.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" +checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" dependencies = [ "bytemuck", - "byteorder", + "byteorder-lite", "color_quant", "exr", "gif", @@ -2810,9 +2810,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" dependencies = [ "byteorder-lite", "quick-error", @@ -3329,6 +3329,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" dependencies = [ "cfg-if 1.0.0", + "rayon", ] [[package]] @@ -4874,6 +4875,7 @@ dependencies = [ "loop9", "quick-error", "rav1e", + "rayon", "rgb", ] @@ -5057,9 +5059,6 @@ name = "rgb" version = "0.8.50" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" -dependencies = [ - "bytemuck", -] [[package]] name = "ring" @@ -5275,7 +5274,7 @@ dependencies = [ [[package]] name = "screenpipe-app" -version = "0.29.6" +version = "0.30.3" dependencies = [ "anyhow", "async-stream", @@ -5289,6 +5288,7 @@ dependencies = [ "futures", "gtk", "http 0.2.12", + "image 0.25.5", "lazy_static", "libc", "log", @@ -5326,8 +5326,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "url", - "uuid", "windows-icons", "winreg 0.52.0", ] @@ -6136,7 +6134,7 @@ dependencies = [ "heck 0.5.0", "http 1.1.0", "http-range", - "image 0.25.1", + "image 0.25.5", "jni", "libc", "log", @@ -7744,7 +7742,7 @@ version = "0.1.1" source = "git+https://github.com/tribhuwan-kumar/windows-icons.git#f7db8e815b2d71c3b57bc6084b3451268be0d84a" dependencies = [ "base64 0.22.1", - "image 0.25.1", + "image 0.25.5", "regex", "tokio", "winapi", diff --git a/screenpipe-app-tauri/src-tauri/src/main.rs b/screenpipe-app-tauri/src-tauri/src/main.rs index d74983c76..fd0139963 100755 --- a/screenpipe-app-tauri/src-tauri/src/main.rs +++ b/screenpipe-app-tauri/src-tauri/src/main.rs @@ -22,7 +22,6 @@ use tauri::{ }; use tauri_plugin_autostart::MacosLauncher; use tauri_plugin_autostart::ManagerExt; -use tauri_plugin_deep_link::DeepLinkExt; use tauri_plugin_global_shortcut::ShortcutState; use tauri_plugin_notification::NotificationExt; #[allow(unused_imports)] @@ -358,24 +357,10 @@ async fn get_pipe_port(pipe_id: &str) -> anyhow::Result { .ok_or_else(|| anyhow::anyhow!("no port found for pipe {}", pipe_id)) } -async fn list_pipes() -> anyhow::Result { - let client = reqwest::Client::new(); - let response = client - .get("http://localhost:3030/pipes/list") - .send() - .await? - .json::() - .await?; - - Ok(response) -} - - pub fn get_base_dir( app: &tauri::AppHandle, custom_path: Option, ) -> anyhow::Result { - let default_path = app.path().local_data_dir().unwrap().join("screenpipe"); let local_data_dir = custom_path.map(PathBuf::from).unwrap_or(default_path); diff --git a/screenpipe-app-tauri/src-tauri/src/server.rs b/screenpipe-app-tauri/src-tauri/src/server.rs index ad7e8d80b..8e160f315 100644 --- a/screenpipe-app-tauri/src-tauri/src/server.rs +++ b/screenpipe-app-tauri/src-tauri/src/server.rs @@ -1,4 +1,4 @@ -use crate::{get_base_dir, get_store, icons::AppIcon}; +use crate::{get_base_dir, get_store}; use axum::body::Bytes; use axum::response::sse::{Event, Sse}; use axum::response::IntoResponse; diff --git a/screenpipe-app-tauri/src-tauri/src/sidecar.rs b/screenpipe-app-tauri/src-tauri/src/sidecar.rs index ab87df879..97176ce31 100644 --- a/screenpipe-app-tauri/src-tauri/src/sidecar.rs +++ b/screenpipe-app-tauri/src-tauri/src/sidecar.rs @@ -72,9 +72,7 @@ impl User { .get("user.credits.created_at") .and_then(|v| v.as_str().map(String::from)), }), - cloud_subscribed: store - .get("user.cloud_subscribed") - .and_then(|v| v.as_bool()), + cloud_subscribed: store.get("user.cloud_subscribed").and_then(|v| v.as_bool()), } } } @@ -303,7 +301,7 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { } } - if deepgram_api_key != "default" { + if deepgram_api_key != "default" && deepgram_api_key != "" { args.push("--deepgram-api-key"); let key = deepgram_api_key.as_str(); args.push(key); @@ -396,6 +394,7 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { // args.push("--debug"); + if cfg!(windows) { let mut c = app.shell().sidecar("screenpipe").unwrap(); if use_chinese_mirror { @@ -403,7 +402,7 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { } // if a user with credits is provided, add the AI proxy env var api url for deepgram as env var https://ai-proxy.i-f9f.workers.dev/v1/listen - if user.cloud_subscribed.is_some() { + if user.cloud_subscribed.is_some() && (deepgram_api_key == "default" || deepgram_api_key == "") { c = c.env( "DEEPGRAM_API_URL", "https://ai-proxy.i-f9f.workers.dev/v1/listen", @@ -412,6 +411,8 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { // Add token if screenpipe-cloud is selected and user has a token if user.id.is_some() { c = c.env("CUSTOM_DEEPGRAM_API_TOKEN", user.id.as_ref().unwrap()); + args.push("--deepgram-api-key"); + args.push(user.id.as_ref().unwrap()); } } @@ -436,7 +437,11 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { } // if a user with credits is provided, add the AI proxy env var api url for deepgram as env var https://ai-proxy.i-f9f.workers.dev/v1/listen - if user.cloud_subscribed.is_some() { + if user.cloud_subscribed.is_some() && (deepgram_api_key == "default" || deepgram_api_key == "") { + info!( + "Adding AI proxy env vars for deepgram: {:?}", + user.id.as_ref().unwrap() + ); c = c.env( "DEEPGRAM_API_URL", "https://ai-proxy.i-f9f.workers.dev/v1/listen", @@ -445,6 +450,8 @@ fn spawn_sidecar(app: &tauri::AppHandle) -> Result { // Add token if screenpipe-cloud is selected and user has a token if user.id.is_some() { c = c.env("CUSTOM_DEEPGRAM_API_TOKEN", user.id.as_ref().unwrap()); + args.push("--deepgram-api-key"); + args.push(user.id.as_ref().unwrap()); } } diff --git a/screenpipe-app-tauri/src-tauri/ui_monitor-aarch64-apple-darwin b/screenpipe-app-tauri/src-tauri/ui_monitor-aarch64-apple-darwin index 4f59805f1..9aba920f6 100755 Binary files a/screenpipe-app-tauri/src-tauri/ui_monitor-aarch64-apple-darwin and b/screenpipe-app-tauri/src-tauri/ui_monitor-aarch64-apple-darwin differ diff --git a/screenpipe-audio/src/deepgram/realtime.rs b/screenpipe-audio/src/deepgram/realtime.rs index 83f14fa0d..866ad365d 100644 --- a/screenpipe-audio/src/deepgram/realtime.rs +++ b/screenpipe-audio/src/deepgram/realtime.rs @@ -19,6 +19,7 @@ use std::sync::{atomic::AtomicBool, Arc}; use std::time::Duration; use tokio::sync::broadcast::Receiver; use tokio::sync::oneshot; +use tracing::info; pub async fn stream_transcription_deepgram( stream: Arc, @@ -67,6 +68,11 @@ pub async fn start_deepgram_stream( } }); + info!( + "Starting deepgram stream with api key: {:?} and url: {:?}", + api_key, DEEPGRAM_WEBSOCKET_URL.as_str() + ); + let deepgram = match DEEPGRAM_WEBSOCKET_URL.as_str().is_empty() { true => deepgram::Deepgram::new(api_key)?, false => {