diff --git a/Cargo.lock b/Cargo.lock index e101802..52fa6ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1888,6 +1888,7 @@ dependencies = [ "nnnoiseless", "opus", "rubato", + "rubato-audio-source", "send_safe", "serde", "serde_json", @@ -3084,6 +3085,15 @@ dependencies = [ "realfft", ] +[[package]] +name = "rubato-audio-source" +version = "0.1.0" +dependencies = [ + "insanity-core", + "log", + "rubato", +] + [[package]] name = "rusqlite" version = "0.32.1" diff --git a/Cargo.toml b/Cargo.toml index 9f2f6e1..9831333 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] -members = ["insanity-core", "insanity-native-tui-app", "insanity-tui-adapter"] +members = ["insanity-core", "insanity-native-tui-app", "insanity-tui-adapter", "rubato-audio-source"] resolver = "2" diff --git a/insanity-native-tui-app/Cargo.toml b/insanity-native-tui-app/Cargo.toml index 5e547eb..1363514 100644 --- a/insanity-native-tui-app/Cargo.toml +++ b/insanity-native-tui-app/Cargo.toml @@ -26,6 +26,7 @@ serde_json = "1.0.114" rubato = "0.10" insanity-core = { path = "../insanity-core" } insanity-tui-adapter = { path = "../insanity-tui-adapter" } +rubato-audio-source = { path = "../rubato-audio-source" } sha2 = "0.10.8" whoami = "1.5.1" sled = "0.34.7" diff --git a/insanity-native-tui-app/src/clerver.rs b/insanity-native-tui-app/src/clerver.rs index 16477c7..fd58e21 100644 --- a/insanity-native-tui-app/src/clerver.rs +++ b/insanity-native-tui-app/src/clerver.rs @@ -6,6 +6,7 @@ use cpal::traits::{HostTrait, StreamTrait}; use insanity_core::audio_source::AudioSource; use insanity_tui_adapter::AppEvent; use opus::{Application, Channels, Decoder, Encoder}; +use rubato_audio_source::ResampledAudioSource; use serde::{Deserialize, Serialize}; use tokio::sync::{broadcast, mpsc}; use veq::veq::VeqSessionAlias; @@ -14,7 +15,6 @@ use crate::{ client::{get_output_config, setup_output_stream}, processor::{AudioChunk, AudioFormat, AudioProcessor, AUDIO_CHUNK_SIZE}, protocol::ProtocolMessage, - resampler::ResampledAudioSource, server::make_audio_receiver, }; @@ -33,7 +33,7 @@ async fn run_audio_sender( let channels_count = audio_receiver.channels(); let channels = u16_to_channels(channels_count); - let mut audio_receiver = ResampledAudioSource::new(audio_receiver, 48000); + let mut audio_receiver = ResampledAudioSource::new(audio_receiver, 48000, AUDIO_CHUNK_SIZE); let mut encoder = Encoder::new(48000, channels, Application::Audio).unwrap(); let mut sequence_number = 0; diff --git a/insanity-native-tui-app/src/lib.rs b/insanity-native-tui-app/src/lib.rs index d43a6f1..3aa2ba5 100644 --- a/insanity-native-tui-app/src/lib.rs +++ b/insanity-native-tui-app/src/lib.rs @@ -5,6 +5,5 @@ pub mod managed_peer; pub mod processor; pub mod protocol; pub mod realtime_buffer; -pub mod resampler; pub mod room_handler; pub mod server; diff --git a/insanity-native-tui-app/src/processor.rs b/insanity-native-tui-app/src/processor.rs index 88259b8..06de3a8 100644 --- a/insanity-native-tui-app/src/processor.rs +++ b/insanity-native-tui-app/src/processor.rs @@ -8,10 +8,10 @@ use std::sync::{Arc, Mutex}; use cpal::{Sample, SampleRate}; use insanity_core::audio_source::SyncAudioSource; use nnnoiseless::DenoiseState; +use rubato_audio_source::ResampledAudioSource; use serde::{Deserialize, Serialize}; use crate::realtime_buffer::RealTimeBuffer; -use crate::resampler::ResampledAudioSource; use crate::server::RealtimeAudioSource; pub const AUDIO_CHUNK_SIZE: usize = 480; @@ -203,7 +203,8 @@ impl AudioProcessor<'_> { ) -> Self { let chunk_buffer = Arc::new(Mutex::new(RealTimeBuffer::new(10))); let audio_receiver = RealtimeAudioSource::new(chunk_buffer.clone(), 48000, 2); - let audio_receiver = ResampledAudioSource::new(audio_receiver, output_sample_rate.0); + let audio_receiver = + ResampledAudioSource::new(audio_receiver, output_sample_rate.0, AUDIO_CHUNK_SIZE); AudioProcessor { enable_denoise, diff --git a/rubato-audio-source/Cargo.toml b/rubato-audio-source/Cargo.toml new file mode 100644 index 0000000..1e936ce --- /dev/null +++ b/rubato-audio-source/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "rubato-audio-source" +version = "0.1.0" +edition = "2021" + +[dependencies] +insanity-core = { path = "../insanity-core" } +log = "0.4.22" +rubato = "0.10" diff --git a/insanity-native-tui-app/src/resampler.rs b/rubato-audio-source/src/lib.rs similarity index 85% rename from insanity-native-tui-app/src/resampler.rs rename to rubato-audio-source/src/lib.rs index a09caf7..1125e4a 100644 --- a/insanity-native-tui-app/src/resampler.rs +++ b/rubato-audio-source/src/lib.rs @@ -1,21 +1,20 @@ -use std::{collections::VecDeque, sync::Mutex}; +use std::collections::VecDeque; use insanity_core::audio_source::{AudioSource, SyncAudioSource}; use log::trace; use rubato::{Resampler, SincFixedIn}; -use crate::processor::AUDIO_CHUNK_SIZE; - pub struct ResampledAudioSource { - resampler: Mutex>, + resampler: SincFixedIn, resampled_buffer: VecDeque, original_samples_buffer: VecDeque, delegate: R, sample_rate: u32, + chunk_size: usize, } impl ResampledAudioSource { - pub fn new(delegate: R, sample_rate: u32) -> ResampledAudioSource { + pub fn new(delegate: R, sample_rate: u32, chunk_size: usize) -> ResampledAudioSource { let params = rubato::InterpolationParameters { sinc_len: 256, f_cutoff: 0.95, @@ -26,15 +25,16 @@ impl ResampledAudioSource { let resampler = SincFixedIn::::new( sample_rate as f64 / delegate.sample_rate() as f64, params, - AUDIO_CHUNK_SIZE, + chunk_size, delegate.channels() as usize, ); ResampledAudioSource { - resampler: Mutex::new(resampler), + resampler, resampled_buffer: VecDeque::new(), original_samples_buffer: VecDeque::new(), delegate, sample_rate, + chunk_size, } } } @@ -68,10 +68,10 @@ impl AudioSource for ResampledAudioSource { } if self.resampled_buffer.is_empty() { // First, try to fill the original_samples buffer with enough samples to resample - let target_samples_count = AUDIO_CHUNK_SIZE * self.delegate.channels() as usize; + let target_samples_count = self.chunk_size * self.delegate.channels() as usize; trace!( "Audio chunk size: {}, channels: {}, target samples count: {}", - AUDIO_CHUNK_SIZE, + self.chunk_size, self.delegate.channels(), target_samples_count ); @@ -91,8 +91,7 @@ impl AudioSource for ResampledAudioSource { let samples = self.original_samples_buffer.drain(..).collect::>(); let channels = separate_channels(&samples, self.delegate.channels() as usize); trace!("Separated into {} channels", channels.len()); - let mut resampler_guard = self.resampler.lock().unwrap(); - let resampled_channels = resampler_guard.process(&channels).unwrap(); + let resampled_channels = self.resampler.process(&channels).unwrap(); let resampled_samples = interleave_channels(&resampled_channels); self.resampled_buffer = resampled_samples.into(); } @@ -115,10 +114,10 @@ impl SyncAudioSource for ResampledAudioSource { } if self.resampled_buffer.is_empty() { // First, try to fill the original_samples buffer with enough samples to resample - let target_samples_count = AUDIO_CHUNK_SIZE * self.delegate.channels() as usize; + let target_samples_count = self.chunk_size * self.delegate.channels() as usize; trace!( "Audio chunk size: {}, channels: {}, target samples count: {}", - AUDIO_CHUNK_SIZE, + self.chunk_size, self.delegate.channels(), target_samples_count ); @@ -138,8 +137,7 @@ impl SyncAudioSource for ResampledAudioSource { let samples = self.original_samples_buffer.drain(..).collect::>(); let channels = separate_channels(&samples, self.delegate.channels() as usize); trace!("Separated into {} channels", channels.len()); - let mut resampler_guard = self.resampler.lock().unwrap(); - let resampled_channels = resampler_guard.process(&channels).unwrap(); + let resampled_channels = self.resampler.process(&channels).unwrap(); let resampled_samples = interleave_channels(&resampled_channels); self.resampled_buffer = resampled_samples.into(); }