Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

in progress add buffer processor

Browse source
This commit is contained in:
David Kleingeld 2025-08-25 16:13:09 +02:00
parent d76c7e1ba8
commit 3a5d02cb7d
7 changed files with 265 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

99
Cargo.lock generated
View file

@ -9628,6 +9628,7 @@ dependencies = [
"core-video",
"coreaudio-rs 0.12.1",
"cpal",
"dasp_sample",
"futures 0.3.31",
"gpui",
"gpui_tokio",
@ -13873,14 +13874,15 @@ dependencies = [
[[package]]
name = "rodio"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e40ecf59e742e03336be6a3d53755e789fd05a059fa22dfa0ed624722319e183"
source = "git+https://github.com/RustAudio/rodio?branch=microphone#9af0d470b35c6e1ec6bca88d87bbf4bc52f1f018"
dependencies = [
"cpal",
"dasp_sample",
"hound",
"num-rational",
"rtrb",
"symphonia",
"thiserror 2.0.12",
"tracing",
]
@ -13955,6 +13957,12 @@ dependencies = [
"zeroize",
]
[[package]]
name = "rtrb"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad8388ea1a9e0ea807e442e8263a699e7edcb320ecbcd21b4fa8ff859acce3ba"
[[package]]
name = "rules_library"
version = "0.1.0"
@ -15954,12 +15962,53 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-isomp4",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72e34f34298a7308d4397a6c7fbf5b84c5d491231ce3dd379707ba673ab3bd97"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdbf25b545ad0d3ee3e891ea643ad115aff4ca92f6aec472086b957a58522f70"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.5.4"
@ -15970,6 +16019,17 @@ dependencies = [
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a98765fb46a0a6732b007f7e2870c2129b6f78d87db7987e6533c8f164a9f30"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.5.4"
@ -15983,6 +16043,31 @@ dependencies = [
"log",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abfdf178d697e50ce1e5d9b982ba1b94c47218e03ec35022d9f0e071a16dc844"
dependencies = [
"encoding_rs",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ada3505789516bcf00fc1157c67729eded428b455c27ca370e41f4d785bfa931"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-riff"
version = "0.5.4"
@ -16007,6 +16092,16 @@ dependencies = [
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "484472580fa49991afda5f6550ece662237b00c6f562c7d9638d1b086ed010fe"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.109"

2
Cargo.toml
View file

@ -361,7 +361,7 @@ remote_server = { path = "crates/remote_server" }
repl = { path = "crates/repl" }
reqwest_client = { path = "crates/reqwest_client" }
rich_text = { path = "crates/rich_text" }
rodio = { version = "0.21.1", default-features = false }
rodio = { git = "https://github.com/RustAudio/rodio", branch = "microphone"}
rope = { path = "crates/rope" }
rpc = { path = "crates/rpc" }
rules_library = { path = "crates/rules_library" }

3
crates/livekit_client/Cargo.toml
View file

@ -41,7 +41,8 @@ tokio-tungstenite.workspace = true
util.workspace = true
workspace-hack.workspace = true
rodio = { workspace = true, features = ["wav_output"] }
rodio = { workspace = true, features = ["wav_output", "recording"] }
dasp_sample = "0.11"
[target.'cfg(not(any(all(target_os = "windows", target_env = "gnu"), target_os = "freebsd")))'.dependencies]
libwebrtc = { rev = "5f04705ac3f356350ae31534ffbc476abc9ea83d", git = "https://github.com/zed-industries/livekit-rust-sdks" }

66
crates/livekit_client/src/lib.rs
View file

@ -9,19 +9,19 @@ use rodio::DeviceTrait as _;
mod record;
pub use record::CaptureInput;
#[cfg(not(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu"),
target_os = "freebsd"
)))]
// #[cfg(not(any(
// test,
// feature = "test-support",
// all(target_os = "windows", target_env = "gnu"),
// target_os = "freebsd"
// )))]
mod livekit_client;
#[cfg(not(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu"),
target_os = "freebsd"
)))]
// #[cfg(not(any(
// test,
// feature = "test-support",
// all(target_os = "windows", target_env = "gnu"),
// target_os = "freebsd"
// )))]
pub use livekit_client::*;
// If you need proper LSP in livekit_client you've got to comment
@ -29,27 +29,27 @@ pub use livekit_client::*;
// - the mods: mock_client & test and their conditional blocks
// - the pub use mock_client::* and their conditional blocks
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu"),
target_os = "freebsd"
))]
mod mock_client;
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu"),
target_os = "freebsd"
))]
pub mod test;
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu"),
target_os = "freebsd"
))]
pub use mock_client::*;
// #[cfg(any(
// test,
// feature = "test-support",
// all(target_os = "windows", target_env = "gnu"),
// target_os = "freebsd"
// ))]
// mod mock_client;
// #[cfg(any(
// test,
// feature = "test-support",
// all(target_os = "windows", target_env = "gnu"),
// target_os = "freebsd"
// ))]
// pub mod test;
// #[cfg(any(
// test,
// feature = "test-support",
// all(target_os = "windows", target_env = "gnu"),
// target_os = "freebsd"
// ))]
// pub use mock_client::*;
#[derive(Debug, Clone)]
pub enum Participant {

46
crates/livekit_client/src/livekit_client/playback.rs
View file

@ -1,6 +1,8 @@
use anyhow::{Context as _, Result};
use cpal::Sample;
use cpal::traits::{DeviceTrait, StreamTrait as _};
use dasp_sample::ToSample;
use futures::channel::mpsc::UnboundedSender;
use futures::{Stream, StreamExt as _};
use gpui::{
@ -19,7 +21,9 @@ use livekit::webrtc::{
};
use parking_lot::Mutex;
use rodio::Source;
use rodio::source::{LimitSettings, UniformSourceIterator};
use std::cell::RefCell;
use std::num::NonZero;
use std::sync::Weak;
use std::sync::atomic::{AtomicBool, AtomicI32, Ordering};
use std::time::Duration;
@ -254,6 +258,48 @@ impl AudioStack {
}
}
async fn capture_input_rodio(
apm: Arc<Mutex<apm::AudioProcessingModule>>,
frame_tx: UnboundedSender<AudioFrame<'static>>,
sample_rate: u32,
num_channels: u32,
) -> Result<()> {
use crate::livekit_client::playback::source::RodioExt;
thread::spawn(|| {
let stream = rodio::microphone::MicrophoneBuilder::new()
.with_default_device()?
.with_default_config()?
.open_stream()?;
let stream = UniformSourceIterator::new(
stream,
NonZero::new(1).expect("1 is not zero"),
NonZero::new(SAMPLE_RATE).expect("constant is not zero"),
)
.limit(LimitSettings::live_performance())
.process_buffer(|buffer| {
let mut int_buffer: [i16; _] = buffer.map(|s| s.to_sample());
apm.lock()
.process_stream(&mut int_buffer, sample_rate as i32, num_channels as i32)
.unwrap(); // TODO dvdsk fix this
for (sample, processed) in buffer.iter().zip(&int_buffer) {
*sample = (*processed).to_sample_();
}
})
.automatic_gain_control(1.0, 4.0, 0.0, 5.0);
loop {
frame_tx.unbounded_send(AudioFrame {
data: Cow::Owned(sampled),
sample_rate,
num_channels,
samples_per_channel: sample_rate / 100,
})
}
});
Ok(())
}
async fn capture_input(
apm: Arc<Mutex<apm::AudioProcessingModule>>,
frame_tx: UnboundedSender<AudioFrame<'static>>,

81
crates/livekit_client/src/livekit_client/playback/source.rs
View file

@ -1,3 +1,5 @@
use std::num::NonZero;
use futures::StreamExt;
use libwebrtc::{audio_stream::native::NativeAudioStream, prelude::AudioFrame};
use livekit::track::RemoteAudioTrack;
@ -9,7 +11,11 @@ fn frame_to_samplesbuffer(frame: AudioFrame) -> SamplesBuffer {
let samples = frame.data.iter().copied();
let samples = SampleTypeConverter::<_, _>::new(samples);
let samples: Vec<f32> = samples.collect();
SamplesBuffer::new(frame.num_channels as u16, frame.sample_rate, samples)
SamplesBuffer::new(
NonZero::new(frame.num_channels as u16).expect("audio frame channels is nonzero"),
NonZero::new(frame.sample_rate).expect("audio frame sample rate is nonzero"),
samples,
)
}
pub struct LiveKitStream {
@ -65,3 +71,76 @@ impl Source for LiveKitStream {
self.inner.total_duration()
}
}
pub trait RodioExt: Source + Sized {
fn process_buffer<F>(
self,
callback: impl FnMut(&mut [rodio::Sample; 200]),
) -> ProcessBuffer<Self, F>
where
F: FnMut(&mut [rodio::Sample]);
}
impl<S: Source> RodioExt for S {
fn process_buffer<F>(
self,
callback: impl FnMut(&mut [rodio::Sample; 200]),
) -> ProcessBuffer<Self, F>
where
F: FnMut(&mut [rodio::Sample]),
{
ProcessBuffer {
inner: self,
callback,
in_buffer: [0.0; 200],
out_buffer: [0.0; 200],
}
}
}
pub struct ProcessBuffer<S, F>
where
S: Source + Sized,
F: FnMut(&mut [rodio::Sample; 200]),
{
inner: S,
callback: F,
in_buffer: [rodio::Sample; 200],
out_buffer: std::array::IntoIter<rodio::Sample, N>,
}
impl<S, F> Iterator for ProcessBuffer<S, F>
where
S: Source + Sized,
F: FnMut(&mut [rodio::Sample; 200]),
{
type Item = rodio::Sample;
fn next(&mut self) -> Option<Self::Item> {
for sample in &mut in_buffer {
*sample = self.inner.next()?;
}
}
}
impl<S, F> Source for ProcessBuffer<S, F>
where
S: Source + Sized,
F: FnMut(&mut [rodio::Sample; 200]),
{
fn current_span_len(&self) -> Option<usize> {
todo!()
}
fn channels(&self) -> rodio::ChannelCount {
todo!()
}
fn sample_rate(&self) -> rodio::SampleRate {
todo!()
}
fn total_duration(&self) -> Option<std::time::Duration> {
todo!()
}
}

7
crates/livekit_client/src/record.rs
View file

@ -1,5 +1,6 @@
use std::{
env,
num::NonZero,
path::{Path, PathBuf},
sync::{Arc, Mutex},
time::Duration,
@ -83,7 +84,11 @@ fn write_out(
.expect("Stream has ended, callback cant hold the lock"),
);
let samples: Vec<f32> = SampleTypeConverter::<_, f32>::new(samples.into_iter()).collect();
let mut samples = SamplesBuffer::new(config.channels(), config.sample_rate().0, samples);
let mut samples = SamplesBuffer::new(
NonZero::new(config.channels()).expect("config channel is never zero"),
NonZero::new(config.sample_rate().0).expect("config sample_rate is never zero"),
samples,
);
match rodio::output_to_wav(&mut samples, path) {
Ok(_) => Ok(()),
Err(e) => Err(anyhow::anyhow!("Failed to write wav file: {}", e)),