Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Switch fully to Rust Livekit (redux) (#27126)

Browse source 
Swift bindings BEGONE

Release Notes:

- Switched from using the Swift LiveKit bindings, to the Rust bindings,
fixing https://github.com/zed-industries/zed/issues/9396, a crash when
leaving a collaboration session, and making Zed easier to build.

---------

Co-authored-by: Conrad Irwin <conrad.irwin@gmail.com>
Co-authored-by: Michael Sloan <michael@zed.dev>
This commit is contained in:
Mikayla Maki 2025-03-28 10:58:23 -07:00 • committed by GitHub
parent c8fb95cd1b
commit 8a307e7b89
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
68 changed files with 2393 additions and 7579 deletions
Download patch file Download diff file Expand all files Collapse all files

19
crates/livekit_client/Cargo.toml
View file

@ -10,46 +10,47 @@ license = "GPL-3.0-or-later"
workspace = true
[lib]
path = "src/livekit_client.rs"
path = "src/lib.rs"
doctest = false
[[example]]
name = "test_app"
[features]
no-webrtc = []
test-support = ["collections/test-support", "gpui/test-support", "nanoid"]
test-support = ["collections/test-support", "gpui/test-support"]
[dependencies]
gpui_tokio.workspace = true
anyhow.workspace = true
async-trait.workspace = true
collections.workspace = true
cpal = "0.15"
futures.workspace = true
gpui.workspace = true
http_2 = { package = "http", version = "0.2.1" }
livekit_api.workspace = true
log.workspace = true
media.workspace = true
nanoid = { workspace = true, optional = true }
nanoid.workspace = true
parking_lot.workspace = true
postage.workspace = true
util.workspace = true
http_client.workspace = true
smallvec.workspace = true
image.workspace = true
tokio-tungstenite.workspace = true
http_client_tls.workspace = true
[target.'cfg(not(all(target_os = "windows", target_env = "gnu")))'.dependencies]
livekit.workspace = true
livekit = { rev = "102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8", git = "https://github.com/zed-industries/livekit-rust-sdks", features = ["__rustls-tls"]}
libwebrtc = { rev = "102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8", git = "https://github.com/zed-industries/livekit-rust-sdks"}
[target.'cfg(target_os = "macos")'.dependencies]
core-foundation.workspace = true
coreaudio-rs = "0.12.1"
objc = "0.2"
core-video.workspace = true
[dev-dependencies]
collections = { workspace = true, features = ["test-support"] }
gpui = { workspace = true, features = ["test-support"] }
nanoid.workspace = true
sha2.workspace = true
simplelog.workspace = true

121
crates/livekit_client/examples/test_app.rs
View file

@ -1,8 +1,6 @@
#![cfg_attr(windows, allow(unused))]
// TODO: For some reason mac build complains about import of postage::stream::Stream, but removal of
// it causes compile errors.
#![cfg_attr(target_os = "macos", allow(unused_imports))]
use std::sync::Arc;
use futures::StreamExt;
use gpui::{
actions, bounds, div, point,
prelude::{FluentBuilder as _, IntoElement},
@ -11,26 +9,9 @@ use gpui::{
StatefulInteractiveElement as _, Styled, Task, Window, WindowBounds, WindowHandle,
WindowOptions,
};
#[cfg(not(target_os = "windows"))]
use livekit_client::{
capture_local_audio_track, capture_local_video_track,
id::ParticipantIdentity,
options::{TrackPublishOptions, VideoCodec},
participant::{Participant, RemoteParticipant},
play_remote_audio_track,
publication::{LocalTrackPublication, RemoteTrackPublication},
track::{LocalTrack, RemoteTrack, RemoteVideoTrack, TrackSource},
AudioStream, RemoteVideoTrackView, Room, RoomEvent, RoomOptions,
};
#[cfg(not(target_os = "windows"))]
use postage::stream::Stream;
#[cfg(target_os = "windows")]
use livekit_client::{
participant::{Participant, RemoteParticipant},
publication::{LocalTrackPublication, RemoteTrackPublication},
track::{LocalTrack, RemoteTrack, RemoteVideoTrack},
AudioStream, RemoteVideoTrackView, Room, RoomEvent,
AudioStream, LocalTrackPublication, Participant, ParticipantIdentity, RemoteParticipant,
RemoteTrackPublication, RemoteVideoTrack, RemoteVideoTrackView, Room, RoomEvent,
};
use livekit_api::token::{self, VideoGrant};
@ -39,25 +20,18 @@ use simplelog::SimpleLogger;
actions!(livekit_client, [Quit]);
#[cfg(windows)]
fn main() {}
#[cfg(not(windows))]
fn main() {
SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger");
gpui::Application::new().run(|cx| {
livekit_client::init(
cx.background_executor().dispatcher.clone(),
cx.http_client(),
);
#[cfg(any(test, feature = "test-support"))]
println!("USING TEST LIVEKIT");
#[cfg(not(any(test, feature = "test-support")))]
println!("USING REAL LIVEKIT");
gpui_tokio::init(cx);
cx.activate(true);
cx.on_action(quit);
cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]);
@ -83,14 +57,12 @@ fn main() {
&livekit_key,
&livekit_secret,
Some(&format!("test-participant-{i}")),
VideoGrant::to_join("test-room"),
VideoGrant::to_join("wtej-trty"),
)
.unwrap();
let bounds = bounds(point(width * i, px(0.0)), size(width, height));
let window =
LivekitWindow::new(livekit_url.as_str(), token.as_str(), bounds, cx.clone())
.await;
let window = LivekitWindow::new(livekit_url.clone(), token, bounds, cx).await;
windows.push(window);
}
})
@ -103,12 +75,11 @@ fn quit(_: &Quit, cx: &mut gpui::App) {
}
struct LivekitWindow {
room: Room,
room: Arc<livekit_client::Room>,
microphone_track: Option<LocalTrackPublication>,
screen_share_track: Option<LocalTrackPublication>,
microphone_stream: Option<AudioStream>,
microphone_stream: Option<livekit_client::AudioStream>,
screen_share_stream: Option<Box<dyn ScreenCaptureStream>>,
#[cfg(not(target_os = "windows"))]
remote_participants: Vec<(ParticipantIdentity, ParticipantState)>,
_events_task: Task<()>,
}
@ -121,17 +92,23 @@ struct ParticipantState {
speaking: bool,
}
#[cfg(not(windows))]
impl LivekitWindow {
async fn new(
url: &str,
token: &str,
url: String,
token: String,
bounds: Bounds<Pixels>,
cx: AsyncApp,
cx: &mut AsyncApp,
) -> WindowHandle<Self> {
let (room, mut events) = Room::connect(url, token, RoomOptions::default())
.await
.unwrap();
let (room, mut events) =
Room::connect(url.clone(), token, cx)
.await
.unwrap_or_else(|err| {
eprintln!(
"Failed to connect to {url}: {err}.\nTry `foreman start` to run the livekit server"
);
std::process::exit(1)
});
cx.update(|cx| {
cx.open_window(
@ -142,7 +119,7 @@ impl LivekitWindow {
|window, cx| {
cx.new(|cx| {
let _events_task = cx.spawn_in(window, async move |this, cx| {
while let Some(event) = events.recv().await {
while let Some(event) = events.next().await {
cx.update(|window, cx| {
this.update(cx, |this: &mut LivekitWindow, cx| {
this.handle_room_event(event, window, cx)
@ -153,7 +130,7 @@ impl LivekitWindow {
});
Self {
room,
room: Arc::new(room),
microphone_track: None,
microphone_stream: None,
screen_share_track: None,
@ -201,15 +178,16 @@ impl LivekitWindow {
participant,
track,
} => {
let room = self.room.clone();
let output = self.remote_participant(participant);
match track {
RemoteTrack::Audio(track) => {
livekit_client::RemoteTrack::Audio(track) => {
output.audio_output_stream = Some((
publication.clone(),
play_remote_audio_track(&track, cx.background_executor()).unwrap(),
room.play_remote_audio_track(&track, cx).unwrap(),
));
}
RemoteTrack::Video(track) => {
livekit_client::RemoteTrack::Video(track) => {
output.screen_share_output_view = Some((
track.clone(),
cx.new(|cx| RemoteVideoTrackView::new(track, window, cx)),
@ -269,25 +247,15 @@ impl LivekitWindow {
fn toggle_mute(&mut self, window: &mut Window, cx: &mut Context<Self>) {
if let Some(track) = &self.microphone_track {
if track.is_muted() {
track.unmute();
track.unmute(cx);
} else {
track.mute();
track.mute(cx);
}
cx.notify();
} else {
let participant = self.room.local_participant();
let room = self.room.clone();
cx.spawn_in(window, async move |this, cx| {
let (track, stream) = capture_local_audio_track(cx.background_executor())?.await;
let publication = participant
.publish_track(
LocalTrack::Audio(track),
TrackPublishOptions {
source: TrackSource::Microphone,
..Default::default()
},
)
.await
.unwrap();
let (publication, stream) = room.publish_local_microphone_track(cx).await.unwrap();
this.update(cx, |this, cx| {
this.microphone_track = Some(publication);
this.microphone_stream = Some(stream);
@ -302,8 +270,8 @@ impl LivekitWindow {
if let Some(track) = self.screen_share_track.take() {
self.screen_share_stream.take();
let participant = self.room.local_participant();
cx.background_spawn(async move {
participant.unpublish_track(&track.sid()).await.unwrap();
cx.spawn(async move |_, cx| {
participant.unpublish_track(track.sid(), cx).await.unwrap();
})
.detach();
cx.notify();
@ -313,16 +281,9 @@ impl LivekitWindow {
cx.spawn_in(window, async move |this, cx| {
let sources = sources.await.unwrap()?;
let source = sources.into_iter().next().unwrap();
let (track, stream) = capture_local_video_track(&*source).await?;
let publication = participant
.publish_track(
LocalTrack::Video(track),
TrackPublishOptions {
source: TrackSource::Screenshare,
video_codec: VideoCodec::H264,
..Default::default()
},
)
let (publication, stream) = participant
.publish_screenshare_track(&*source, cx)
.await
.unwrap();
this.update(cx, |this, cx| {
@ -338,7 +299,6 @@ impl LivekitWindow {
fn toggle_remote_audio_for_participant(
&mut self,
identity: &ParticipantIdentity,
cx: &mut Context<Self>,
) -> Option<()> {
let participant = self.remote_participants.iter().find_map(|(id, state)| {
@ -349,13 +309,12 @@ impl LivekitWindow {
}
})?;
let publication = &participant.audio_output_stream.as_ref()?.0;
publication.set_enabled(!publication.is_enabled());
publication.set_enabled(!publication.is_enabled(), cx);
cx.notify();
Some(())
}
}
#[cfg(not(windows))]
impl Render for LivekitWindow {
fn render(&mut self, _window: &mut Window, cx: &mut Context<Self>) -> impl IntoElement {
fn button() -> gpui::Div {
@ -407,7 +366,7 @@ impl Render for LivekitWindow {
.flex_grow()
.children(self.remote_participants.iter().map(|(identity, state)| {
div()
.h(px(300.0))
.h(px(1080.0))
.flex()
.flex_col()
.m_2()

165
crates/livekit_client/src/lib.rs Normal file
View file

@ -0,0 +1,165 @@
use collections::HashMap;
mod remote_video_track_view;
pub use remote_video_track_view::{RemoteVideoTrackView, RemoteVideoTrackViewEvent};
#[cfg(not(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu")
)))]
mod livekit_client;
#[cfg(not(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu")
)))]
pub use livekit_client::*;
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu")
))]
mod mock_client;
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu")
))]
pub mod test;
#[cfg(any(
test,
feature = "test-support",
all(target_os = "windows", target_env = "gnu")
))]
pub use mock_client::*;
#[derive(Debug, Clone)]
pub enum Participant {
Local(LocalParticipant),
Remote(RemoteParticipant),
}
#[derive(Debug, Clone)]
pub enum TrackPublication {
Local(LocalTrackPublication),
Remote(RemoteTrackPublication),
}
impl TrackPublication {
pub fn sid(&self) -> TrackSid {
match self {
TrackPublication::Local(local) => local.sid(),
TrackPublication::Remote(remote) => remote.sid(),
}
}
pub fn is_muted(&self) -> bool {
match self {
TrackPublication::Local(local) => local.is_muted(),
TrackPublication::Remote(remote) => remote.is_muted(),
}
}
}
#[derive(Clone, Debug)]
pub enum RemoteTrack {
Audio(RemoteAudioTrack),
Video(RemoteVideoTrack),
}
impl RemoteTrack {
pub fn sid(&self) -> TrackSid {
match self {
RemoteTrack::Audio(remote_audio_track) => remote_audio_track.sid(),
RemoteTrack::Video(remote_video_track) => remote_video_track.sid(),
}
}
}
#[derive(Clone, Debug)]
pub enum LocalTrack {
Audio(LocalAudioTrack),
Video(LocalVideoTrack),
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum RoomEvent {
ParticipantConnected(RemoteParticipant),
ParticipantDisconnected(RemoteParticipant),
LocalTrackPublished {
publication: LocalTrackPublication,
track: LocalTrack,
participant: LocalParticipant,
},
LocalTrackUnpublished {
publication: LocalTrackPublication,
participant: LocalParticipant,
},
LocalTrackSubscribed {
track: LocalTrack,
},
TrackSubscribed {
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackUnsubscribed {
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackSubscriptionFailed {
participant: RemoteParticipant,
// error: livekit::track::TrackError,
track_sid: TrackSid,
},
TrackPublished {
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackUnpublished {
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackMuted {
participant: Participant,
publication: TrackPublication,
},
TrackUnmuted {
participant: Participant,
publication: TrackPublication,
},
RoomMetadataChanged {
old_metadata: String,
metadata: String,
},
ParticipantMetadataChanged {
participant: Participant,
old_metadata: String,
metadata: String,
},
ParticipantNameChanged {
participant: Participant,
old_name: String,
name: String,
},
ParticipantAttributesChanged {
participant: Participant,
changed_attributes: HashMap<String, String>,
},
ActiveSpeakersChanged {
speakers: Vec<Participant>,
},
ConnectionStateChanged(ConnectionState),
Connected {
participants_with_tracks: Vec<(RemoteParticipant, Vec<RemoteTrackPublication>)>,
},
Disconnected {
reason: &'static str,
},
Reconnecting,
Reconnected,
}

1124
crates/livekit_client/src/livekit_client.rs
View file

File diff suppressed because it is too large Load diff

763
crates/livekit_client/src/livekit_client/playback.rs Normal file
View file

@ -0,0 +1,763 @@
use anyhow::{anyhow, Context as _, Result};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait as _};
use futures::channel::mpsc::UnboundedSender;
use futures::{Stream, StreamExt as _};
use gpui::{
BackgroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task,
};
use libwebrtc::native::{apm, audio_mixer, audio_resampler};
use livekit::track;
use livekit::webrtc::{
audio_frame::AudioFrame,
audio_source::{native::NativeAudioSource, AudioSourceOptions, RtcAudioSource},
audio_stream::native::NativeAudioStream,
video_frame::{VideoBuffer, VideoFrame, VideoRotation},
video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution},
video_stream::native::NativeVideoStream,
};
use parking_lot::Mutex;
use std::cell::RefCell;
use std::sync::atomic::{self, AtomicI32};
use std::sync::Weak;
use std::time::Duration;
use std::{borrow::Cow, collections::VecDeque, sync::Arc, thread};
use util::{maybe, ResultExt as _};
pub(crate) struct AudioStack {
executor: BackgroundExecutor,
apm: Arc<Mutex<apm::AudioProcessingModule>>,
mixer: Arc<Mutex<audio_mixer::AudioMixer>>,
_output_task: RefCell<Weak<Task<()>>>,
next_ssrc: AtomicI32,
}
// NOTE: We use WebRTC's mixer which only supports
// 16kHz, 32kHz and 48kHz. As 48 is the most common "next step up"
// for audio output devices like speakers/bluetooth, we just hard-code
// this; and downsample when we need to.
const SAMPLE_RATE: u32 = 48000;
const NUM_CHANNELS: u32 = 2;
impl AudioStack {
pub(crate) fn new(executor: BackgroundExecutor) -> Self {
let apm = Arc::new(Mutex::new(apm::AudioProcessingModule::new(
true, true, true, true,
)));
let mixer = Arc::new(Mutex::new(audio_mixer::AudioMixer::new()));
Self {
executor,
apm,
mixer,
_output_task: RefCell::new(Weak::new()),
next_ssrc: AtomicI32::new(1),
}
}
pub(crate) fn play_remote_audio_track(
&self,
track: &livekit::track::RemoteAudioTrack,
) -> AudioStream {
let output_task = self.start_output();
let next_ssrc = self.next_ssrc.fetch_add(1, atomic::Ordering::Relaxed);
let source = AudioMixerSource {
ssrc: next_ssrc,
sample_rate: SAMPLE_RATE,
num_channels: NUM_CHANNELS,
buffer: Arc::default(),
};
self.mixer.lock().add_source(source.clone());
let mut stream = NativeAudioStream::new(
track.rtc_track(),
source.sample_rate as i32,
source.num_channels as i32,
);
let receive_task = self.executor.spawn({
let source = source.clone();
async move {
while let Some(frame) = stream.next().await {
source.receive(frame);
}
}
});
let mixer = self.mixer.clone();
let on_drop = util::defer(move || {
mixer.lock().remove_source(source.ssrc);
drop(receive_task);
drop(output_task);
});
AudioStream::Output {
_drop: Box::new(on_drop),
}
}
pub(crate) fn capture_local_microphone_track(
&self,
) -> Result<(crate::LocalAudioTrack, AudioStream)> {
let source = NativeAudioSource::new(
// n.b. this struct's options are always ignored, noise cancellation is provided by apm.
AudioSourceOptions::default(),
SAMPLE_RATE,
NUM_CHANNELS,
10,
);
let track = track::LocalAudioTrack::create_audio_track(
"microphone",
RtcAudioSource::Native(source.clone()),
);
let apm = self.apm.clone();
let (frame_tx, mut frame_rx) = futures::channel::mpsc::unbounded();
let transmit_task = self.executor.spawn({
let source = source.clone();
async move {
while let Some(frame) = frame_rx.next().await {
source.capture_frame(&frame).await.log_err();
}
}
});
let capture_task = self.executor.spawn(async move {
Self::capture_input(apm, frame_tx, SAMPLE_RATE, NUM_CHANNELS).await
});
let on_drop = util::defer(|| {
drop(transmit_task);
drop(capture_task);
});
return Ok((
super::LocalAudioTrack(track),
AudioStream::Output {
_drop: Box::new(on_drop),
},
));
}
fn start_output(&self) -> Arc<Task<()>> {
if let Some(task) = self._output_task.borrow().upgrade() {
return task;
}
let task = Arc::new(self.executor.spawn({
let apm = self.apm.clone();
let mixer = self.mixer.clone();
async move {
Self::play_output(apm, mixer, SAMPLE_RATE, NUM_CHANNELS)
.await
.log_err();
}
}));
*self._output_task.borrow_mut() = Arc::downgrade(&task);
task
}
async fn play_output(
apm: Arc<Mutex<apm::AudioProcessingModule>>,
mixer: Arc<Mutex<audio_mixer::AudioMixer>>,
sample_rate: u32,
num_channels: u32,
) -> Result<()> {
let mut default_change_listener = DeviceChangeListener::new(false)?;
loop {
let (output_device, output_config) = default_device(false)?;
let (end_on_drop_tx, end_on_drop_rx) = std::sync::mpsc::channel::<()>();
let mixer = mixer.clone();
let apm = apm.clone();
let mut resampler = audio_resampler::AudioResampler::default();
let mut buf = Vec::new();
thread::spawn(move || {
let output_stream = output_device.build_output_stream(
&output_config.config(),
{
move |mut data, _info| {
while data.len() > 0 {
if data.len() <= buf.len() {
let rest = buf.split_off(data.len());
data.copy_from_slice(&buf);
buf = rest;
return;
}
if buf.len() > 0 {
let (prefix, suffix) = data.split_at_mut(buf.len());
prefix.copy_from_slice(&buf);
data = suffix;
}
let mut mixer = mixer.lock();
let mixed = mixer.mix(output_config.channels() as usize);
let sampled = resampler.remix_and_resample(
mixed,
sample_rate / 100,
num_channels,
sample_rate,
output_config.channels() as u32,
output_config.sample_rate().0,
);
buf = sampled.to_vec();
apm.lock()
.process_reverse_stream(
&mut buf,
output_config.sample_rate().0 as i32,
output_config.channels() as i32,
)
.ok();
}
}
},
|error| log::error!("error playing audio track: {:?}", error),
Some(Duration::from_millis(100)),
);
let Some(output_stream) = output_stream.log_err() else {
return;
};
output_stream.play().log_err();
// Block forever to keep the output stream alive
end_on_drop_rx.recv().ok();
});
default_change_listener.next().await;
drop(end_on_drop_tx)
}
}
async fn capture_input(
apm: Arc<Mutex<apm::AudioProcessingModule>>,
frame_tx: UnboundedSender<AudioFrame<'static>>,
sample_rate: u32,
num_channels: u32,
) -> Result<()> {
let mut default_change_listener = DeviceChangeListener::new(true)?;
loop {
let (device, config) = default_device(true)?;
let (end_on_drop_tx, end_on_drop_rx) = std::sync::mpsc::channel::<()>();
let apm = apm.clone();
let frame_tx = frame_tx.clone();
let mut resampler = audio_resampler::AudioResampler::default();
thread::spawn(move || {
maybe!({
if let Some(name) = device.name().ok() {
log::info!("Using microphone: {}", name)
} else {
log::info!("Using microphone: <unknown>");
}
let ten_ms_buffer_size =
(config.channels() as u32 * config.sample_rate().0 / 100) as usize;
let mut buf: Vec<i16> = Vec::with_capacity(ten_ms_buffer_size);
let stream = device
.build_input_stream_raw(
&config.config(),
cpal::SampleFormat::I16,
move |data, _: &_| {
let mut data = data.as_slice::<i16>().unwrap();
while data.len() > 0 {
let remainder = (buf.capacity() - buf.len()).min(data.len());
buf.extend_from_slice(&data[..remainder]);
data = &data[remainder..];
if buf.capacity() == buf.len() {
let mut sampled = resampler
.remix_and_resample(
buf.as_slice(),
config.sample_rate().0 as u32 / 100,
config.channels() as u32,
config.sample_rate().0 as u32,
num_channels,
sample_rate,
)
.to_owned();
apm.lock()
.process_stream(
&mut sampled,
sample_rate as i32,
num_channels as i32,
)
.log_err();
buf.clear();
frame_tx
.unbounded_send(AudioFrame {
data: Cow::Owned(sampled),
sample_rate,
num_channels,
samples_per_channel: sample_rate / 100,
})
.ok();
}
}
},
|err| log::error!("error capturing audio track: {:?}", err),
Some(Duration::from_millis(100)),
)
.context("failed to build input stream")?;
stream.play()?;
// Keep the thread alive and holding onto the `stream`
end_on_drop_rx.recv().ok();
anyhow::Ok(Some(()))
})
.log_err();
});
default_change_listener.next().await;
drop(end_on_drop_tx)
}
}
}
use super::LocalVideoTrack;
pub enum AudioStream {
Input { _task: Task<()> },
Output { _drop: Box<dyn std::any::Any> },
}
pub(crate) async fn capture_local_video_track(
capture_source: &dyn ScreenCaptureSource,
cx: &mut gpui::AsyncApp,
) -> Result<(crate::LocalVideoTrack, Box<dyn ScreenCaptureStream>)> {
let resolution = capture_source.resolution()?;
let track_source = gpui_tokio::Tokio::spawn(cx, async move {
NativeVideoSource::new(VideoResolution {
width: resolution.width.0 as u32,
height: resolution.height.0 as u32,
})
})?
.await?;
let capture_stream = capture_source
.stream({
let track_source = track_source.clone();
Box::new(move |frame| {
if let Some(buffer) = video_frame_buffer_to_webrtc(frame) {
track_source.capture_frame(&VideoFrame {
rotation: VideoRotation::VideoRotation0,
timestamp_us: 0,
buffer,
});
}
})
})
.await??;
Ok((
LocalVideoTrack(track::LocalVideoTrack::create_video_track(
"screen share",
RtcVideoSource::Native(track_source),
)),
capture_stream,
))
}
fn default_device(input: bool) -> Result<(cpal::Device, cpal::SupportedStreamConfig)> {
let device;
let config;
if input {
device = cpal::default_host()
.default_input_device()
.ok_or_else(|| anyhow!("no audio input device available"))?;
config = device
.default_input_config()
.context("failed to get default input config")?;
} else {
device = cpal::default_host()
.default_output_device()
.ok_or_else(|| anyhow!("no audio output device available"))?;
config = device
.default_output_config()
.context("failed to get default output config")?;
}
Ok((device, config))
}
#[derive(Clone)]
struct AudioMixerSource {
ssrc: i32,
sample_rate: u32,
num_channels: u32,
buffer: Arc<Mutex<VecDeque<Vec<i16>>>>,
}
impl AudioMixerSource {
fn receive(&self, frame: AudioFrame) {
assert_eq!(
frame.data.len() as u32,
self.sample_rate * self.num_channels / 100
);
let mut buffer = self.buffer.lock();
buffer.push_back(frame.data.to_vec());
while buffer.len() > 10 {
buffer.pop_front();
}
}
}
impl libwebrtc::native::audio_mixer::AudioMixerSource for AudioMixerSource {
fn ssrc(&self) -> i32 {
self.ssrc
}
fn preferred_sample_rate(&self) -> u32 {
self.sample_rate
}
fn get_audio_frame_with_info<'a>(&self, target_sample_rate: u32) -> Option<AudioFrame> {
assert_eq!(self.sample_rate, target_sample_rate);
let buf = self.buffer.lock().pop_front()?;
Some(AudioFrame {
data: Cow::Owned(buf),
sample_rate: self.sample_rate,
num_channels: self.num_channels,
samples_per_channel: self.sample_rate / 100,
})
}
}
pub fn play_remote_video_track(
track: &crate::RemoteVideoTrack,
) -> impl Stream<Item = RemoteVideoFrame> {
#[cfg(target_os = "macos")]
{
let mut pool = None;
let most_recent_frame_size = (0, 0);
NativeVideoStream::new(track.0.rtc_track()).filter_map(move |frame| {
if pool == None
|| most_recent_frame_size != (frame.buffer.width(), frame.buffer.height())
{
pool = create_buffer_pool(frame.buffer.width(), frame.buffer.height()).log_err();
}
let pool = pool.clone();
async move {
if frame.buffer.width() < 10 && frame.buffer.height() < 10 {
// when the remote stops sharing, we get an 8x8 black image.
// In a lil bit, the unpublish will come through and close the view,
// but until then, don't flash black.
return None;
}
video_frame_buffer_from_webrtc(pool?, frame.buffer)
}
})
}
#[cfg(not(target_os = "macos"))]
{
NativeVideoStream::new(track.0.rtc_track())
.filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) })
}
}
#[cfg(target_os = "macos")]
fn create_buffer_pool(
width: u32,
height: u32,
) -> Result<core_video::pixel_buffer_pool::CVPixelBufferPool> {
use core_foundation::{base::TCFType, number::CFNumber, string::CFString};
use core_video::pixel_buffer;
use core_video::{
pixel_buffer::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
pixel_buffer_io_surface::kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey,
pixel_buffer_pool::{self},
};
let width_key: CFString =
unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferWidthKey) };
let height_key: CFString =
unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferHeightKey) };
let animation_key: CFString = unsafe {
CFString::wrap_under_get_rule(kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey)
};
let format_key: CFString =
unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferPixelFormatTypeKey) };
let yes: CFNumber = 1.into();
let width: CFNumber = (width as i32).into();
let height: CFNumber = (height as i32).into();
let format: CFNumber = (kCVPixelFormatType_420YpCbCr8BiPlanarFullRange as i64).into();
let buffer_attributes = core_foundation::dictionary::CFDictionary::from_CFType_pairs(&[
(width_key, width.into_CFType()),
(height_key, height.into_CFType()),
(animation_key, yes.into_CFType()),
(format_key, format.into_CFType()),
]);
pixel_buffer_pool::CVPixelBufferPool::new(None, Some(&buffer_attributes)).map_err(|cv_return| {
anyhow!(
"failed to create pixel buffer pool: CVReturn({})",
cv_return
)
})
}
#[cfg(target_os = "macos")]
pub type RemoteVideoFrame = core_video::pixel_buffer::CVPixelBuffer;
#[cfg(target_os = "macos")]
fn video_frame_buffer_from_webrtc(
pool: core_video::pixel_buffer_pool::CVPixelBufferPool,
buffer: Box<dyn VideoBuffer>,
) -> Option<RemoteVideoFrame> {
use core_foundation::base::TCFType;
use core_video::{pixel_buffer::CVPixelBuffer, r#return::kCVReturnSuccess};
use livekit::webrtc::native::yuv_helper::i420_to_nv12;
if let Some(native) = buffer.as_native() {
let pixel_buffer = native.get_cv_pixel_buffer();
if pixel_buffer.is_null() {
return None;
}
return unsafe { Some(CVPixelBuffer::wrap_under_get_rule(pixel_buffer as _)) };
}
let i420_buffer = buffer.as_i420()?;
let pixel_buffer = pool.create_pixel_buffer().log_err()?;
let image_buffer = unsafe {
if pixel_buffer.lock_base_address(0) != kCVReturnSuccess {
return None;
}
let dst_y = pixel_buffer.get_base_address_of_plane(0);
let dst_y_stride = pixel_buffer.get_bytes_per_row_of_plane(0);
let dst_y_len = pixel_buffer.get_height_of_plane(0) * dst_y_stride;
let dst_uv = pixel_buffer.get_base_address_of_plane(1);
let dst_uv_stride = pixel_buffer.get_bytes_per_row_of_plane(1);
let dst_uv_len = pixel_buffer.get_height_of_plane(1) * dst_uv_stride;
let width = pixel_buffer.get_width();
let height = pixel_buffer.get_height();
let dst_y_buffer = std::slice::from_raw_parts_mut(dst_y as *mut u8, dst_y_len);
let dst_uv_buffer = std::slice::from_raw_parts_mut(dst_uv as *mut u8, dst_uv_len);
let (stride_y, stride_u, stride_v) = i420_buffer.strides();
let (src_y, src_u, src_v) = i420_buffer.data();
i420_to_nv12(
src_y,
stride_y,
src_u,
stride_u,
src_v,
stride_v,
dst_y_buffer,
dst_y_stride as u32,
dst_uv_buffer,
dst_uv_stride as u32,
width as i32,
height as i32,
);
if pixel_buffer.unlock_base_address(0) != kCVReturnSuccess {
return None;
}
pixel_buffer
};
Some(image_buffer)
}
#[cfg(not(target_os = "macos"))]
pub type RemoteVideoFrame = Arc<gpui::RenderImage>;
#[cfg(not(target_os = "macos"))]
fn video_frame_buffer_from_webrtc(buffer: Box<dyn VideoBuffer>) -> Option<RemoteVideoFrame> {
use gpui::RenderImage;
use image::{Frame, RgbaImage};
use livekit::webrtc::prelude::VideoFormatType;
use smallvec::SmallVec;
use std::alloc::{alloc, Layout};
let width = buffer.width();
let height = buffer.height();
let stride = width * 4;
let byte_len = (stride * height) as usize;
let argb_image = unsafe {
// Motivation for this unsafe code is to avoid initializing the frame data, since to_argb
// will write all bytes anyway.
let start_ptr = alloc(Layout::array::<u8>(byte_len).log_err()?);
if start_ptr.is_null() {
return None;
}
let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len);
buffer.to_argb(
VideoFormatType::ARGB, // For some reason, this displays correctly while RGBA (the correct format) does not
bgra_frame_slice,
stride,
width as i32,
height as i32,
);
Vec::from_raw_parts(start_ptr, byte_len, byte_len)
};
Some(Arc::new(RenderImage::new(SmallVec::from_elem(
Frame::new(
RgbaImage::from_raw(width, height, argb_image)
.with_context(|| "Bug: not enough bytes allocated for image.")
.log_err()?,
),
1,
))))
}
#[cfg(target_os = "macos")]
fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option<impl AsRef<dyn VideoBuffer>> {
use livekit::webrtc;
let pixel_buffer = frame.0.as_concrete_TypeRef();
std::mem::forget(frame.0);
unsafe {
Some(webrtc::video_frame::native::NativeBuffer::from_cv_pixel_buffer(pixel_buffer as _))
}
}
#[cfg(not(target_os = "macos"))]
fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option<impl AsRef<dyn VideoBuffer>> {
None as Option<Box<dyn VideoBuffer>>
}
trait DeviceChangeListenerApi: Stream<Item = ()> + Sized {
fn new(input: bool) -> Result<Self>;
}
#[cfg(target_os = "macos")]
mod macos {
use coreaudio::sys::{
kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice,
kAudioObjectPropertyElementMaster, kAudioObjectPropertyScopeGlobal,
kAudioObjectSystemObject, AudioObjectAddPropertyListener, AudioObjectID,
AudioObjectPropertyAddress, AudioObjectRemovePropertyListener, OSStatus,
};
use futures::{channel::mpsc::UnboundedReceiver, StreamExt};
/// Implementation from: https://github.com/zed-industries/cpal/blob/fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50/src/host/coreaudio/macos/property_listener.rs#L15
pub struct CoreAudioDefaultDeviceChangeListener {
rx: UnboundedReceiver<()>,
callback: Box<PropertyListenerCallbackWrapper>,
input: bool,
}
trait _AssertSend: Send {}
impl _AssertSend for CoreAudioDefaultDeviceChangeListener {}
struct PropertyListenerCallbackWrapper(Box<dyn FnMut() + Send>);
unsafe extern "C" fn property_listener_handler_shim(
_: AudioObjectID,
_: u32,
_: *const AudioObjectPropertyAddress,
callback: *mut ::std::os::raw::c_void,
) -> OSStatus {
let wrapper = callback as *mut PropertyListenerCallbackWrapper;
(*wrapper).0();
0
}
impl super::DeviceChangeListenerApi for CoreAudioDefaultDeviceChangeListener {
fn new(input: bool) -> gpui::Result<Self> {
let (tx, rx) = futures::channel::mpsc::unbounded();
let callback = Box::new(PropertyListenerCallbackWrapper(Box::new(move || {
tx.unbounded_send(()).ok();
})));
unsafe {
coreaudio::Error::from_os_status(AudioObjectAddPropertyListener(
kAudioObjectSystemObject,
&AudioObjectPropertyAddress {
mSelector: if input {
kAudioHardwarePropertyDefaultInputDevice
} else {
kAudioHardwarePropertyDefaultOutputDevice
},
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMaster,
},
Some(property_listener_handler_shim),
&*callback as *const _ as *mut _,
))?;
}
Ok(Self {
rx,
callback,
input,
})
}
}
impl Drop for CoreAudioDefaultDeviceChangeListener {
fn drop(&mut self) {
unsafe {
AudioObjectRemovePropertyListener(
kAudioObjectSystemObject,
&AudioObjectPropertyAddress {
mSelector: if self.input {
kAudioHardwarePropertyDefaultInputDevice
} else {
kAudioHardwarePropertyDefaultOutputDevice
},
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMaster,
},
Some(property_listener_handler_shim),
&*self.callback as *const _ as *mut _,
);
}
}
}
impl futures::Stream for CoreAudioDefaultDeviceChangeListener {
type Item = ();
fn poll_next(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
self.rx.poll_next_unpin(cx)
}
}
}
#[cfg(target_os = "macos")]
type DeviceChangeListener = macos::CoreAudioDefaultDeviceChangeListener;
#[cfg(not(target_os = "macos"))]
mod noop_change_listener {
use std::task::Poll;
use super::DeviceChangeListenerApi;
pub struct NoopOutputDeviceChangelistener {}
impl DeviceChangeListenerApi for NoopOutputDeviceChangelistener {
fn new(_input: bool) -> anyhow::Result<Self> {
Ok(NoopOutputDeviceChangelistener {})
}
}
impl futures::Stream for NoopOutputDeviceChangelistener {
type Item = ();
fn poll_next(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> Poll<Option<Self::Item>> {
Poll::Pending
}
}
}
#[cfg(not(target_os = "macos"))]
type DeviceChangeListener = noop_change_listener::NoopOutputDeviceChangelistener;

38
crates/livekit_client/src/mock_client.rs Normal file
View file

@ -0,0 +1,38 @@
use crate::test;
pub(crate) mod participant;
pub(crate) mod publication;
pub(crate) mod track;
pub type RemoteVideoTrack = track::RemoteVideoTrack;
pub type RemoteAudioTrack = track::RemoteAudioTrack;
pub type RemoteTrackPublication = publication::RemoteTrackPublication;
pub type RemoteParticipant = participant::RemoteParticipant;
pub type LocalVideoTrack = track::LocalVideoTrack;
pub type LocalAudioTrack = track::LocalAudioTrack;
pub type LocalTrackPublication = publication::LocalTrackPublication;
pub type LocalParticipant = participant::LocalParticipant;
pub type Room = test::Room;
pub use test::{ConnectionState, ParticipantIdentity, TrackSid};
pub struct AudioStream {}
#[cfg(not(target_os = "macos"))]
pub type RemoteVideoFrame = std::sync::Arc<gpui::RenderImage>;
#[cfg(target_os = "macos")]
#[derive(Clone)]
pub(crate) struct RemoteVideoFrame {}
#[cfg(target_os = "macos")]
impl Into<gpui::SurfaceSource> for RemoteVideoFrame {
fn into(self) -> gpui::SurfaceSource {
unimplemented!()
}
}
pub(crate) fn play_remote_video_track(
_track: &crate::RemoteVideoTrack,
) -> impl futures::Stream<Item = RemoteVideoFrame> {
futures::stream::pending()
}

88
crates/livekit_client/src/test/participant.rs → crates/livekit_client/src/mock_client/participant.rs
View file

@ -1,26 +1,24 @@
use super::*;
#[derive(Clone, Debug)]
pub enum Participant {
Local(LocalParticipant),
Remote(RemoteParticipant),
}
use crate::{
test::{Room, WeakRoom},
AudioStream, LocalAudioTrack, LocalTrackPublication, LocalVideoTrack, Participant,
ParticipantIdentity, RemoteTrack, RemoteTrackPublication, TrackSid,
};
use anyhow::Result;
use collections::HashMap;
use gpui::{AsyncApp, ScreenCaptureSource, ScreenCaptureStream};
#[derive(Clone, Debug)]
pub struct LocalParticipant {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) identity: ParticipantIdentity,
pub(super) room: Room,
pub(crate) identity: ParticipantIdentity,
pub(crate) room: Room,
}
#[derive(Clone, Debug)]
pub struct RemoteParticipant {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) identity: ParticipantIdentity,
pub(super) room: WeakRoom,
pub(crate) identity: ParticipantIdentity,
pub(crate) room: WeakRoom,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl Participant {
pub fn identity(&self) -> ParticipantIdentity {
match self {
@ -30,41 +28,53 @@ impl Participant {
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl LocalParticipant {
pub async fn unpublish_track(&self, track: &TrackSid) -> Result<()> {
pub async fn unpublish_track(&self, track: TrackSid, _cx: &AsyncApp) -> Result<()> {
self.room
.test_server()
.unpublish_track(self.room.token(), track)
.unpublish_track(self.room.token(), &track)
.await
}
pub async fn publish_track(
pub(crate) async fn publish_microphone_track(
&self,
track: LocalTrack,
_options: TrackPublishOptions,
) -> Result<LocalTrackPublication> {
_cx: &AsyncApp,
) -> Result<(LocalTrackPublication, AudioStream)> {
let this = self.clone();
let track = track.clone();
let server = this.room.test_server();
let sid = match track {
LocalTrack::Video(track) => {
server.publish_video_track(this.room.token(), track).await?
}
LocalTrack::Audio(track) => {
server
.publish_audio_track(this.room.token(), &track)
.await?
}
};
Ok(LocalTrackPublication {
room: self.room.downgrade(),
sid,
})
let sid = server
.publish_audio_track(this.room.token(), &LocalAudioTrack {})
.await?;
Ok((
LocalTrackPublication {
room: self.room.downgrade(),
sid,
},
AudioStream {},
))
}
pub async fn publish_screenshare_track(
&self,
_source: &dyn ScreenCaptureSource,
_cx: &mut AsyncApp,
) -> Result<(LocalTrackPublication, Box<dyn ScreenCaptureStream>)> {
let this = self.clone();
let server = this.room.test_server();
let sid = server
.publish_video_track(this.room.token(), LocalVideoTrack {})
.await?;
Ok((
LocalTrackPublication {
room: self.room.downgrade(),
sid,
},
Box::new(TestScreenCaptureStream {}),
))
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RemoteParticipant {
pub fn track_publications(&self) -> HashMap<TrackSid, RemoteTrackPublication> {
if let Some(room) = self.room.upgrade() {
@ -109,3 +119,7 @@ impl RemoteParticipant {
self.identity.clone()
}
}
struct TestScreenCaptureStream;
impl gpui::ScreenCaptureStream for TestScreenCaptureStream {}

48
crates/livekit_client/src/test/publication.rs → crates/livekit_client/src/mock_client/publication.rs
View file

@ -1,54 +1,30 @@
use super::*;
use gpui::App;
#[derive(Clone, Debug)]
pub enum TrackPublication {
Local(LocalTrackPublication),
Remote(RemoteTrackPublication),
}
use crate::{test::WeakRoom, RemoteTrack, TrackSid};
#[derive(Clone, Debug)]
pub struct LocalTrackPublication {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(crate) sid: TrackSid,
pub(crate) room: WeakRoom,
}
#[derive(Clone, Debug)]
pub struct RemoteTrackPublication {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(crate) sid: TrackSid,
pub(crate) room: WeakRoom,
pub(crate) track: RemoteTrack,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl TrackPublication {
pub fn sid(&self) -> TrackSid {
match self {
TrackPublication::Local(track) => track.sid(),
TrackPublication::Remote(track) => track.sid(),
}
}
pub fn is_muted(&self) -> bool {
match self {
TrackPublication::Local(track) => track.is_muted(),
TrackPublication::Remote(track) => track.is_muted(),
}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl LocalTrackPublication {
pub fn sid(&self) -> TrackSid {
self.sid.clone()
}
pub fn mute(&self) {
pub fn mute(&self, _cx: &App) {
self.set_mute(true)
}
pub fn unmute(&self) {
pub fn unmute(&self, _cx: &App) {
self.set_mute(false)
}
@ -71,7 +47,6 @@ impl LocalTrackPublication {
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RemoteTrackPublication {
pub fn sid(&self) -> TrackSid {
self.sid.clone()
@ -81,8 +56,8 @@ impl RemoteTrackPublication {
Some(self.track.clone())
}
pub fn kind(&self) -> TrackKind {
self.track.kind()
pub fn is_audio(&self) -> bool {
matches!(self.track, RemoteTrack::Audio(_))
}
pub fn is_muted(&self) -> bool {
@ -103,7 +78,7 @@ impl RemoteTrackPublication {
}
}
pub fn set_enabled(&self, enabled: bool) {
pub fn set_enabled(&self, enabled: bool, _cx: &App) {
if let Some(room) = self.room.upgrade() {
let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks;
if enabled {
@ -114,3 +89,12 @@ impl RemoteTrackPublication {
}
}
}
impl RemoteTrack {
pub fn set_enabled(&self, enabled: bool, _cx: &App) {
match self {
RemoteTrack::Audio(remote_audio_track) => remote_audio_track.set_enabled(enabled),
RemoteTrack::Video(remote_video_track) => remote_video_track.set_enabled(enabled),
}
}
}

75
crates/livekit_client/src/mock_client/track.rs Normal file
View file

@ -0,0 +1,75 @@
use std::sync::Arc;
use crate::{
test::{TestServerAudioTrack, TestServerVideoTrack, WeakRoom},
ParticipantIdentity, TrackSid,
};
#[derive(Clone, Debug)]
pub struct LocalVideoTrack {}
#[derive(Clone, Debug)]
pub struct LocalAudioTrack {}
#[derive(Clone, Debug)]
pub struct RemoteVideoTrack {
pub(crate) server_track: Arc<TestServerVideoTrack>,
pub(crate) _room: WeakRoom,
}
#[derive(Clone, Debug)]
pub struct RemoteAudioTrack {
pub(crate) server_track: Arc<TestServerAudioTrack>,
pub(crate) room: WeakRoom,
}
impl RemoteAudioTrack {
pub fn sid(&self) -> TrackSid {
self.server_track.sid.clone()
}
pub fn publisher_id(&self) -> ParticipantIdentity {
self.server_track.publisher_id.clone()
}
pub fn enabled(&self) -> bool {
if let Some(room) = self.room.upgrade() {
!room
.0
.lock()
.paused_audio_tracks
.contains(&self.server_track.sid)
} else {
false
}
}
pub fn set_enabled(&self, enabled: bool) {
let Some(room) = self.room.upgrade() else {
return;
};
if enabled {
room.0
.lock()
.paused_audio_tracks
.remove(&self.server_track.sid);
} else {
room.0
.lock()
.paused_audio_tracks
.insert(self.server_track.sid.clone());
}
}
}
impl RemoteVideoTrack {
pub fn sid(&self) -> TrackSid {
self.server_track.sid.clone()
}
pub fn publisher_id(&self) -> ParticipantIdentity {
self.server_track.publisher_id.clone()
}
pub(crate) fn set_enabled(&self, _enabled: bool) {}
}

53
crates/livekit_client/src/remote_video_track_view.rs
View file

@ -1,5 +1,4 @@
use crate::track::RemoteVideoTrack;
use anyhow::Result;
use super::RemoteVideoTrack;
use futures::StreamExt as _;
use gpui::{
AppContext as _, Context, Empty, Entity, EventEmitter, IntoElement, Render, Task, Window,
@ -12,7 +11,7 @@ pub struct RemoteVideoTrackView {
current_rendered_frame: Option<crate::RemoteVideoFrame>,
#[cfg(not(target_os = "macos"))]
previous_rendered_frame: Option<crate::RemoteVideoFrame>,
_maintain_frame: Task<Result<()>>,
_maintain_frame: Task<()>,
}
#[derive(Debug)]
@ -23,8 +22,27 @@ pub enum RemoteVideoTrackViewEvent {
impl RemoteVideoTrackView {
pub fn new(track: RemoteVideoTrack, window: &mut Window, cx: &mut Context<Self>) -> Self {
cx.focus_handle();
let frames = super::play_remote_video_track(&track);
let _window_handle = window.window_handle();
let frames = crate::play_remote_video_track(&track);
#[cfg(not(target_os = "macos"))]
{
use util::ResultExt;
let window_handle = window.window_handle();
cx.on_release(move |this, cx| {
if let Some(frame) = this.previous_rendered_frame.take() {
window_handle
.update(cx, |_, window, _cx| window.drop_image(frame).log_err())
.ok();
}
if let Some(frame) = this.current_rendered_frame.take() {
window_handle
.update(cx, |_, window, _cx| window.drop_image(frame).log_err())
.ok();
}
})
.detach();
}
Self {
track,
@ -35,28 +53,11 @@ impl RemoteVideoTrackView {
this.update(cx, |this, cx| {
this.latest_frame = Some(frame);
cx.notify();
})?;
})
.ok();
}
this.update(cx, |_this, cx| {
#[cfg(not(target_os = "macos"))]
{
use util::ResultExt as _;
if let Some(frame) = _this.previous_rendered_frame.take() {
_window_handle
.update(cx, |_, window, _cx| window.drop_image(frame).log_err())
.ok();
}
// TODO(mgsloan): This might leak the last image of the screenshare if
// render is called after the screenshare ends.
if let Some(frame) = _this.current_rendered_frame.take() {
_window_handle
.update(cx, |_, window, _cx| window.drop_image(frame).log_err())
.ok();
}
}
cx.emit(RemoteVideoTrackViewEvent::Close)
})?;
Ok(())
this.update(cx, |_this, cx| cx.emit(RemoteVideoTrackViewEvent::Close))
.ok();
}),
#[cfg(not(target_os = "macos"))]
current_rendered_frame: None,

248
crates/livekit_client/src/test.rs
View file

@ -1,19 +1,10 @@
pub mod participant;
pub mod publication;
pub mod track;
use crate::{AudioStream, Participant, RemoteTrack, RoomEvent, TrackPublication};
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub mod webrtc;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
use self::id::*;
use self::{participant::*, publication::*, track::*};
use crate::mock_client::{participant::*, publication::*, track::*};
use anyhow::{anyhow, Context as _, Result};
use async_trait::async_trait;
use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet};
use gpui::BackgroundExecutor;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
use livekit::options::TrackPublishOptions;
use gpui::{App, AsyncApp, BackgroundExecutor};
use livekit_api::{proto, token};
use parking_lot::Mutex;
use postage::{mpsc, sink::Sink};
@ -22,8 +13,32 @@ use std::sync::{
Arc, Weak,
};
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub use livekit::{id, options, ConnectionState, DisconnectReason, RoomOptions};
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct ParticipantIdentity(pub String);
#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub struct TrackSid(pub(crate) String);
impl std::fmt::Display for TrackSid {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl TryFrom<String> for TrackSid {
type Error = anyhow::Error;
fn try_from(value: String) -> Result<Self, Self::Error> {
Ok(TrackSid(value))
}
}
#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
#[non_exhaustive]
pub enum ConnectionState {
Connected,
Disconnected,
}
static SERVERS: Mutex<BTreeMap<String, Arc<TestServer>>> = Mutex::new(BTreeMap::new());
@ -31,12 +46,10 @@ pub struct TestServer {
pub url: String,
pub api_key: String,
pub secret_key: String,
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
rooms: Mutex<HashMap<String, TestServerRoom>>,
executor: BackgroundExecutor,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl TestServer {
pub fn create(
url: String,
@ -83,7 +96,7 @@ impl TestServer {
}
pub async fn create_room(&self, room: String) -> Result<()> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let mut server_rooms = self.rooms.lock();
if let Entry::Vacant(e) = server_rooms.entry(room.clone()) {
@ -95,7 +108,7 @@ impl TestServer {
}
async fn delete_room(&self, room: String) -> Result<()> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let mut server_rooms = self.rooms.lock();
server_rooms
@ -105,7 +118,7 @@ impl TestServer {
}
async fn join_room(&self, token: String, client_room: Room) -> Result<ParticipantIdentity> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -172,7 +185,7 @@ impl TestServer {
}
async fn leave_room(&self, token: String) -> Result<()> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -229,7 +242,7 @@ impl TestServer {
room_name: String,
identity: ParticipantIdentity,
) -> Result<()> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let mut server_rooms = self.rooms.lock();
let room = server_rooms
@ -251,7 +264,7 @@ impl TestServer {
identity: String,
permission: proto::ParticipantPermission,
) -> Result<()> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let mut server_rooms = self.rooms.lock();
let room = server_rooms
@ -265,7 +278,7 @@ impl TestServer {
pub async fn disconnect_client(&self, client_identity: String) {
let client_identity = ParticipantIdentity(client_identity);
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let mut server_rooms = self.rooms.lock();
for room in server_rooms.values_mut() {
@ -274,19 +287,19 @@ impl TestServer {
room.connection_state = ConnectionState::Disconnected;
room.updates_tx
.blocking_send(RoomEvent::Disconnected {
reason: DisconnectReason::SignalClose,
reason: "SIGNAL_CLOSED",
})
.ok();
}
}
}
async fn publish_video_track(
pub(crate) async fn publish_video_track(
&self,
token: String,
_local_track: LocalVideoTrack,
) -> Result<TrackSid> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -347,12 +360,12 @@ impl TestServer {
Ok(sid)
}
async fn publish_audio_track(
pub(crate) async fn publish_audio_track(
&self,
token: String,
_local_track: &LocalAudioTrack,
) -> Result<TrackSid> {
self.executor.simulate_random_delay().await;
self.simulate_random_delay().await;
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -414,11 +427,16 @@ impl TestServer {
Ok(sid)
}
async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> {
pub(crate) async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> {
Ok(())
}
fn set_track_muted(&self, token: &str, track_sid: &TrackSid, muted: bool) -> Result<()> {
pub(crate) fn set_track_muted(
&self,
token: &str,
track_sid: &TrackSid,
muted: bool,
) -> Result<()> {
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let room_name = claims.video.room.unwrap();
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -472,7 +490,7 @@ impl TestServer {
Ok(())
}
fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option<bool> {
pub(crate) fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option<bool> {
let claims = livekit_api::token::validate(&token, &self.secret_key).ok()?;
let room_name = claims.video.room.unwrap();
@ -487,7 +505,7 @@ impl TestServer {
})
}
fn video_tracks(&self, token: String) -> Result<Vec<RemoteVideoTrack>> {
pub(crate) fn video_tracks(&self, token: String) -> Result<Vec<RemoteVideoTrack>> {
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let room_name = claims.video.room.unwrap();
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -510,7 +528,7 @@ impl TestServer {
.collect())
}
fn audio_tracks(&self, token: String) -> Result<Vec<RemoteAudioTrack>> {
pub(crate) fn audio_tracks(&self, token: String) -> Result<Vec<RemoteAudioTrack>> {
let claims = livekit_api::token::validate(&token, &self.secret_key)?;
let room_name = claims.video.room.unwrap();
let identity = ParticipantIdentity(claims.sub.unwrap().to_string());
@ -532,9 +550,13 @@ impl TestServer {
})
.collect())
}
async fn simulate_random_delay(&self) {
#[cfg(any(test, feature = "test-support"))]
self.executor.simulate_random_delay().await;
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
#[derive(Default, Debug)]
struct TestServerRoom {
client_rooms: HashMap<ParticipantIdentity, Room>,
@ -543,103 +565,24 @@ struct TestServerRoom {
participant_permissions: HashMap<ParticipantIdentity, proto::ParticipantPermission>,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
#[derive(Debug)]
struct TestServerVideoTrack {
sid: TrackSid,
publisher_id: ParticipantIdentity,
pub(crate) struct TestServerVideoTrack {
pub(crate) sid: TrackSid,
pub(crate) publisher_id: ParticipantIdentity,
// frames_rx: async_broadcast::Receiver<Frame>,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
#[derive(Debug)]
struct TestServerAudioTrack {
sid: TrackSid,
publisher_id: ParticipantIdentity,
muted: AtomicBool,
pub(crate) struct TestServerAudioTrack {
pub(crate) sid: TrackSid,
pub(crate) publisher_id: ParticipantIdentity,
pub(crate) muted: AtomicBool,
}
pub struct TestApiClient {
url: String,
}
#[derive(Clone, Debug)]
#[non_exhaustive]
pub enum RoomEvent {
ParticipantConnected(RemoteParticipant),
ParticipantDisconnected(RemoteParticipant),
LocalTrackPublished {
publication: LocalTrackPublication,
track: LocalTrack,
participant: LocalParticipant,
},
LocalTrackUnpublished {
publication: LocalTrackPublication,
participant: LocalParticipant,
},
TrackSubscribed {
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackUnsubscribed {
track: RemoteTrack,
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackSubscriptionFailed {
participant: RemoteParticipant,
error: String,
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
track_sid: TrackSid,
},
TrackPublished {
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackUnpublished {
publication: RemoteTrackPublication,
participant: RemoteParticipant,
},
TrackMuted {
participant: Participant,
publication: TrackPublication,
},
TrackUnmuted {
participant: Participant,
publication: TrackPublication,
},
RoomMetadataChanged {
old_metadata: String,
metadata: String,
},
ParticipantMetadataChanged {
participant: Participant,
old_metadata: String,
metadata: String,
},
ParticipantNameChanged {
participant: Participant,
old_name: String,
name: String,
},
ActiveSpeakersChanged {
speakers: Vec<Participant>,
},
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
ConnectionStateChanged(ConnectionState),
Connected {
participants_with_tracks: Vec<(RemoteParticipant, Vec<RemoteTrackPublication>)>,
},
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
Disconnected {
reason: DisconnectReason,
},
Reconnecting,
Reconnected,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
#[async_trait]
impl livekit_api::Client for TestApiClient {
fn url(&self) -> &str {
@ -700,25 +643,21 @@ impl livekit_api::Client for TestApiClient {
}
}
struct RoomState {
url: String,
token: String,
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
local_identity: ParticipantIdentity,
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
connection_state: ConnectionState,
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
paused_audio_tracks: HashSet<TrackSid>,
updates_tx: mpsc::Sender<RoomEvent>,
pub(crate) struct RoomState {
pub(crate) url: String,
pub(crate) token: String,
pub(crate) local_identity: ParticipantIdentity,
pub(crate) connection_state: ConnectionState,
pub(crate) paused_audio_tracks: HashSet<TrackSid>,
pub(crate) updates_tx: mpsc::Sender<RoomEvent>,
}
#[derive(Clone, Debug)]
pub struct Room(Arc<Mutex<RoomState>>);
pub struct Room(pub(crate) Arc<Mutex<RoomState>>);
#[derive(Clone, Debug)]
pub(crate) struct WeakRoom(Weak<Mutex<RoomState>>);
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl std::fmt::Debug for RoomState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Room")
@ -731,19 +670,8 @@ impl std::fmt::Debug for RoomState {
}
}
#[cfg(all(target_os = "windows", target_env = "gnu"))]
impl std::fmt::Debug for RoomState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Room")
.field("url", &self.url)
.field("token", &self.token)
.finish()
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl Room {
fn downgrade(&self) -> WeakRoom {
pub(crate) fn downgrade(&self) -> WeakRoom {
WeakRoom(Arc::downgrade(&self.0))
}
@ -760,9 +688,9 @@ impl Room {
}
pub async fn connect(
url: &str,
token: &str,
_options: RoomOptions,
url: String,
token: String,
_cx: &mut AsyncApp,
) -> Result<(Self, mpsc::Receiver<RoomEvent>)> {
let server = TestServer::get(&url)?;
let (updates_tx, updates_rx) = mpsc::channel(1024);
@ -794,16 +722,34 @@ impl Room {
.unwrap()
}
fn test_server(&self) -> Arc<TestServer> {
pub(crate) fn test_server(&self) -> Arc<TestServer> {
TestServer::get(&self.0.lock().url).unwrap()
}
fn token(&self) -> String {
pub(crate) fn token(&self) -> String {
self.0.lock().token.clone()
}
pub fn play_remote_audio_track(
&self,
_track: &RemoteAudioTrack,
_cx: &App,
) -> anyhow::Result<AudioStream> {
Ok(AudioStream {})
}
pub async fn unpublish_local_track(&self, sid: TrackSid, cx: &mut AsyncApp) -> Result<()> {
self.local_participant().unpublish_track(sid, cx).await
}
pub async fn publish_local_microphone_track(
&self,
cx: &mut AsyncApp,
) -> Result<(LocalTrackPublication, AudioStream)> {
self.local_participant().publish_microphone_track(cx).await
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl Drop for RoomState {
fn drop(&mut self) {
if self.connection_state == ConnectionState::Connected {
@ -819,7 +765,7 @@ impl Drop for RoomState {
}
impl WeakRoom {
fn upgrade(&self) -> Option<Room> {
pub(crate) fn upgrade(&self) -> Option<Room> {
self.0.upgrade().map(Room)
}
}

201
crates/livekit_client/src/test/track.rs
View file

@ -1,201 +0,0 @@
use super::*;
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
use webrtc::{audio_source::RtcAudioSource, video_source::RtcVideoSource};
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub use livekit::track::{TrackKind, TrackSource};
#[derive(Clone, Debug)]
pub enum LocalTrack {
Audio(LocalAudioTrack),
Video(LocalVideoTrack),
}
#[derive(Clone, Debug)]
pub enum RemoteTrack {
Audio(RemoteAudioTrack),
Video(RemoteVideoTrack),
}
#[derive(Clone, Debug)]
pub struct LocalVideoTrack {}
#[derive(Clone, Debug)]
pub struct LocalAudioTrack {}
#[derive(Clone, Debug)]
pub struct RemoteVideoTrack {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) server_track: Arc<TestServerVideoTrack>,
pub(super) _room: WeakRoom,
}
#[derive(Clone, Debug)]
pub struct RemoteAudioTrack {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) server_track: Arc<TestServerAudioTrack>,
pub(super) room: WeakRoom,
}
pub enum RtcTrack {
Audio(RtcAudioTrack),
Video(RtcVideoTrack),
}
pub struct RtcAudioTrack {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) server_track: Arc<TestServerAudioTrack>,
pub(super) room: WeakRoom,
}
pub struct RtcVideoTrack {
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
pub(super) _server_track: Arc<TestServerVideoTrack>,
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RemoteTrack {
pub fn sid(&self) -> TrackSid {
match self {
RemoteTrack::Audio(track) => track.sid(),
RemoteTrack::Video(track) => track.sid(),
}
}
pub fn kind(&self) -> TrackKind {
match self {
RemoteTrack::Audio(_) => TrackKind::Audio,
RemoteTrack::Video(_) => TrackKind::Video,
}
}
pub fn publisher_id(&self) -> ParticipantIdentity {
match self {
RemoteTrack::Audio(track) => track.publisher_id(),
RemoteTrack::Video(track) => track.publisher_id(),
}
}
pub fn rtc_track(&self) -> RtcTrack {
match self {
RemoteTrack::Audio(track) => RtcTrack::Audio(track.rtc_track()),
RemoteTrack::Video(track) => RtcTrack::Video(track.rtc_track()),
}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl LocalVideoTrack {
pub fn create_video_track(_name: &str, _source: RtcVideoSource) -> Self {
Self {}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl LocalAudioTrack {
pub fn create_audio_track(_name: &str, _source: RtcAudioSource) -> Self {
Self {}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RemoteAudioTrack {
pub fn sid(&self) -> TrackSid {
self.server_track.sid.clone()
}
pub fn publisher_id(&self) -> ParticipantIdentity {
self.server_track.publisher_id.clone()
}
pub fn start(&self) {
if let Some(room) = self.room.upgrade() {
room.0
.lock()
.paused_audio_tracks
.remove(&self.server_track.sid);
}
}
pub fn stop(&self) {
if let Some(room) = self.room.upgrade() {
room.0
.lock()
.paused_audio_tracks
.insert(self.server_track.sid.clone());
}
}
pub fn rtc_track(&self) -> RtcAudioTrack {
RtcAudioTrack {
server_track: self.server_track.clone(),
room: self.room.clone(),
}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RemoteVideoTrack {
pub fn sid(&self) -> TrackSid {
self.server_track.sid.clone()
}
pub fn publisher_id(&self) -> ParticipantIdentity {
self.server_track.publisher_id.clone()
}
pub fn rtc_track(&self) -> RtcVideoTrack {
RtcVideoTrack {
_server_track: self.server_track.clone(),
}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RtcTrack {
pub fn enabled(&self) -> bool {
match self {
RtcTrack::Audio(track) => track.enabled(),
RtcTrack::Video(track) => track.enabled(),
}
}
pub fn set_enabled(&self, enabled: bool) {
match self {
RtcTrack::Audio(track) => track.set_enabled(enabled),
RtcTrack::Video(_) => {}
}
}
}
#[cfg(not(all(target_os = "windows", target_env = "gnu")))]
impl RtcAudioTrack {
pub fn set_enabled(&self, enabled: bool) {
if let Some(room) = self.room.upgrade() {
let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks;
if enabled {
paused_audio_tracks.remove(&self.server_track.sid);
} else {
paused_audio_tracks.insert(self.server_track.sid.clone());
}
}
}
pub fn enabled(&self) -> bool {
if let Some(room) = self.room.upgrade() {
!room
.0
.lock()
.paused_audio_tracks
.contains(&self.server_track.sid)
} else {
false
}
}
}
impl RtcVideoTrack {
pub fn enabled(&self) -> bool {
true
}
}

136
crates/livekit_client/src/test/webrtc.rs
View file

@ -1,136 +0,0 @@
use super::track::{RtcAudioTrack, RtcVideoTrack};
use futures::Stream;
use livekit::webrtc as real;
use std::{
pin::Pin,
task::{Context, Poll},
};
pub mod video_stream {
use super::*;
pub mod native {
use super::*;
use real::video_frame::BoxVideoFrame;
pub struct NativeVideoStream {
pub track: RtcVideoTrack,
}
impl NativeVideoStream {
pub fn new(track: RtcVideoTrack) -> Self {
Self { track }
}
}
impl Stream for NativeVideoStream {
type Item = BoxVideoFrame;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Option<Self::Item>> {
Poll::Pending
}
}
}
}
pub mod audio_stream {
use super::*;
pub mod native {
use super::*;
use real::audio_frame::AudioFrame;
pub struct NativeAudioStream {
pub track: RtcAudioTrack,
}
impl NativeAudioStream {
pub fn new(track: RtcAudioTrack, _sample_rate: i32, _num_channels: i32) -> Self {
Self { track }
}
}
impl Stream for NativeAudioStream {
type Item = AudioFrame<'static>;
fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<Option<Self::Item>> {
Poll::Pending
}
}
}
}
pub mod audio_source {
use super::*;
pub use real::audio_source::AudioSourceOptions;
pub mod native {
use std::sync::Arc;
use super::*;
use real::{audio_frame::AudioFrame, RtcError};
#[derive(Clone)]
pub struct NativeAudioSource {
pub options: Arc<AudioSourceOptions>,
pub sample_rate: u32,
pub num_channels: u32,
}
impl NativeAudioSource {
pub fn new(
options: AudioSourceOptions,
sample_rate: u32,
num_channels: u32,
_queue_size_ms: u32,
) -> Self {
Self {
options: Arc::new(options),
sample_rate,
num_channels,
}
}
pub async fn capture_frame(&self, _frame: &AudioFrame<'_>) -> Result<(), RtcError> {
Ok(())
}
}
}
pub enum RtcAudioSource {
Native(native::NativeAudioSource),
}
}
pub use livekit::webrtc::audio_frame;
pub use livekit::webrtc::video_frame;
pub mod video_source {
use super::*;
pub use real::video_source::VideoResolution;
pub struct RTCVideoSource;
pub mod native {
use super::*;
use real::video_frame::{VideoBuffer, VideoFrame};
#[derive(Clone)]
pub struct NativeVideoSource {
pub resolution: VideoResolution,
}
impl NativeVideoSource {
pub fn new(resolution: super::VideoResolution) -> Self {
Self { resolution }
}
pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, _frame: &VideoFrame<T>) {}
}
}
pub enum RtcVideoSource {
Native(native::NativeVideoSource),
}
}