Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Revert "Use livekit's Rust SDK instead of their swift SDK (#13343)" (#20809)

Browse source 
Issues found:

* audio does not work well with various set-ups using USB
* switching audio during initial join may leave the client with no audio
at all
* audio streaming is done on the main thread, beachballing certain
set-ups
* worse screenshare quality (seems that there's no dynamic scaling
anymore, compared to the Swift SDK)

This reverts commit 1235d0808e.

Release Notes:

- N/A
This commit is contained in:
Kirill Bulatov 2024-11-18 11:43:53 +02:00 • committed by GitHub
parent 59a355da74
commit d92166f9f6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
48 changed files with 3599 additions and 4288 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/gpui/src/app.rs
View file

@ -33,8 +33,8 @@ use crate::{
Entity, EventEmitter, ForegroundExecutor, Global, KeyBinding, Keymap, Keystroke, LayoutId,
Menu, MenuItem, OwnedMenu, PathPromptOptions, Pixels, Platform, PlatformDisplay, Point,
PromptBuilder, PromptHandle, PromptLevel, Render, RenderablePromptHandle, Reservation,
ScreenCaptureSource, SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem,
View, ViewContext, Window, WindowAppearance, WindowContext, WindowHandle, WindowId,
SharedString, SubscriberSet, Subscription, SvgRenderer, Task, TextSystem, View, ViewContext,
Window, WindowAppearance, WindowContext, WindowHandle, WindowId,
};
mod async_context;
@ -599,13 +599,6 @@ impl AppContext {
self.platform.primary_display()
}
/// Returns a list of available screen capture sources.
pub fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
self.platform.screen_capture_sources()
}
/// Returns the display with the given ID, if one exists.
pub fn find_display(&self, id: DisplayId) -> Option<Rc<dyn PlatformDisplay>> {
self.displays()

10
crates/gpui/src/app/test_context.rs
View file

@ -4,8 +4,8 @@ use crate::{
Element, Empty, Entity, EventEmitter, ForegroundExecutor, Global, InputEvent, Keystroke, Model,
ModelContext, Modifiers, ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseMoveEvent,
MouseUpEvent, Pixels, Platform, Point, Render, Result, Size, Task, TestDispatcher,
TestPlatform, TestScreenCaptureSource, TestWindow, TextSystem, View, ViewContext,
VisualContext, WindowBounds, WindowContext, WindowHandle, WindowOptions,
TestPlatform, TestWindow, TextSystem, View, ViewContext, VisualContext, WindowBounds,
WindowContext, WindowHandle, WindowOptions,
};
use anyhow::{anyhow, bail};
use futures::{channel::oneshot, Stream, StreamExt};
@ -287,12 +287,6 @@ impl TestAppContext {
self.test_window(window_handle).simulate_resize(size);
}
/// Causes the given sources to be returned if the application queries for screen
/// capture sources.
pub fn set_screen_capture_sources(&self, sources: Vec<TestScreenCaptureSource>) {
self.test_platform.set_screen_capture_sources(sources);
}
/// Returns all windows open in the test.
pub fn windows(&self) -> Vec<AnyWindowHandle> {
self.app.borrow().windows().clone()

5
crates/gpui/src/geometry.rs
View file

@ -704,11 +704,6 @@ pub struct Bounds<T: Clone + Default + Debug> {
pub size: Size<T>,
}
/// Create a bounds with the given origin and size
pub fn bounds<T: Clone + Default + Debug>(origin: Point<T>, size: Size<T>) -> Bounds<T> {
Bounds { origin, size }
}
impl Bounds<Pixels> {
/// Generate a centered bounds for the given display or primary display if none is provided
pub fn centered(display_id: Option<DisplayId>, size: Size<Pixels>, cx: &AppContext) -> Self {

26
crates/gpui/src/platform.rs
View file

@ -70,9 +70,6 @@ pub(crate) use test::*;
#[cfg(target_os = "windows")]
pub(crate) use windows::*;
#[cfg(any(test, feature = "test-support"))]
pub use test::TestScreenCaptureSource;
#[cfg(target_os = "macos")]
pub(crate) fn current_platform(headless: bool) -> Rc<dyn Platform> {
Rc::new(MacPlatform::new(headless))
@ -152,10 +149,6 @@ pub(crate) trait Platform: 'static {
None
}
fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>>;
fn open_window(
&self,
handle: AnyWindowHandle,
@ -235,25 +228,6 @@ pub trait PlatformDisplay: Send + Sync + Debug {
}
}
/// A source of on-screen video content that can be captured.
pub trait ScreenCaptureSource {
/// Returns the video resolution of this source.
fn resolution(&self) -> Result<Size<Pixels>>;
/// Start capture video from this source, invoking the given callback
/// with each frame.
fn stream(
&self,
frame_callback: Box<dyn Fn(ScreenCaptureFrame)>,
) -> oneshot::Receiver<Result<Box<dyn ScreenCaptureStream>>>;
}
/// A video stream captured from a screen.
pub trait ScreenCaptureStream {}
/// A frame of video captured from a screen.
pub struct ScreenCaptureFrame(pub PlatformScreenCaptureFrame);
/// An opaque identifier for a hardware display
#[derive(PartialEq, Eq, Hash, Copy, Clone)]
pub struct DisplayId(pub(crate) u32);

2
crates/gpui/src/platform/linux.rs
View file

@ -20,5 +20,3 @@ pub(crate) use text_system::*;
pub(crate) use wayland::*;
#[cfg(feature = "x11")]
pub(crate) use x11::*;
pub(crate) type PlatformScreenCaptureFrame = ();

12
crates/gpui/src/platform/linux/platform.rs
View file

@ -35,8 +35,8 @@ use crate::{
px, Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DisplayId,
ForegroundExecutor, Keymap, Keystroke, LinuxDispatcher, Menu, MenuItem, Modifiers, OwnedMenu,
PathPromptOptions, Pixels, Platform, PlatformDisplay, PlatformInputHandler, PlatformTextSystem,
PlatformWindow, Point, PromptLevel, Result, ScreenCaptureSource, SemanticVersion, SharedString,
Size, Task, WindowAppearance, WindowOptions, WindowParams,
PlatformWindow, Point, PromptLevel, Result, SemanticVersion, SharedString, Size, Task,
WindowAppearance, WindowOptions, WindowParams,
};
pub(crate) const SCROLL_LINES: f32 = 3.0;
@ -242,14 +242,6 @@ impl<P: LinuxClient + 'static> Platform for P {
self.displays()
}
fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
let (mut tx, rx) = oneshot::channel();
tx.send(Err(anyhow!("screen capture not implemented"))).ok();
rx
}
fn active_window(&self) -> Option<AnyWindowHandle> {
self.active_window()
}

5
crates/gpui/src/platform/mac.rs
View file

@ -4,14 +4,12 @@ mod dispatcher;
mod display;
mod display_link;
mod events;
mod screen_capture;
#[cfg(not(feature = "macos-blade"))]
mod metal_atlas;
#[cfg(not(feature = "macos-blade"))]
pub mod metal_renderer;
use media::core_video::CVImageBuffer;
#[cfg(not(feature = "macos-blade"))]
use metal_renderer as renderer;
@ -51,9 +49,6 @@ pub(crate) use window::*;
#[cfg(feature = "font-kit")]
pub(crate) use text_system::*;
/// A frame of video captured from a screen.
pub(crate) type PlatformScreenCaptureFrame = CVImageBuffer;
trait BoolExt {
fn to_objc(self) -> BOOL;
}

14
crates/gpui/src/platform/mac/platform.rs
View file

@ -1,14 +1,14 @@
use super::{
attributed_string::{NSAttributedString, NSMutableAttributedString},
events::key_to_native,
renderer, screen_capture, BoolExt,
BoolExt,
};
use crate::{
hash, Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem,
ClipboardString, CursorStyle, ForegroundExecutor, Image, ImageFormat, Keymap, MacDispatcher,
MacDisplay, MacWindow, Menu, MenuItem, PathPromptOptions, Platform, PlatformDisplay,
PlatformTextSystem, PlatformWindow, Result, ScreenCaptureSource, SemanticVersion, Task,
WindowAppearance, WindowParams,
PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task, WindowAppearance,
WindowParams,
};
use anyhow::anyhow;
use block::ConcreteBlock;
@ -58,6 +58,8 @@ use std::{
};
use strum::IntoEnumIterator;
use super::renderer;
#[allow(non_upper_case_globals)]
const NSUTF8StringEncoding: NSUInteger = 4;
@ -548,12 +550,6 @@ impl Platform for MacPlatform {
.collect()
}
fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
screen_capture::get_sources()
}
fn active_window(&self) -> Option<AnyWindowHandle> {
MacWindow::active_window()
}

239
crates/gpui/src/platform/mac/screen_capture.rs
View file

@ -1,239 +0,0 @@
use crate::{
platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream},
px, size, Pixels, Size,
};
use anyhow::{anyhow, Result};
use block::ConcreteBlock;
use cocoa::{
base::{id, nil, YES},
foundation::NSArray,
};
use core_foundation::base::TCFType;
use ctor::ctor;
use futures::channel::oneshot;
use media::core_media::{CMSampleBuffer, CMSampleBufferRef};
use metal::NSInteger;
use objc::{
class,
declare::ClassDecl,
msg_send,
runtime::{Class, Object, Sel},
sel, sel_impl,
};
use std::{cell::RefCell, ffi::c_void, mem, ptr, rc::Rc};
#[derive(Clone)]
pub struct MacScreenCaptureSource {
sc_display: id,
}
pub struct MacScreenCaptureStream {
sc_stream: id,
sc_stream_output: id,
}
#[link(name = "ScreenCaptureKit", kind = "framework")]
extern "C" {}
static mut DELEGATE_CLASS: *const Class = ptr::null();
static mut OUTPUT_CLASS: *const Class = ptr::null();
const FRAME_CALLBACK_IVAR: &str = "frame_callback";
#[allow(non_upper_case_globals)]
const SCStreamOutputTypeScreen: NSInteger = 0;
impl ScreenCaptureSource for MacScreenCaptureSource {
fn resolution(&self) -> Result<Size<Pixels>> {
unsafe {
let width: i64 = msg_send![self.sc_display, width];
let height: i64 = msg_send![self.sc_display, height];
Ok(size(px(width as f32), px(height as f32)))
}
}
fn stream(
&self,
frame_callback: Box<dyn Fn(ScreenCaptureFrame)>,
) -> oneshot::Receiver<Result<Box<dyn ScreenCaptureStream>>> {
unsafe {
let stream: id = msg_send![class!(SCStream), alloc];
let filter: id = msg_send![class!(SCContentFilter), alloc];
let configuration: id = msg_send![class!(SCStreamConfiguration), alloc];
let delegate: id = msg_send![DELEGATE_CLASS, alloc];
let output: id = msg_send![OUTPUT_CLASS, alloc];
let excluded_windows = NSArray::array(nil);
let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows];
let configuration: id = msg_send![configuration, init];
let delegate: id = msg_send![delegate, init];
let output: id = msg_send![output, init];
output.as_mut().unwrap().set_ivar(
FRAME_CALLBACK_IVAR,
Box::into_raw(Box::new(frame_callback)) as *mut c_void,
);
let stream: id = msg_send![stream, initWithFilter:filter configuration:configuration delegate:delegate];
let (mut tx, rx) = oneshot::channel();
let mut error: id = nil;
let _: () = msg_send![stream, addStreamOutput:output type:SCStreamOutputTypeScreen sampleHandlerQueue:0 error:&mut error as *mut id];
if error != nil {
let message: id = msg_send![error, localizedDescription];
tx.send(Err(anyhow!("failed to add stream output {message:?}")))
.ok();
return rx;
}
let tx = Rc::new(RefCell::new(Some(tx)));
let handler = ConcreteBlock::new({
move |error: id| {
let result = if error == nil {
let stream = MacScreenCaptureStream {
sc_stream: stream,
sc_stream_output: output,
};
Ok(Box::new(stream) as Box<dyn ScreenCaptureStream>)
} else {
let message: id = msg_send![error, localizedDescription];
Err(anyhow!("failed to stop screen capture stream {message:?}"))
};
if let Some(tx) = tx.borrow_mut().take() {
tx.send(result).ok();
}
}
});
let handler = handler.copy();
let _: () = msg_send![stream, startCaptureWithCompletionHandler:handler];
rx
}
}
}
impl Drop for MacScreenCaptureSource {
fn drop(&mut self) {
unsafe {
let _: () = msg_send![self.sc_display, release];
}
}
}
impl ScreenCaptureStream for MacScreenCaptureStream {}
impl Drop for MacScreenCaptureStream {
fn drop(&mut self) {
unsafe {
let mut error: id = nil;
let _: () = msg_send![self.sc_stream, removeStreamOutput:self.sc_stream_output type:SCStreamOutputTypeScreen error:&mut error as *mut _];
if error != nil {
let message: id = msg_send![error, localizedDescription];
log::error!("failed to add stream output {message:?}");
}
let handler = ConcreteBlock::new(move |error: id| {
if error != nil {
let message: id = msg_send![error, localizedDescription];
log::error!("failed to stop screen capture stream {message:?}");
}
});
let block = handler.copy();
let _: () = msg_send![self.sc_stream, stopCaptureWithCompletionHandler:block];
let _: () = msg_send![self.sc_stream, release];
let _: () = msg_send![self.sc_stream_output, release];
}
}
}
pub(crate) fn get_sources() -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
unsafe {
let (mut tx, rx) = oneshot::channel();
let tx = Rc::new(RefCell::new(Some(tx)));
let block = ConcreteBlock::new(move |shareable_content: id, error: id| {
let Some(mut tx) = tx.borrow_mut().take() else {
return;
};
let result = if error == nil {
let displays: id = msg_send![shareable_content, displays];
let mut result = Vec::new();
for i in 0..displays.count() {
let display = displays.objectAtIndex(i);
let source = MacScreenCaptureSource {
sc_display: msg_send![display, retain],
};
result.push(Box::new(source) as Box<dyn ScreenCaptureSource>);
}
Ok(result)
} else {
let msg: id = msg_send![error, localizedDescription];
Err(anyhow!("Failed to register: {:?}", msg))
};
tx.send(result).ok();
});
let block = block.copy();
let _: () = msg_send![
class!(SCShareableContent),
getShareableContentExcludingDesktopWindows:YES
onScreenWindowsOnly:YES
completionHandler:block];
rx
}
}
#[ctor]
unsafe fn build_classes() {
let mut decl = ClassDecl::new("GPUIStreamDelegate", class!(NSObject)).unwrap();
decl.add_method(
sel!(outputVideoEffectDidStartForStream:),
output_video_effect_did_start_for_stream as extern "C" fn(&Object, Sel, id),
);
decl.add_method(
sel!(outputVideoEffectDidStopForStream:),
output_video_effect_did_stop_for_stream as extern "C" fn(&Object, Sel, id),
);
decl.add_method(
sel!(stream:didStopWithError:),
stream_did_stop_with_error as extern "C" fn(&Object, Sel, id, id),
);
DELEGATE_CLASS = decl.register();
let mut decl = ClassDecl::new("GPUIStreamOutput", class!(NSObject)).unwrap();
decl.add_method(
sel!(stream:didOutputSampleBuffer:ofType:),
stream_did_output_sample_buffer_of_type as extern "C" fn(&Object, Sel, id, id, NSInteger),
);
decl.add_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR);
OUTPUT_CLASS = decl.register();
}
extern "C" fn output_video_effect_did_start_for_stream(_this: &Object, _: Sel, _stream: id) {}
extern "C" fn output_video_effect_did_stop_for_stream(_this: &Object, _: Sel, _stream: id) {}
extern "C" fn stream_did_stop_with_error(_this: &Object, _: Sel, _stream: id, _error: id) {}
extern "C" fn stream_did_output_sample_buffer_of_type(
this: &Object,
_: Sel,
_stream: id,
sample_buffer: id,
buffer_type: NSInteger,
) {
if buffer_type != SCStreamOutputTypeScreen {
return;
}
unsafe {
let sample_buffer = sample_buffer as CMSampleBufferRef;
let sample_buffer = CMSampleBuffer::wrap_under_get_rule(sample_buffer);
if let Some(buffer) = sample_buffer.image_buffer() {
let callback: Box<Box<dyn Fn(ScreenCaptureFrame)>> =
Box::from_raw(*this.get_ivar::<*mut c_void>(FRAME_CALLBACK_IVAR) as *mut _);
callback(ScreenCaptureFrame(buffer));
mem::forget(callback);
}
}
}

2
crates/gpui/src/platform/test.rs
View file

@ -7,5 +7,3 @@ pub(crate) use dispatcher::*;
pub(crate) use display::*;
pub(crate) use platform::*;
pub(crate) use window::*;
pub use platform::TestScreenCaptureSource;

58
crates/gpui/src/platform/test/platform.rs
View file

@ -1,7 +1,7 @@
use crate::{
px, size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor,
Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource,
ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams,
AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap,
Platform, PlatformDisplay, PlatformTextSystem, Task, TestDisplay, TestWindow, WindowAppearance,
WindowParams,
};
use anyhow::Result;
use collections::VecDeque;
@ -31,7 +31,6 @@ pub(crate) struct TestPlatform {
#[cfg(any(target_os = "linux", target_os = "freebsd"))]
current_primary_item: Mutex<Option<ClipboardItem>>,
pub(crate) prompts: RefCell<TestPrompts>,
screen_capture_sources: RefCell<Vec<TestScreenCaptureSource>>,
pub opened_url: RefCell<Option<String>>,
pub text_system: Arc<dyn PlatformTextSystem>,
#[cfg(target_os = "windows")]
@ -39,31 +38,6 @@ pub(crate) struct TestPlatform {
weak: Weak<Self>,
}
#[derive(Clone)]
/// A fake screen capture source, used for testing.
pub struct TestScreenCaptureSource {}
pub struct TestScreenCaptureStream {}
impl ScreenCaptureSource for TestScreenCaptureSource {
fn resolution(&self) -> Result<crate::Size<crate::Pixels>> {
Ok(size(px(1.), px(1.)))
}
fn stream(
&self,
_frame_callback: Box<dyn Fn(ScreenCaptureFrame)>,
) -> oneshot::Receiver<Result<Box<dyn ScreenCaptureStream>>> {
let (mut tx, rx) = oneshot::channel();
let stream = TestScreenCaptureStream {};
tx.send(Ok(Box::new(stream) as Box<dyn ScreenCaptureStream>))
.ok();
rx
}
}
impl ScreenCaptureStream for TestScreenCaptureStream {}
#[derive(Default)]
pub(crate) struct TestPrompts {
multiple_choice: VecDeque<oneshot::Sender<usize>>,
@ -98,7 +72,6 @@ impl TestPlatform {
background_executor: executor,
foreground_executor,
prompts: Default::default(),
screen_capture_sources: Default::default(),
active_cursor: Default::default(),
active_display: Rc::new(TestDisplay::new()),
active_window: Default::default(),
@ -141,10 +114,6 @@ impl TestPlatform {
!self.prompts.borrow().multiple_choice.is_empty()
}
pub(crate) fn set_screen_capture_sources(&self, sources: Vec<TestScreenCaptureSource>) {
*self.screen_capture_sources.borrow_mut() = sources;
}
pub(crate) fn prompt(&self, msg: &str, detail: Option<&str>) -> oneshot::Receiver<usize> {
let (tx, rx) = oneshot::channel();
self.background_executor()
@ -233,20 +202,6 @@ impl Platform for TestPlatform {
Some(self.active_display.clone())
}
fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
let (mut tx, rx) = oneshot::channel();
tx.send(Ok(self
.screen_capture_sources
.borrow()
.iter()
.map(|source| Box::new(source.clone()) as Box<dyn ScreenCaptureSource>)
.collect()))
.ok();
rx
}
fn active_window(&self) -> Option<crate::AnyWindowHandle> {
self.active_window
.borrow()
@ -375,13 +330,6 @@ impl Platform for TestPlatform {
}
}
impl TestScreenCaptureSource {
/// Create a fake screen capture source, for testing.
pub fn new() -> Self {
Self {}
}
}
#[cfg(target_os = "windows")]
impl Drop for TestPlatform {
fn drop(&mut self) {

2
crates/gpui/src/platform/windows.rs
View file

@ -21,5 +21,3 @@ pub(crate) use window::*;
pub(crate) use wrapper::*;
pub(crate) use windows::Win32::Foundation::HWND;
pub(crate) type PlatformScreenCaptureFrame = ();

8
crates/gpui/src/platform/windows/platform.rs
View file

@ -325,14 +325,6 @@ impl Platform for WindowsPlatform {
WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc<dyn PlatformDisplay>)
}
fn screen_capture_sources(
&self,
) -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptureSource>>>> {
let (mut tx, rx) = oneshot::channel();
tx.send(Err(anyhow!("screen capture not implemented"))).ok();
rx
}
fn active_window(&self) -> Option<AnyWindowHandle> {
let active_window_hwnd = unsafe { GetActiveWindow() };
self.try_get_windows_inner_from_hwnd(active_window_hwnd)