From c2afc2271b1dd420f482217cb0fa9dafcd17aab6 Mon Sep 17 00:00:00 2001 From: Michael Sloan Date: Fri, 4 Apr 2025 15:31:03 -0600 Subject: [PATCH] Use scap library to implement screensharing on X11 (#27807) While `scap` does have support for Wayland and Windows, but haven't seen screensharing work properly there yet. So for now just adding support for X11 screensharing. WIP branches for enabling wayland and windows support: * https://github.com/zed-industries/zed/tree/wayland-screenshare * https://github.com/zed-industries/zed/tree/windows-screenshare Release Notes: - Added support for screensharing on X11 (Linux) --------- Co-authored-by: Conrad Co-authored-by: Mikayla Co-authored-by: Junkui Zhang <364772080@qq.com> --- Cargo.lock | 152 ++++++++++ Cargo.toml | 11 +- crates/gpui/Cargo.toml | 14 +- crates/gpui/src/app.rs | 5 + crates/gpui/src/platform.rs | 12 +- crates/gpui/src/platform/linux.rs | 3 + .../src/platform/linux/headless/client.rs | 22 +- crates/gpui/src/platform/linux/platform.rs | 13 +- .../gpui/src/platform/linux/wayland/client.rs | 24 +- crates/gpui/src/platform/linux/x11/client.rs | 19 +- crates/gpui/src/platform/mac/platform.rs | 4 + .../gpui/src/platform/mac/screen_capture.rs | 14 +- .../gpui/src/platform/scap_screen_capture.rs | 282 ++++++++++++++++++ crates/gpui/src/platform/test/platform.rs | 18 +- crates/gpui/src/platform/windows/platform.rs | 4 + crates/livekit_client/Cargo.toml | 9 +- .../src/livekit_client/playback.rs | 46 ++- crates/title_bar/src/collab.rs | 5 +- crates/workspace/src/workspace.rs | 12 - script/linux | 4 + 20 files changed, 624 insertions(+), 49 deletions(-) create mode 100644 crates/gpui/src/platform/scap_screen_capture.rs diff --git a/Cargo.lock b/Cargo.lock index 3a1fbe5e17..0073e341a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3454,6 +3454,19 @@ dependencies = [ "libc", ] +[[package]] +name = "core-graphics-helmer-fork" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32eb7c354ae9f6d437a6039099ce7ecd049337a8109b23d73e48e8ffba8e9cd5" +dependencies = [ + "bitflags 2.9.0", + "core-foundation 0.9.4", + "core-graphics-types 0.1.3", + "foreign-types 0.5.0", + "libc", +] + [[package]] name = "core-graphics-types" version = "0.1.3" @@ -4429,6 +4442,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "dispatch" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd0c93bb4b0c6d9b77f4435b0ae98c24d17f1c45b2ff844c6151a07256ca923b" + [[package]] name = "displaydoc" version = "0.2.5" @@ -6099,6 +6118,7 @@ dependencies = [ "refineable", "reqwest_client", "resvg", + "scap", "schemars", "seahash", "semantic_version", @@ -8125,6 +8145,7 @@ dependencies = [ "objc", "parking_lot", "postage", + "scap", "serde", "serde_json", "sha2", @@ -9136,6 +9157,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" dependencies = [ "malloc_buf", + "objc_exception", +] + +[[package]] +name = "objc-foundation" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1add1b659e36c9607c7aab864a76c7a4c2760cd0cd2e120f3fb8b952c7e22bf9" +dependencies = [ + "block", + "objc", + "objc_id", ] [[package]] @@ -9340,6 +9373,24 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + +[[package]] +name = "objc_id" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92d4ddb4bd7b50d730c215ff871754d0da6b2178849f8a2a2ab69712d0c073b" +dependencies = [ + "objc", +] + [[package]] name = "object" version = "0.36.7" @@ -11146,6 +11197,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" +[[package]] +name = "quick-xml" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +dependencies = [ + "memchr", +] + [[package]] name = "quick-xml" version = "0.32.0" @@ -12374,6 +12434,27 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scap" +version = "0.0.8" +source = "git+https://github.com/zed-industries/scap?rev=5715067104794aa356977c543e2f3e95c6183044#5715067104794aa356977c543e2f3e95c6183044" +dependencies = [ + "anyhow", + "cocoa 0.25.0", + "core-graphics-helmer-fork", + "log", + "objc", + "rand 0.8.5", + "screencapturekit", + "screencapturekit-sys", + "sysinfo", + "tao-core-video-sys", + "windows 0.61.1", + "windows-capture", + "x11", + "xcb", +] + [[package]] name = "schannel" version = "0.1.27" @@ -12440,6 +12521,29 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" +[[package]] +name = "screencapturekit" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5eeeb57ac94960cfe5ff4c402be6585ae4c8d29a2cf41b276048c2e849d64e" +dependencies = [ + "screencapturekit-sys", +] + +[[package]] +name = "screencapturekit-sys" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22411b57f7d49e7fe08025198813ee6fd65e1ee5eff4ebc7880c12c82bde4c60" +dependencies = [ + "block", + "dispatch", + "objc", + "objc-foundation", + "objc_id", + "once_cell", +] + [[package]] name = "scrypt" version = "0.11.0" @@ -14002,6 +14106,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bdb6fa0dfa67b38c1e66b7041ba9dcf23b99d8121907cd31c807a332f7a0bbb" +[[package]] +name = "tao-core-video-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271450eb289cb4d8d0720c6ce70c72c8c858c93dd61fc625881616752e6b98f6" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "objc", +] + [[package]] name = "tap" version = "1.0.1" @@ -16671,6 +16787,20 @@ dependencies = [ "windows-numerics", ] +[[package]] +name = "windows-capture" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6001b777f61cafce437201de46a019ed7f4afed3b669f02e5ce4e0759164cb3e" +dependencies = [ + "clap", + "ctrlc", + "parking_lot", + "rayon", + "thiserror 1.0.69", + "windows 0.58.0", +] + [[package]] name = "windows-collections" version = "0.2.0" @@ -17674,6 +17804,16 @@ dependencies = [ "tap", ] +[[package]] +name = "x11" +version = "2.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "502da5464ccd04011667b11c435cb992822c2c0dbde1770c988480d312a0db2e" +dependencies = [ + "libc", + "pkg-config", +] + [[package]] name = "x11-clipboard" version = "0.9.3" @@ -17712,6 +17852,18 @@ dependencies = [ "libc", ] +[[package]] +name = "xcb" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e2f212bb1a92cd8caac8051b829a6582ede155ccb60b5d5908b81b100952be" +dependencies = [ + "bitflags 1.3.2", + "libc", + "quick-xml 0.30.0", + "x11", +] + [[package]] name = "xcursor" version = "0.3.8" diff --git a/Cargo.toml b/Cargo.toml index bd4b9bbd2f..6693d57b4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,8 +400,12 @@ async-tungstenite = "0.28" async-watch = "0.3.1" async_zip = { version = "0.0.17", features = ["deflate", "deflate64"] } aws-config = { version = "1.5.16", features = ["behavior-version-latest"] } -aws-credential-types = { version = "1.2.1", features = ["hardcoded-credentials"] } -aws-sdk-bedrockruntime = { version = "1.73.0", features = ["behavior-version-latest"] } +aws-credential-types = { version = "1.2.1", features = [ + "hardcoded-credentials", +] } +aws-sdk-bedrockruntime = { version = "1.73.0", features = [ + "behavior-version-latest", +] } aws-smithy-runtime-api = { version = "1.7.3", features = ["http-1x", "client"] } aws-smithy-types = { version = "1.2.13", features = ["http-body-1-x"] } base64 = "0.22" @@ -508,6 +512,7 @@ rust-embed = { version = "8.4", features = ["include-exclude"] } rustc-hash = "2.1.0" rustls = { version = "0.23.22" } rustls-platform-verifier = "0.5.0" +scap = { git = "https://github.com/zed-industries/scap", rev = "5715067104794aa356977c543e2f3e95c6183044", default-features = false } schemars = { version = "0.8", features = ["impl_json_schema", "indexmap2"] } semver = "1.0" serde = { version = "1.0", features = ["derive", "rc"] } @@ -547,7 +552,7 @@ time = { version = "0.3", features = [ tiny_http = "0.8" toml = "0.8" tokio = { version = "1" } -tokio-tungstenite = { version = "0.26", features = ["__rustls-tls"]} +tokio-tungstenite = { version = "0.26", features = ["__rustls-tls"] } tower-http = "0.4.4" tree-sitter = { version = "0.25.3", features = ["wasm"] } tree-sitter-bash = "0.23" diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index 2a7218f584..78a460a1f6 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -49,6 +49,7 @@ wayland = [ "filedescriptor", "xkbcommon", "open", + "scap", ] x11 = [ "blade-graphics", @@ -65,6 +66,7 @@ x11 = [ "x11-clipboard", "filedescriptor", "open", + "scap" ] @@ -99,7 +101,11 @@ profiling.workspace = true rand = { optional = true, workspace = true } raw-window-handle = "0.6" refineable.workspace = true -resvg = { version = "0.45.0", default-features = false, features = ["text", "system-fonts", "memmap-fonts"] } +resvg = { version = "0.45.0", default-features = false, features = [ + "text", + "system-fonts", + "memmap-fonts", +] } usvg = { version = "0.45.0", default-features = false } schemars.workspace = true seahash = "4.1" @@ -159,6 +165,7 @@ cosmic-text = { version = "0.13.2", optional = true } font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "5474cfad4b719a72ec8ed2cb7327b2b01fd10568", features = [ "source-fontconfig-dlopen", ], optional = true } +scap = { workspace = true, optional = true } calloop = { version = "0.13.0" } filedescriptor = { version = "0.8.2", optional = true } @@ -193,7 +200,10 @@ x11rb = { version = "0.13.1", features = [ "resource_manager", "sync", ], optional = true } -xkbcommon = { version = "0.8.0", features = ["wayland", "x11"], optional = true } +xkbcommon = { version = "0.8.0", features = [ + "wayland", + "x11", +], optional = true } xim = { git = "https://github.com/XDeme1/xim-rs", rev = "d50d461764c2213655cd9cf65a0ea94c70d3c4fd", features = [ "x11rb-xcb", "x11rb-client", diff --git a/crates/gpui/src/app.rs b/crates/gpui/src/app.rs index 95d16c0dfe..8d0b186a52 100644 --- a/crates/gpui/src/app.rs +++ b/crates/gpui/src/app.rs @@ -650,6 +650,11 @@ impl App { self.platform.primary_display() } + /// Returns whether `screen_capture_sources` may work. + pub fn is_screen_capture_supported(&self) -> bool { + self.platform.is_screen_capture_supported() + } + /// Returns a list of available screen capture sources. pub fn screen_capture_sources( &self, diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index fdd4974be3..c118aa6249 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -26,6 +26,12 @@ mod test; #[cfg(target_os = "windows")] mod windows; +#[cfg(all( + any(target_os = "linux", target_os = "freebsd"), + any(feature = "wayland", feature = "x11"), +))] +pub(crate) mod scap_screen_capture; + use crate::{ Action, AnyWindowHandle, App, AsyncWindowContext, BackgroundExecutor, Bounds, DEFAULT_WINDOW_SIZE, DevicePixels, DispatchEventResult, Font, FontId, FontMetrics, FontRun, @@ -158,6 +164,7 @@ pub(crate) trait Platform: 'static { None } + fn is_screen_capture_supported(&self) -> bool; fn screen_capture_sources( &self, ) -> oneshot::Receiver>>>; @@ -246,13 +253,14 @@ pub trait PlatformDisplay: Send + Sync + Debug { /// A source of on-screen video content that can be captured. pub trait ScreenCaptureSource { /// Returns the video resolution of this source. - fn resolution(&self) -> Result>; + fn resolution(&self) -> Result>; /// Start capture video from this source, invoking the given callback /// with each frame. fn stream( &self, - frame_callback: Box, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, ) -> oneshot::Receiver>>; } diff --git a/crates/gpui/src/platform/linux.rs b/crates/gpui/src/platform/linux.rs index 089b52cf1e..20bc2ddca9 100644 --- a/crates/gpui/src/platform/linux.rs +++ b/crates/gpui/src/platform/linux.rs @@ -21,4 +21,7 @@ pub(crate) use wayland::*; #[cfg(feature = "x11")] pub(crate) use x11::*; +#[cfg(any(feature = "wayland", feature = "x11"))] +pub(crate) type PlatformScreenCaptureFrame = scap::frame::Frame; +#[cfg(not(any(feature = "wayland", feature = "x11")))] pub(crate) type PlatformScreenCaptureFrame = (); diff --git a/crates/gpui/src/platform/linux/headless/client.rs b/crates/gpui/src/platform/linux/headless/client.rs index 71fdc26d9e..46a3032fe0 100644 --- a/crates/gpui/src/platform/linux/headless/client.rs +++ b/crates/gpui/src/platform/linux/headless/client.rs @@ -1,13 +1,16 @@ use std::cell::RefCell; use std::rc::Rc; +use anyhow::anyhow; use calloop::{EventLoop, LoopHandle}; - +use futures::channel::oneshot; use util::ResultExt; use crate::platform::linux::LinuxClient; use crate::platform::{LinuxCommon, PlatformWindow}; -use crate::{AnyWindowHandle, CursorStyle, DisplayId, PlatformDisplay, WindowParams}; +use crate::{ + AnyWindowHandle, CursorStyle, DisplayId, PlatformDisplay, ScreenCaptureSource, WindowParams, +}; pub struct HeadlessClientState { pub(crate) _loop_handle: LoopHandle<'static, HeadlessClient>, @@ -63,6 +66,21 @@ impl LinuxClient for HeadlessClient { None } + fn is_screen_capture_supported(&self) -> bool { + false + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + let (mut tx, rx) = oneshot::channel(); + tx.send(Err(anyhow!( + "Headless mode does not support screen capture." + ))) + .ok(); + rx + } + fn active_window(&self) -> Option { None } diff --git a/crates/gpui/src/platform/linux/platform.rs b/crates/gpui/src/platform/linux/platform.rs index cbf5861df2..d02eea6dac 100644 --- a/crates/gpui/src/platform/linux/platform.rs +++ b/crates/gpui/src/platform/linux/platform.rs @@ -28,6 +28,7 @@ use crate::{ Pixels, Platform, PlatformDisplay, PlatformTextSystem, PlatformWindow, Point, Result, ScreenCaptureSource, Task, WindowAppearance, WindowParams, px, }; + #[cfg(any(feature = "wayland", feature = "x11"))] pub(crate) const SCROLL_LINES: f32 = 3.0; @@ -50,6 +51,10 @@ pub trait LinuxClient { #[allow(unused)] fn display(&self, id: DisplayId) -> Option>; fn primary_display(&self) -> Option>; + fn is_screen_capture_supported(&self) -> bool; + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>>; fn open_window( &self, @@ -230,12 +235,14 @@ impl Platform for P { self.displays() } + fn is_screen_capture_supported(&self) -> bool { + self.is_screen_capture_supported() + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { - let (mut tx, rx) = oneshot::channel(); - tx.send(Err(anyhow!("screen capture not implemented"))).ok(); - rx + self.screen_capture_sources() } fn active_window(&self) -> Option { diff --git a/crates/gpui/src/platform/linux/wayland/client.rs b/crates/gpui/src/platform/linux/wayland/client.rs index 6906dc123e..0255dd8776 100644 --- a/crates/gpui/src/platform/linux/wayland/client.rs +++ b/crates/gpui/src/platform/linux/wayland/client.rs @@ -7,6 +7,7 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::anyhow; use calloop::{ EventLoop, LoopHandle, timer::{TimeoutAction, Timer}, @@ -14,7 +15,7 @@ use calloop::{ use calloop_wayland_source::WaylandSource; use collections::HashMap; use filedescriptor::Pipe; - +use futures::channel::oneshot; use http_client::Url; use smallvec::SmallVec; use util::ResultExt; @@ -85,7 +86,8 @@ use crate::{ FileDropEvent, ForegroundExecutor, KeyDownEvent, KeyUpEvent, Keystroke, LinuxCommon, Modifiers, ModifiersChangedEvent, MouseButton, MouseDownEvent, MouseExitEvent, MouseMoveEvent, MouseUpEvent, NavigationDirection, Pixels, PlatformDisplay, PlatformInput, Point, SCROLL_LINES, - ScaledPixels, ScrollDelta, ScrollWheelEvent, Size, TouchPhase, WindowParams, point, px, size, + ScaledPixels, ScreenCaptureSource, ScrollDelta, ScrollWheelEvent, Size, TouchPhase, + WindowParams, point, px, size, }; /// Used to convert evdev scancode to xkb scancode @@ -633,6 +635,24 @@ impl LinuxClient for WaylandClient { None } + fn is_screen_capture_supported(&self) -> bool { + false + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + // TODO: Get screen capture working on wayland. Be sure to try window resizing as that may + // be tricky. + // + // start_scap_default_target_source() + let (sources_tx, sources_rx) = oneshot::channel(); + sources_tx + .send(Err(anyhow!("Wayland screen capture not yet implemented."))) + .ok(); + sources_rx + } + fn open_window( &self, handle: AnyWindowHandle, diff --git a/crates/gpui/src/platform/linux/x11/client.rs b/crates/gpui/src/platform/linux/x11/client.rs index 45c37221ab..3e375aac07 100644 --- a/crates/gpui/src/platform/linux/x11/client.rs +++ b/crates/gpui/src/platform/linux/x11/client.rs @@ -1,3 +1,4 @@ +use crate::platform::scap_screen_capture::scap_screen_sources; use core::str; use std::{ cell::RefCell, @@ -8,13 +9,13 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::Context as _; use calloop::{ EventLoop, LoopHandle, RegistrationToken, generic::{FdWrapper, Generic}, }; - -use anyhow::Context as _; use collections::HashMap; +use futures::channel::oneshot; use http_client::Url; use smallvec::SmallVec; use util::ResultExt; @@ -59,8 +60,8 @@ use crate::platform::{ use crate::{ AnyWindowHandle, Bounds, ClipboardItem, CursorStyle, DisplayId, FileDropEvent, Keystroke, Modifiers, ModifiersChangedEvent, MouseButton, Pixels, Platform, PlatformDisplay, - PlatformInput, Point, RequestFrameOptions, ScaledPixels, ScrollDelta, Size, TouchPhase, - WindowParams, X11Window, modifiers_from_xinput_info, point, px, + PlatformInput, Point, RequestFrameOptions, ScaledPixels, ScreenCaptureSource, ScrollDelta, + Size, TouchPhase, WindowParams, X11Window, modifiers_from_xinput_info, point, px, }; /// Value for DeviceId parameters which selects all devices. @@ -1327,6 +1328,16 @@ impl LinuxClient for X11Client { )) } + fn is_screen_capture_supported(&self) -> bool { + true + } + + fn screen_capture_sources( + &self, + ) -> oneshot::Receiver>>> { + scap_screen_sources(&self.0.borrow().common.foreground_executor) + } + fn open_window( &self, handle: AnyWindowHandle, diff --git a/crates/gpui/src/platform/mac/platform.rs b/crates/gpui/src/platform/mac/platform.rs index c830522cbe..0bda71369e 100644 --- a/crates/gpui/src/platform/mac/platform.rs +++ b/crates/gpui/src/platform/mac/platform.rs @@ -552,6 +552,10 @@ impl Platform for MacPlatform { .collect() } + fn is_screen_capture_supported(&self) -> bool { + true + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/gpui/src/platform/mac/screen_capture.rs b/crates/gpui/src/platform/mac/screen_capture.rs index 900e9d2181..8e9fc3d3f9 100644 --- a/crates/gpui/src/platform/mac/screen_capture.rs +++ b/crates/gpui/src/platform/mac/screen_capture.rs @@ -1,7 +1,7 @@ use crate::{ - Pixels, Size, + DevicePixels, ForegroundExecutor, Size, platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream}, - px, size, + size, }; use anyhow::{Result, anyhow}; use block::ConcreteBlock; @@ -48,7 +48,7 @@ const FRAME_CALLBACK_IVAR: &str = "frame_callback"; const SCStreamOutputTypeScreen: NSInteger = 0; impl ScreenCaptureSource for MacScreenCaptureSource { - fn resolution(&self) -> Result> { + fn resolution(&self) -> Result> { unsafe { let display_id: CGDirectDisplayID = msg_send![self.sc_display, displayID]; let display_mode_ref = CGDisplayCopyDisplayMode(display_id); @@ -56,13 +56,17 @@ impl ScreenCaptureSource for MacScreenCaptureSource { let height = CGDisplayModeGetPixelHeight(display_mode_ref); CGDisplayModeRelease(display_mode_ref); - Ok(size(px(width as f32), px(height as f32))) + Ok(size( + DevicePixels(width as i32), + DevicePixels(height as i32), + )) } } fn stream( &self, - frame_callback: Box, + _foreground_executor: &ForegroundExecutor, + frame_callback: Box, ) -> oneshot::Receiver>> { unsafe { let stream: id = msg_send![class!(SCStream), alloc]; diff --git a/crates/gpui/src/platform/scap_screen_capture.rs b/crates/gpui/src/platform/scap_screen_capture.rs new file mode 100644 index 0000000000..c5e2267a37 --- /dev/null +++ b/crates/gpui/src/platform/scap_screen_capture.rs @@ -0,0 +1,282 @@ +//! Screen capture for Linux and Windows +use crate::{ + DevicePixels, ForegroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, + Size, size, +}; +use anyhow::{Context as _, Result, anyhow}; +use futures::channel::oneshot; +use std::sync::Arc; +use std::sync::atomic::{self, AtomicBool}; + +/// Populates the receiver with the screens that can be captured. +/// +/// `scap_default_target_source` should be used instead on Wayland, since `scap_screen_sources` +/// won't return any results. +#[allow(dead_code)] +pub(crate) fn scap_screen_sources( + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (sources_tx, sources_rx) = oneshot::channel(); + get_screen_targets(sources_tx); + to_dyn_screen_capture_sources(sources_rx, foreground_executor) +} + +/// Starts screen capture for the default target, and populates the receiver with a single source +/// for it. The first frame of the screen capture is used to determine the size of the stream. +/// +/// On Wayland (Linux), prompts the user to select a target, and populates the receiver with a +/// single screen capture source for their selection. +#[allow(dead_code)] +pub(crate) fn start_scap_default_target_source( + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (sources_tx, sources_rx) = oneshot::channel(); + start_default_target_screen_capture(sources_tx); + to_dyn_screen_capture_sources(sources_rx, foreground_executor) +} + +struct ScapCaptureSource { + target: scap::Target, + size: Size, +} + +/// Populates the sender with the screens available for capture. +fn get_screen_targets(sources_tx: oneshot::Sender>>) { + // Due to use of blocking APIs, a new thread is used. + std::thread::spawn(|| { + let targets = match scap::get_all_targets() { + Ok(targets) => targets, + Err(err) => { + sources_tx.send(Err(err)).ok(); + return; + } + }; + let sources = targets + .iter() + .filter_map(|target| match target { + scap::Target::Display(display) => { + let size = Size { + width: DevicePixels(display.width as i32), + height: DevicePixels(display.height as i32), + }; + Some(ScapCaptureSource { + target: target.clone(), + size, + }) + } + scap::Target::Window(_) => None, + }) + .collect::>(); + sources_tx.send(Ok(sources)).ok(); + }); +} + +impl ScreenCaptureSource for ScapCaptureSource { + fn resolution(&self) -> Result> { + Ok(self.size) + } + + fn stream( + &self, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, + ) -> oneshot::Receiver>> { + let (stream_tx, stream_rx) = oneshot::channel(); + let target = self.target.clone(); + + // Due to use of blocking APIs, a dedicated thread is used. + std::thread::spawn(move || match new_scap_capturer(Some(target)) { + Ok(mut capturer) => { + capturer.start_capture(); + run_capture(capturer, frame_callback, stream_tx); + } + Err(e) => { + stream_tx.send(Err(e)).ok(); + } + }); + + to_dyn_screen_capture_stream(stream_rx, foreground_executor) + } +} + +struct ScapDefaultTargetCaptureSource { + // Sender populated by single call to `ScreenCaptureSource::stream`. + stream_call_tx: std::sync::mpsc::SyncSender<( + // Provides the result of `ScreenCaptureSource::stream`. + oneshot::Sender>, + // Callback for frames. + Box, + )>, + size: Size, +} + +/// Starts screen capture on the default capture target, and populates the sender with the source. +fn start_default_target_screen_capture( + sources_tx: oneshot::Sender>>, +) { + // Due to use of blocking APIs, a dedicated thread is used. + std::thread::spawn(|| { + let start_result = util::maybe!({ + let mut capturer = new_scap_capturer(None)?; + capturer.start_capture(); + let first_frame = capturer + .get_next_frame() + .context("Failed to get first frame of screenshare to get the size.")?; + let size = frame_size(&first_frame); + Ok((capturer, size)) + }); + + match start_result { + Err(e) => { + sources_tx.send(Err(e)).ok(); + } + Ok((capturer, size)) => { + let (stream_call_tx, stream_rx) = std::sync::mpsc::sync_channel(1); + sources_tx + .send(Ok(vec![ScapDefaultTargetCaptureSource { + stream_call_tx, + size, + }])) + .ok(); + let Ok((stream_tx, frame_callback)) = stream_rx.recv() else { + return; + }; + run_capture(capturer, frame_callback, stream_tx); + } + } + }); +} + +impl ScreenCaptureSource for ScapDefaultTargetCaptureSource { + fn resolution(&self) -> Result> { + Ok(self.size) + } + + fn stream( + &self, + foreground_executor: &ForegroundExecutor, + frame_callback: Box, + ) -> oneshot::Receiver>> { + let (tx, rx) = oneshot::channel(); + match self.stream_call_tx.try_send((tx, frame_callback)) { + Ok(()) => {} + Err(std::sync::mpsc::TrySendError::Full((tx, _))) + | Err(std::sync::mpsc::TrySendError::Disconnected((tx, _))) => { + // Note: support could be added for being called again after end of prior stream. + tx.send(Err(anyhow!( + "Can't call ScapDefaultTargetCaptureSource::stream multiple times." + ))) + .ok(); + } + } + to_dyn_screen_capture_stream(rx, foreground_executor) + } +} + +fn new_scap_capturer(target: Option) -> Result { + scap::capturer::Capturer::build(scap::capturer::Options { + fps: 60, + show_cursor: true, + show_highlight: true, + // Note that the actual frame output type may differ. + output_type: scap::frame::FrameType::YUVFrame, + output_resolution: scap::capturer::Resolution::Captured, + crop_area: None, + target, + excluded_targets: None, + }) +} + +fn run_capture( + mut capturer: scap::capturer::Capturer, + frame_callback: Box, + stream_tx: oneshot::Sender>, +) { + let cancel_stream = Arc::new(AtomicBool::new(false)); + let stream_send_result = stream_tx.send(Ok(ScapStream { + cancel_stream: cancel_stream.clone(), + })); + if let Err(_) = stream_send_result { + return; + } + while !cancel_stream.load(std::sync::atomic::Ordering::SeqCst) { + match capturer.get_next_frame() { + Ok(frame) => frame_callback(ScreenCaptureFrame(frame)), + Err(err) => { + log::error!("Halting screen capture due to error: {err}"); + break; + } + } + } + capturer.stop_capture(); +} + +struct ScapStream { + cancel_stream: Arc, +} + +impl ScreenCaptureStream for ScapStream {} + +impl Drop for ScapStream { + fn drop(&mut self) { + self.cancel_stream.store(true, atomic::Ordering::SeqCst); + } +} + +fn frame_size(frame: &scap::frame::Frame) -> Size { + let (width, height) = match frame { + scap::frame::Frame::YUVFrame(frame) => (frame.width, frame.height), + scap::frame::Frame::RGB(frame) => (frame.width, frame.height), + scap::frame::Frame::RGBx(frame) => (frame.width, frame.height), + scap::frame::Frame::XBGR(frame) => (frame.width, frame.height), + scap::frame::Frame::BGRx(frame) => (frame.width, frame.height), + scap::frame::Frame::BGR0(frame) => (frame.width, frame.height), + scap::frame::Frame::BGRA(frame) => (frame.width, frame.height), + }; + size(DevicePixels(width), DevicePixels(height)) +} + +/// This is used by `get_screen_targets` and `start_default_target_screen_capture` to turn their +/// results into `Box`. They need to `Send` their capture source, and so +/// the capture source structs are used as `Box` is not `Send`. +fn to_dyn_screen_capture_sources( + sources_rx: oneshot::Receiver>>, + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>>> { + let (dyn_sources_tx, dyn_sources_rx) = oneshot::channel(); + foreground_executor + .spawn(async move { + match sources_rx.await { + Ok(Ok(results)) => dyn_sources_tx + .send(Ok(results + .into_iter() + .map(|source| Box::new(source) as Box) + .collect::>())) + .ok(), + Ok(Err(err)) => dyn_sources_tx.send(Err(err)).ok(), + Err(oneshot::Canceled) => None, + } + }) + .detach(); + dyn_sources_rx +} + +/// Same motivation as `to_dyn_screen_capture_sources` above. +fn to_dyn_screen_capture_stream( + sources_rx: oneshot::Receiver>, + foreground_executor: &ForegroundExecutor, +) -> oneshot::Receiver>> { + let (dyn_sources_tx, dyn_sources_rx) = oneshot::channel(); + foreground_executor + .spawn(async move { + match sources_rx.await { + Ok(Ok(stream)) => dyn_sources_tx + .send(Ok(Box::new(stream) as Box)) + .ok(), + Ok(Err(err)) => dyn_sources_tx.send(Err(err)).ok(), + Err(oneshot::Canceled) => None, + } + }) + .detach(); + dyn_sources_rx +} diff --git a/crates/gpui/src/platform/test/platform.rs b/crates/gpui/src/platform/test/platform.rs index e66465d2d2..90e3cf2fa6 100644 --- a/crates/gpui/src/platform/test/platform.rs +++ b/crates/gpui/src/platform/test/platform.rs @@ -1,7 +1,8 @@ use crate::{ - AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor, Keymap, - Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource, - ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams, px, size, + AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DevicePixels, + ForegroundExecutor, Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, + ScreenCaptureSource, ScreenCaptureStream, Size, Task, TestDisplay, TestWindow, + WindowAppearance, WindowParams, size, }; use anyhow::Result; use collections::VecDeque; @@ -46,13 +47,14 @@ pub struct TestScreenCaptureSource {} pub struct TestScreenCaptureStream {} impl ScreenCaptureSource for TestScreenCaptureSource { - fn resolution(&self) -> Result> { - Ok(size(px(1.), px(1.))) + fn resolution(&self) -> Result> { + Ok(size(DevicePixels(1), DevicePixels(1))) } fn stream( &self, - _frame_callback: Box, + _foreground_executor: &ForegroundExecutor, + _frame_callback: Box, ) -> oneshot::Receiver>> { let (mut tx, rx) = oneshot::channel(); let stream = TestScreenCaptureStream {}; @@ -271,6 +273,10 @@ impl Platform for TestPlatform { Some(self.active_display.clone()) } + fn is_screen_capture_supported(&self) -> bool { + true + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/gpui/src/platform/windows/platform.rs b/crates/gpui/src/platform/windows/platform.rs index e991bc93c6..116b2253d1 100644 --- a/crates/gpui/src/platform/windows/platform.rs +++ b/crates/gpui/src/platform/windows/platform.rs @@ -396,6 +396,10 @@ impl Platform for WindowsPlatform { WindowsDisplay::primary_monitor().map(|display| Rc::new(display) as Rc) } + fn is_screen_capture_supported(&self) -> bool { + false + } + fn screen_capture_sources( &self, ) -> oneshot::Receiver>>> { diff --git a/crates/livekit_client/Cargo.toml b/crates/livekit_client/Cargo.toml index dbd685ca03..7c9e07cd5c 100644 --- a/crates/livekit_client/Cargo.toml +++ b/crates/livekit_client/Cargo.toml @@ -25,7 +25,7 @@ async-trait.workspace = true collections.workspace = true cpal = "0.15" futures.workspace = true -gpui.workspace = true +gpui = { workspace = true, features = ["x11", "wayland"] } gpui_tokio.workspace = true http_client_tls.workspace = true image.workspace = true @@ -41,7 +41,12 @@ workspace-hack.workspace = true [target.'cfg(not(all(target_os = "windows", target_env = "gnu")))'.dependencies] libwebrtc = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks" } -livekit = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks", features = ["__rustls-tls"] } +livekit = { rev = "80bb8f4c9112789f7c24cc98d8423010977806a6", git = "https://github.com/zed-industries/livekit-rust-sdks", features = [ + "__rustls-tls" +] } + +[target.'cfg(any(target_os = "linux", target_os = "freebsd"))'.dependencies] +scap.workspace = true [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true diff --git a/crates/livekit_client/src/livekit_client/playback.rs b/crates/livekit_client/src/livekit_client/playback.rs index f3df5b86ca..18144e6948 100644 --- a/crates/livekit_client/src/livekit_client/playback.rs +++ b/crates/livekit_client/src/livekit_client/playback.rs @@ -336,7 +336,7 @@ pub(crate) async fn capture_local_video_track( .await?; let capture_stream = capture_source - .stream({ + .stream(cx.foreground_executor(), { let track_source = track_source.clone(); Box::new(move |frame| { if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { @@ -620,7 +620,49 @@ fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option Option> { + use libwebrtc::native::yuv_helper::argb_to_nv12; + use livekit::webrtc::prelude::NV12Buffer; + match frame.0 { + scap::frame::Frame::BGRx(frame) => { + let mut buffer = NV12Buffer::new(frame.width as u32, frame.height as u32); + let (stride_y, stride_uv) = buffer.strides(); + let (data_y, data_uv) = buffer.data_mut(); + argb_to_nv12( + &frame.data, + frame.width as u32 * 4, + data_y, + stride_y, + data_uv, + stride_uv, + frame.width, + frame.height, + ); + Some(buffer) + } + scap::frame::Frame::YUVFrame(yuvframe) => { + let mut buffer = NV12Buffer::with_strides( + yuvframe.width as u32, + yuvframe.height as u32, + yuvframe.luminance_stride as u32, + yuvframe.chrominance_stride as u32, + ); + let (luminance, chrominance) = buffer.data_mut(); + luminance.copy_from_slice(yuvframe.luminance_bytes.as_slice()); + chrominance.copy_from_slice(yuvframe.chrominance_bytes.as_slice()); + Some(buffer) + } + _ => { + log::error!( + "Expected BGRx or YUV frame from scap screen capture but got some other format." + ); + None + } + } +} + +#[cfg(target_os = "windows")] fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { None as Option> } diff --git a/crates/title_bar/src/collab.rs b/crates/title_bar/src/collab.rs index 20dec0e6ea..4a096ed034 100644 --- a/crates/title_bar/src/collab.rs +++ b/crates/title_bar/src/collab.rs @@ -299,10 +299,7 @@ impl TitleBar { let is_screen_sharing = room.is_screen_sharing(); let can_use_microphone = room.can_use_microphone(); let can_share_projects = room.can_share_projects(); - let screen_sharing_supported = match self.platform_style { - PlatformStyle::Mac => true, - PlatformStyle::Linux | PlatformStyle::Windows => false, - }; + let screen_sharing_supported = cx.is_screen_capture_supported(); let mut children = Vec::new(); diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 813e9ce005..397c3764bc 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -4423,18 +4423,6 @@ impl Workspace { None } - #[cfg(target_os = "windows")] - fn shared_screen_for_peer( - &self, - _peer_id: PeerId, - _pane: &Entity, - _window: &mut Window, - _cx: &mut App, - ) -> Option> { - None - } - - #[cfg(not(target_os = "windows"))] fn shared_screen_for_peer( &self, peer_id: PeerId, diff --git a/script/linux b/script/linux index 943c9d61b4..79e03df841 100755 --- a/script/linux +++ b/script/linux @@ -28,6 +28,7 @@ if [[ -n $apt ]]; then libasound2-dev libfontconfig-dev libwayland-dev + libx11-xcb-dev libxkbcommon-x11-dev libssl-dev libzstd-dev @@ -76,6 +77,7 @@ if [[ -n $dnf ]] || [[ -n $yum ]]; then alsa-lib-devel fontconfig-devel wayland-devel + libxcb-devel libxkbcommon-x11-devel openssl-devel libzstd-devel @@ -144,6 +146,7 @@ if [[ -n $zyp ]]; then gzip jq libvulkan1 + libxcb-devel libxkbcommon-devel libxkbcommon-x11-devel libzstd-devel @@ -174,6 +177,7 @@ if [[ -n $pacman ]]; then fontconfig wayland libgit2 + libxcb libxkbcommon-x11 openbsd-netcat openssl