diff --git a/Cargo.lock b/Cargo.lock index 9dbdfe1fc6..7486a62082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,6 +2944,8 @@ dependencies = [ "core-foundation", "core-graphics", "futures", + "media", + "parking_lot 0.11.2", "serde", "serde_json", ] diff --git a/crates/capture/src/main.rs b/crates/capture/src/main.rs index f89bc4bf95..ee58aadf1a 100644 --- a/crates/capture/src/main.rs +++ b/crates/capture/src/main.rs @@ -1,7 +1,6 @@ mod live_kit_token; -use std::time::Duration; - +use futures::StreamExt; use gpui::{ actions, elements::{Canvas, *}, @@ -13,6 +12,7 @@ use live_kit::{LocalVideoTrack, Room}; use log::LevelFilter; use media::core_video::CVImageBuffer; use simplelog::SimpleLogger; +use std::sync::Arc; actions!(capture, [Quit]); @@ -36,47 +36,46 @@ fn main() { let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap(); let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap(); - let background = cx.background().clone(); - cx.foreground() - .spawn(async move { - println!("connecting..."); - let user1_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-1", - ) + cx.spawn(|mut cx| async move { + let user1_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-1", + ) + .unwrap(); + let room1 = Room::new(); + room1.connect(&live_kit_url, &user1_token).await.unwrap(); + + let user2_token = live_kit_token::create_token( + &live_kit_key, + &live_kit_secret, + "test-room", + "test-participant-2", + ) + .unwrap(); + let room2 = Room::new(); + room2.connect(&live_kit_url, &user2_token).await.unwrap(); + cx.add_window(Default::default(), |cx| ScreenCaptureView::new(room2, cx)); + + let windows = live_kit::list_windows(); + let window = windows + .iter() + .find(|w| w.owner_name.as_deref() == Some("Safari")) .unwrap(); - let room1 = Room::new("user-1 room"); - room1.connect(&live_kit_url, &user1_token).await.unwrap(); + let track = LocalVideoTrack::screen_share_for_window(window.id); + room1.publish_video_track(&track).await.unwrap(); - let user2_token = live_kit_token::create_token( - &live_kit_key, - &live_kit_secret, - "test-room", - "test-participant-2", - ) - .unwrap(); - let room2 = Room::new("user-2 room"); - room2.connect(&live_kit_url, &user2_token).await.unwrap(); - - let windows = live_kit::list_windows(); - println!("connected! {:?}", windows); - - let window_id = windows.iter().next().unwrap().id; - let track = LocalVideoTrack::screen_share_for_window(window_id); - room1.publish_video_track(&track).await.unwrap(); - - background.timer(Duration::from_secs(120)).await; - }) - .detach(); - - // cx.add_window(Default::default(), |cx| ScreenCaptureView::new(cx)); + std::mem::forget(track); + std::mem::forget(room1); + }) + .detach(); }); } struct ScreenCaptureView { image_buffer: Option, + _room: Arc, } impl gpui::Entity for ScreenCaptureView { @@ -84,8 +83,25 @@ impl gpui::Entity for ScreenCaptureView { } impl ScreenCaptureView { - pub fn new(_: &mut ViewContext) -> Self { - Self { image_buffer: None } + pub fn new(room: Arc, cx: &mut ViewContext) -> Self { + let mut remote_video_tracks = room.remote_video_tracks(); + cx.spawn_weak(|this, mut cx| async move { + if let Some(video_track) = remote_video_tracks.next().await { + video_track.add_renderer(move |frame| { + if let Some(this) = this.upgrade(&cx) { + this.update(&mut cx, |this, cx| { + this.image_buffer = Some(frame); + cx.notify(); + }); + } + }); + } + }) + .detach(); + Self { + image_buffer: None, + _room: room, + } } } diff --git a/crates/gpui/src/platform/mac/renderer.rs b/crates/gpui/src/platform/mac/renderer.rs index ad0915ed5b..21e67f3233 100644 --- a/crates/gpui/src/platform/mac/renderer.rs +++ b/crates/gpui/src/platform/mac/renderer.rs @@ -822,7 +822,7 @@ impl Renderer { { MTLPixelFormat::BGRA8Unorm } else { - panic!("unsupported pixel format") + MTLPixelFormat::R8Unorm }; let texture = self diff --git a/crates/live_kit/Cargo.toml b/crates/live_kit/Cargo.toml index d98b7eae88..e88d4f7b24 100644 --- a/crates/live_kit/Cargo.toml +++ b/crates/live_kit/Cargo.toml @@ -9,10 +9,13 @@ path = "src/live_kit.rs" doctest = false [dependencies] +media = { path = "../media" } + anyhow = "1.0.38" core-foundation = "0.9.3" core-graphics = "0.22.3" futures = "0.3" +parking_lot = "0.11.1" [build-dependencies] serde = { version = "1.0", features = ["derive", "rc"] } diff --git a/crates/live_kit/LiveKitBridge/Package.resolved b/crates/live_kit/LiveKitBridge/Package.resolved index df0b5d6243..e95cffe979 100644 --- a/crates/live_kit/LiveKitBridge/Package.resolved +++ b/crates/live_kit/LiveKitBridge/Package.resolved @@ -5,8 +5,7 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/livekit/client-sdk-swift.git", "state" : { - "revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d", - "version" : "1.0.4" + "revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4" } }, { diff --git a/crates/live_kit/LiveKitBridge/Package.swift b/crates/live_kit/LiveKitBridge/Package.swift index 9e7039998d..fe71558804 100644 --- a/crates/live_kit/LiveKitBridge/Package.swift +++ b/crates/live_kit/LiveKitBridge/Package.swift @@ -15,7 +15,7 @@ let package = Package( targets: ["LiveKitBridge"]), ], dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"), + .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "5cc3c001779ab147199ce3ea0dce465b846368b4"), ], targets: [ // Targets are the basic building blocks of a package. A target can define a module or a test suite. diff --git a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift index 1802fd91fb..fb1eb9a79e 100644 --- a/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ b/crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift @@ -1,17 +1,49 @@ import Foundation import LiveKit +import WebRTC class LKRoomDelegate: RoomDelegate { var data: UnsafeRawPointer - var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void + var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void - init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { + init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) { self.data = data - self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack + self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack } func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + if track.kind == .video { + self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque()) + } + } +} + +class LKVideoRenderer: NSObject, VideoRenderer { + var data: UnsafeRawPointer + var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void + var onDrop: @convention(c) (UnsafeRawPointer) -> Void + var adaptiveStreamIsEnabled: Bool = false + var adaptiveStreamSize: CGSize = .zero + + init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { + self.data = data + self.onFrame = onFrame + self.onDrop = onDrop + } + + deinit { + self.onDrop(self.data) + } + + func setSize(_ size: CGSize) { + print("Called setSize", size); + } + + func renderFrame(_ frame: RTCVideoFrame?) { + let buffer = frame?.buffer as? RTCCVPixelBuffer + if let pixelBuffer = buffer?.pixelBuffer { + self.onFrame(self.data, pixelBuffer) + } } } @@ -21,8 +53,8 @@ public func LKRelease(ptr: UnsafeRawPointer) { } @_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack) +public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack) return Unmanaged.passRetained(delegate).toOpaque() } @@ -59,3 +91,15 @@ public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutable let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId)) return Unmanaged.passRetained(track).toOpaque() } + +@_cdecl("LKVideoRendererCreate") +public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { + Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() +} + +@_cdecl("LKVideoTrackAddRenderer") +public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { + let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack + let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() + track.add(videoRenderer: renderer) +} diff --git a/crates/live_kit/src/live_kit.rs b/crates/live_kit/src/live_kit.rs index 0c254f99b3..59ce860a78 100644 --- a/crates/live_kit/src/live_kit.rs +++ b/crates/live_kit/src/live_kit.rs @@ -10,7 +10,12 @@ use core_graphics::window::{ kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly, kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo, }; -use futures::{channel::oneshot, Future}; +use futures::{ + channel::{mpsc, oneshot}, + Future, +}; +use media::core_video::{CVImageBuffer, CVImageBufferRef}; +use parking_lot::Mutex; use std::{ ffi::c_void, sync::{Arc, Weak}, @@ -20,8 +25,8 @@ extern "C" { fn LKRelease(object: *const c_void); fn LKRoomDelegateCreate( - callback_data: *const c_void, - on_did_subscribe_to_remote_track: extern "C" fn( + callback_data: *mut c_void, + on_did_subscribe_to_remote_video_track: extern "C" fn( callback_data: *mut c_void, remote_track: *const c_void, ), @@ -42,22 +47,30 @@ extern "C" { callback_data: *mut c_void, ); + fn LKVideoRendererCreate( + callback_data: *mut c_void, + on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef), + on_drop: extern "C" fn(callback_data: *mut c_void), + ) -> *const c_void; + + fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void); + fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void; } pub struct Room { - debug_name: &'static str, native_room: *const c_void, + remote_video_track_subscribers: Mutex>>>, _delegate: RoomDelegate, } impl Room { - pub fn new(debug_name: &'static str) -> Arc { + pub fn new() -> Arc { Arc::new_cyclic(|weak_room| { let delegate = RoomDelegate::new(weak_room.clone()); Self { - debug_name, native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, + remote_video_track_subscribers: Default::default(), _delegate: delegate, } }) @@ -88,8 +101,17 @@ impl Room { async { rx.await.unwrap().context("error publishing video track") } } - fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) { - println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name); + pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver> { + let (tx, rx) = mpsc::unbounded(); + self.remote_video_track_subscribers.lock().push(tx); + rx + } + + fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { + let track = Arc::new(track); + self.remote_video_track_subscribers + .lock() + .retain(|tx| tx.unbounded_send(track.clone()).is_ok()); } fn build_done_callback() -> ( @@ -131,8 +153,8 @@ impl RoomDelegate { let weak_room = Weak::into_raw(weak_room); let native_delegate = unsafe { LKRoomDelegateCreate( - weak_room as *const c_void, - Self::on_did_subscribe_to_remote_track, + weak_room as *mut c_void, + Self::on_did_subscribe_to_remote_video_track, ) }; Self { @@ -141,11 +163,11 @@ impl RoomDelegate { } } - extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) { + extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) { let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track = unsafe { RemoteVideoTrack(track) }; + let track = RemoteVideoTrack(track); if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_track(track); + room.did_subscribe_to_remote_video_track(track); } let _ = Weak::into_raw(room); } @@ -176,6 +198,37 @@ impl Drop for LocalVideoTrack { pub struct RemoteVideoTrack(*const c_void); +impl RemoteVideoTrack { + pub fn add_renderer(&self, callback: F) + where + F: 'static + FnMut(CVImageBuffer), + { + extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) + where + F: FnMut(CVImageBuffer), + { + unsafe { + let buffer = CVImageBuffer::wrap_under_get_rule(frame); + let callback = &mut *(callback_data as *mut F); + callback(buffer); + } + } + + extern "C" fn on_drop(callback_data: *mut c_void) { + unsafe { + let _ = Box::from_raw(callback_data as *mut F); + } + } + + let callback_data = Box::into_raw(Box::new(callback)); + unsafe { + let renderer = + LKVideoRendererCreate(callback_data as *mut c_void, on_frame::, on_drop::); + LKVideoTrackAddRenderer(self.0, renderer); + } + } +} + impl Drop for RemoteVideoTrack { fn drop(&mut self) { unsafe { LKRelease(self.0) }