Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP: render screen-sharing frames

Browse source
This commit is contained in:
Antonio Scandurra 2022-09-08 12:23:36 +02:00
parent 645338cff8
commit d407f521db
8 changed files with 178 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

3
crates/live_kit/Cargo.toml
View file

@ -9,10 +9,13 @@ path = "src/live_kit.rs"
doctest = false
[dependencies]
media = { path = "../media" }
anyhow = "1.0.38"
core-foundation = "0.9.3"
core-graphics = "0.22.3"
futures = "0.3"
parking_lot = "0.11.1"
[build-dependencies]
serde = { version = "1.0", features = ["derive", "rc"] }

3
crates/live_kit/LiveKitBridge/Package.resolved
View file

@ -5,8 +5,7 @@
"kind" : "remoteSourceControl",
"location" : "https://github.com/livekit/client-sdk-swift.git",
"state" : {
"revision" : "7e7decf3a09de4a169dfc0445a14d9fd2d8db58d",
"version" : "1.0.4"
"revision" : "5cc3c001779ab147199ce3ea0dce465b846368b4"
}
},
{

2
crates/live_kit/LiveKitBridge/Package.swift
View file

@ -15,7 +15,7 @@ let package = Package(
targets: ["LiveKitBridge"]),
],
dependencies: [
.package(url: "https://github.com/livekit/client-sdk-swift.git", from: "1.0.0"),
.package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "5cc3c001779ab147199ce3ea0dce465b846368b4"),
],
targets: [
// Targets are the basic building blocks of a package. A target can define a module or a test suite.

56
crates/live_kit/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift
View file

@ -1,17 +1,49 @@
import Foundation
import LiveKit
import WebRTC
class LKRoomDelegate: RoomDelegate {
var data: UnsafeRawPointer
var onDidSubscribeToRemoteTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void
var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void
init(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) {
init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) {
self.data = data
self.onDidSubscribeToRemoteTrack = onDidSubscribeToRemoteTrack
self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack
}
func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) {
self.onDidSubscribeToRemoteTrack(self.data, Unmanaged.passRetained(track).toOpaque())
if track.kind == .video {
self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque())
}
}
}
class LKVideoRenderer: NSObject, VideoRenderer {
var data: UnsafeRawPointer
var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void
var onDrop: @convention(c) (UnsafeRawPointer) -> Void
var adaptiveStreamIsEnabled: Bool = false
var adaptiveStreamSize: CGSize = .zero
init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) {
self.data = data
self.onFrame = onFrame
self.onDrop = onDrop
}
deinit {
self.onDrop(self.data)
}
func setSize(_ size: CGSize) {
print("Called setSize", size);
}
func renderFrame(_ frame: RTCVideoFrame?) {
let buffer = frame?.buffer as? RTCCVPixelBuffer
if let pixelBuffer = buffer?.pixelBuffer {
self.onFrame(self.data, pixelBuffer)
}
}
}
@ -21,8 +53,8 @@ public func LKRelease(ptr: UnsafeRawPointer) {
}
@_cdecl("LKRoomDelegateCreate")
public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer {
let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteTrack: onDidSubscribeToRemoteTrack)
public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer {
let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack)
return Unmanaged.passRetained(delegate).toOpaque()
}
@ -59,3 +91,15 @@ public func LKCreateScreenShareTrackForWindow(windowId: uint32) -> UnsafeMutable
let track = LocalVideoTrack.createMacOSScreenShareTrack(source: .window(id: windowId))
return Unmanaged.passRetained(track).toOpaque()
}
@_cdecl("LKVideoRendererCreate")
public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer {
Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque()
}
@_cdecl("LKVideoTrackAddRenderer")
public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) {
let track = Unmanaged<Track>.fromOpaque(track).takeUnretainedValue() as! VideoTrack
let renderer = Unmanaged<LKVideoRenderer>.fromOpaque(renderer).takeRetainedValue()
track.add(videoRenderer: renderer)
}

79
crates/live_kit/src/live_kit.rs
View file

@ -10,7 +10,12 @@ use core_graphics::window::{
kCGNullWindowID, kCGWindowListOptionExcludeDesktopElements, kCGWindowListOptionOnScreenOnly,
kCGWindowNumber, kCGWindowOwnerName, kCGWindowOwnerPID, CGWindowListCopyWindowInfo,
};
use futures::{channel::oneshot, Future};
use futures::{
channel::{mpsc, oneshot},
Future,
};
use media::core_video::{CVImageBuffer, CVImageBufferRef};
use parking_lot::Mutex;
use std::{
ffi::c_void,
sync::{Arc, Weak},
@ -20,8 +25,8 @@ extern "C" {
fn LKRelease(object: *const c_void);
fn LKRoomDelegateCreate(
callback_data: *const c_void,
on_did_subscribe_to_remote_track: extern "C" fn(
callback_data: *mut c_void,
on_did_subscribe_to_remote_video_track: extern "C" fn(
callback_data: *mut c_void,
remote_track: *const c_void,
),
@ -42,22 +47,30 @@ extern "C" {
callback_data: *mut c_void,
);
fn LKVideoRendererCreate(
callback_data: *mut c_void,
on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef),
on_drop: extern "C" fn(callback_data: *mut c_void),
) -> *const c_void;
fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void);
fn LKCreateScreenShareTrackForWindow(windowId: u32) -> *const c_void;
}
pub struct Room {
debug_name: &'static str,
native_room: *const c_void,
remote_video_track_subscribers: Mutex<Vec<mpsc::UnboundedSender<Arc<RemoteVideoTrack>>>>,
_delegate: RoomDelegate,
}
impl Room {
pub fn new(debug_name: &'static str) -> Arc<Self> {
pub fn new() -> Arc<Self> {
Arc::new_cyclic(|weak_room| {
let delegate = RoomDelegate::new(weak_room.clone());
Self {
debug_name,
native_room: unsafe { LKRoomCreate(delegate.native_delegate) },
remote_video_track_subscribers: Default::default(),
_delegate: delegate,
}
})
@ -88,8 +101,17 @@ impl Room {
async { rx.await.unwrap().context("error publishing video track") }
}
fn did_subscribe_to_remote_track(&self, track: RemoteVideoTrack) {
println!("{}: !!!!!!!!!!!!!!!!!!", self.debug_name);
pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver<Arc<RemoteVideoTrack>> {
let (tx, rx) = mpsc::unbounded();
self.remote_video_track_subscribers.lock().push(tx);
rx
}
fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) {
let track = Arc::new(track);
self.remote_video_track_subscribers
.lock()
.retain(|tx| tx.unbounded_send(track.clone()).is_ok());
}
fn build_done_callback() -> (
@ -131,8 +153,8 @@ impl RoomDelegate {
let weak_room = Weak::into_raw(weak_room);
let native_delegate = unsafe {
LKRoomDelegateCreate(
weak_room as *const c_void,
Self::on_did_subscribe_to_remote_track,
weak_room as *mut c_void,
Self::on_did_subscribe_to_remote_video_track,
)
};
Self {
@ -141,11 +163,11 @@ impl RoomDelegate {
}
}
extern "C" fn on_did_subscribe_to_remote_track(room: *mut c_void, track: *const c_void) {
extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) {
let room = unsafe { Weak::from_raw(room as *mut Room) };
let track = unsafe { RemoteVideoTrack(track) };
let track = RemoteVideoTrack(track);
if let Some(room) = room.upgrade() {
room.did_subscribe_to_remote_track(track);
room.did_subscribe_to_remote_video_track(track);
}
let _ = Weak::into_raw(room);
}
@ -176,6 +198,37 @@ impl Drop for LocalVideoTrack {
pub struct RemoteVideoTrack(*const c_void);
impl RemoteVideoTrack {
pub fn add_renderer<F>(&self, callback: F)
where
F: 'static + FnMut(CVImageBuffer),
{
extern "C" fn on_frame<F>(callback_data: *mut c_void, frame: CVImageBufferRef)
where
F: FnMut(CVImageBuffer),
{
unsafe {
let buffer = CVImageBuffer::wrap_under_get_rule(frame);
let callback = &mut *(callback_data as *mut F);
callback(buffer);
}
}
extern "C" fn on_drop<F>(callback_data: *mut c_void) {
unsafe {
let _ = Box::from_raw(callback_data as *mut F);
}
}
let callback_data = Box::into_raw(Box::new(callback));
unsafe {
let renderer =
LKVideoRendererCreate(callback_data as *mut c_void, on_frame::<F>, on_drop::<F>);
LKVideoTrackAddRenderer(self.0, renderer);
}
}
}
impl Drop for RemoteVideoTrack {
fn drop(&mut self) {
unsafe { LKRelease(self.0) }