Move live_kit to live_kit_client and add live_kit_server

crates/live_kit_client/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "live_kit_client"
+version = "0.1.0"
+edition = "2021"
+description = "Bindings to LiveKit Swift client SDK"
+
+[lib]
+path = "src/live_kit_client.rs"
+doctest = false
+
+[dependencies]
+media = { path = "../media" }
+
+anyhow = "1.0.38"
+core-foundation = "0.9.3"
+core-graphics = "0.22.3"
+futures = "0.3"
+parking_lot = "0.11.1"
+
+[build-dependencies]
+serde = { version = "1.0", features = ["derive", "rc"] }
+serde_json = { version = "1.0", features = ["preserve_order"] }

crates/live_kit_client/LiveKitBridge/.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+/.build
+/Packages
+/*.xcodeproj
+xcuserdata/
+DerivedData/
+.swiftpm/config/registries.json
+.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata
+.netrc

crates/live_kit_client/LiveKitBridge/Package.resolved

@@ -0,0 +1,52 @@
+{
+  "object": {
+    "pins": [
+      {
+        "package": "LiveKit",
+        "repositoryURL": "https://github.com/livekit/client-sdk-swift.git",
+        "state": {
+          "branch": null,
+          "revision": "a90ecba800f65bebfc31f5c9c59f635c8587ec7e",
+          "version": null
+        }
+      },
+      {
+        "package": "Promises",
+        "repositoryURL": "https://github.com/google/promises.git",
+        "state": {
+          "branch": null,
+          "revision": "3e4e743631e86c8c70dbc6efdc7beaa6e90fd3bb",
+          "version": "2.1.1"
+        }
+      },
+      {
+        "package": "WebRTC",
+        "repositoryURL": "https://github.com/webrtc-sdk/Specs.git",
+        "state": {
+          "branch": null,
+          "revision": "5225f2de4b6d0098803b3a0e55b255a41f293dad",
+          "version": "104.5112.2"
+        }
+      },
+      {
+        "package": "swift-log",
+        "repositoryURL": "https://github.com/apple/swift-log.git",
+        "state": {
+          "branch": null,
+          "revision": "6fe203dc33195667ce1759bf0182975e4653ba1c",
+          "version": "1.4.4"
+        }
+      },
+      {
+        "package": "SwiftProtobuf",
+        "repositoryURL": "https://github.com/apple/swift-protobuf.git",
+        "state": {
+          "branch": null,
+          "revision": "88c7d15e1242fdb6ecbafbc7926426a19be1e98a",
+          "version": "1.20.2"
+        }
+      }
+    ]
+  },
+  "version": 1
+}

crates/live_kit_client/LiveKitBridge/Package.swift

@@ -0,0 +1,27 @@
+// swift-tools-version: 5.5
+
+import PackageDescription
+
+let package = Package(
+    name: "LiveKitBridge",
+    platforms: [
+        .macOS(.v10_15)
+    ],
+    products: [
+        // Products define the executables and libraries a package produces, and make them visible to other packages.
+        .library(
+            name: "LiveKitBridge",
+            type: .static,
+            targets: ["LiveKitBridge"]),
+    ],
+    dependencies: [
+        .package(url: "https://github.com/livekit/client-sdk-swift.git", revision: "a90ecba800f65bebfc31f5c9c59f635c8587ec7e"),
+    ],
+    targets: [
+        // Targets are the basic building blocks of a package. A target can define a module or a test suite.
+        // Targets can depend on other targets in this package, and on products in packages this package depends on.
+        .target(
+            name: "LiveKitBridge",
+            dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]),
+    ]
+)

crates/live_kit_client/LiveKitBridge/README.md

@@ -0,0 +1,3 @@
+# LiveKitBridge
+
+A description of this package.

crates/live_kit_client/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift

@@ -0,0 +1,115 @@
+import Foundation
+import LiveKit
+import WebRTC
+
+class LKRoomDelegate: RoomDelegate {
+    var data: UnsafeRawPointer
+    var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void
+    
+    init(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) {
+        self.data = data
+        self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack
+    }
+
+    func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) {
+        if track.kind == .video {
+            self.onDidSubscribeToRemoteVideoTrack(self.data, Unmanaged.passRetained(track).toOpaque())
+        }
+    }
+}
+
+class LKVideoRenderer: NSObject, VideoRenderer {
+    var data: UnsafeRawPointer
+    var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void
+    var onDrop: @convention(c) (UnsafeRawPointer) -> Void
+    var adaptiveStreamIsEnabled: Bool = false
+    var adaptiveStreamSize: CGSize = .zero
+
+    init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) {
+        self.data = data
+        self.onFrame = onFrame
+        self.onDrop = onDrop
+    }
+
+    deinit {
+        self.onDrop(self.data)
+    }
+
+    func setSize(_ size: CGSize) {
+        print("Called setSize", size);
+    }
+
+    func renderFrame(_ frame: RTCVideoFrame?) {
+        let buffer = frame?.buffer as? RTCCVPixelBuffer
+        if let pixelBuffer = buffer?.pixelBuffer {
+            self.onFrame(self.data, pixelBuffer)
+        }
+    }
+}
+
+@_cdecl("LKRelease")
+public func LKRelease(ptr: UnsafeRawPointer)  {
+    let _ = Unmanaged<AnyObject>.fromOpaque(ptr).takeRetainedValue()
+}
+
+@_cdecl("LKRoomDelegateCreate")
+public func LKRoomDelegateCreate(data: UnsafeRawPointer, onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer {
+    let delegate = LKRoomDelegate(data: data, onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack)
+    return Unmanaged.passRetained(delegate).toOpaque()
+}
+
+@_cdecl("LKRoomCreate")
+public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer  {
+    let delegate = Unmanaged<LKRoomDelegate>.fromOpaque(delegate).takeUnretainedValue()
+    return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque()
+}
+
+@_cdecl("LKRoomConnect")
+public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) {
+    let room = Unmanaged<Room>.fromOpaque(room).takeUnretainedValue()
+
+    room.connect(url as String, token as String).then { _ in
+        callback(callback_data, UnsafeRawPointer(nil) as! CFString?)
+    }.catch { error in
+        callback(callback_data, error.localizedDescription as CFString)
+    }
+}
+
+@_cdecl("LKRoomPublishVideoTrack")
+public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) {
+    let room = Unmanaged<Room>.fromOpaque(room).takeUnretainedValue()
+    let track = Unmanaged<LocalVideoTrack>.fromOpaque(track).takeUnretainedValue()
+    room.localParticipant?.publishVideoTrack(track: track).then { _ in
+        callback(callback_data, UnsafeRawPointer(nil) as! CFString?)
+    }.catch { error in
+        callback(callback_data, error.localizedDescription as CFString)
+    }
+}
+
+@_cdecl("LKCreateScreenShareTrackForDisplay")
+public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer {
+    let display = Unmanaged<MacOSDisplay>.fromOpaque(display).takeUnretainedValue()
+    let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy)
+    return Unmanaged.passRetained(track).toOpaque()
+}
+
+@_cdecl("LKVideoRendererCreate")
+public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Void, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer {
+    Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque()
+}
+
+@_cdecl("LKVideoTrackAddRenderer")
+public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) {
+    let track = Unmanaged<Track>.fromOpaque(track).takeUnretainedValue() as! VideoTrack
+    let renderer = Unmanaged<LKVideoRenderer>.fromOpaque(renderer).takeRetainedValue()
+    track.add(videoRenderer: renderer)
+}
+
+@_cdecl("LKDisplaySources")
+public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) {
+    MacOSScreenCapturer.displaySources().then { displaySources in
+        callback(data, displaySources as CFArray, nil)
+    }.catch { error in
+        callback(data, nil, error.localizedDescription as CFString)
+    }
+}

crates/live_kit_client/build.rs

@@ -0,0 +1,148 @@
+use serde::Deserialize;
+use std::{
+    env,
+    path::{Path, PathBuf},
+    process::Command,
+};
+
+const SWIFT_PACKAGE_NAME: &'static str = "LiveKitBridge";
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SwiftTargetInfo {
+    pub triple: String,
+    pub unversioned_triple: String,
+    pub module_triple: String,
+    pub swift_runtime_compatibility_version: String,
+    #[serde(rename = "librariesRequireRPath")]
+    pub libraries_require_rpath: bool,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct SwiftPaths {
+    pub runtime_library_paths: Vec<String>,
+    pub runtime_library_import_paths: Vec<String>,
+    pub runtime_resource_path: String,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct SwiftTarget {
+    pub target: SwiftTargetInfo,
+    pub paths: SwiftPaths,
+}
+
+const MACOS_TARGET_VERSION: &str = "10.15";
+
+fn main() {
+    let swift_target = get_swift_target();
+
+    build_bridge(&swift_target);
+    link_swift_stdlib(&swift_target);
+    link_webrtc_framework(&swift_target);
+}
+
+fn build_bridge(swift_target: &SwiftTarget) {
+    println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME);
+    println!(
+        "cargo:rerun-if-changed={}/Package.swift",
+        SWIFT_PACKAGE_NAME
+    );
+    println!(
+        "cargo:rerun-if-changed={}/Package.resolved",
+        SWIFT_PACKAGE_NAME
+    );
+    let swift_package_root = swift_package_root();
+    if !Command::new("swift")
+        .arg("build")
+        .args(&["--configuration", &env::var("PROFILE").unwrap()])
+        .args(&["--triple", &swift_target.target.triple])
+        .current_dir(&swift_package_root)
+        .status()
+        .unwrap()
+        .success()
+    {
+        panic!(
+            "Failed to compile swift package in {}",
+            swift_package_root.display()
+        );
+    }
+
+    println!(
+        "cargo:rustc-link-search=native={}",
+        swift_target.out_dir_path().display()
+    );
+    println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME);
+}
+
+fn link_swift_stdlib(swift_target: &SwiftTarget) {
+    swift_target
+        .paths
+        .runtime_library_paths
+        .iter()
+        .for_each(|path| {
+            println!("cargo:rustc-link-search=native={}", path);
+        });
+}
+
+fn link_webrtc_framework(swift_target: &SwiftTarget) {
+    let swift_out_dir_path = swift_target.out_dir_path();
+    println!("cargo:rustc-link-lib=framework=WebRTC");
+    println!(
+        "cargo:rustc-link-search=framework={}",
+        swift_out_dir_path.display()
+    );
+    // Find WebRTC.framework as a sibling of the executable when running tests.
+    println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path");
+
+    let source_path = swift_out_dir_path.join("WebRTC.framework");
+    let deps_dir_path =
+        PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework");
+    let target_dir_path =
+        PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework");
+    copy_dir(&source_path, &deps_dir_path);
+    copy_dir(&source_path, &target_dir_path);
+}
+
+fn get_swift_target() -> SwiftTarget {
+    let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
+    if arch == "aarch64" {
+        arch = "arm64".into();
+    }
+    let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION);
+
+    let swift_target_info_str = Command::new("swift")
+        .args(&["-target", &target, "-print-target-info"])
+        .output()
+        .unwrap()
+        .stdout;
+
+    serde_json::from_slice(&swift_target_info_str).unwrap()
+}
+
+fn swift_package_root() -> PathBuf {
+    env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME)
+}
+
+fn copy_dir(source: &Path, destination: &Path) {
+    assert!(
+        Command::new("cp")
+            .arg("-r")
+            .args(&[source, destination])
+            .status()
+            .unwrap()
+            .success(),
+        "could not copy {:?} to {:?}",
+        source,
+        destination
+    );
+}
+
+impl SwiftTarget {
+    fn out_dir_path(&self) -> PathBuf {
+        swift_package_root()
+            .join(".build")
+            .join(&self.target.unversioned_triple)
+            .join(env::var("PROFILE").unwrap())
+    }
+}

crates/live_kit_client/src/live_kit_client.rs

@@ -0,0 +1,270 @@
+use anyhow::{anyhow, Context, Result};
+use core_foundation::{
+    array::{CFArray, CFArrayRef},
+    base::TCFType,
+    string::{CFString, CFStringRef},
+};
+use futures::{
+    channel::{mpsc, oneshot},
+    Future,
+};
+use media::core_video::{CVImageBuffer, CVImageBufferRef};
+use parking_lot::Mutex;
+use std::{
+    ffi::c_void,
+    sync::{Arc, Weak},
+};
+
+extern "C" {
+    fn LKRelease(object: *const c_void);
+
+    fn LKRoomDelegateCreate(
+        callback_data: *mut c_void,
+        on_did_subscribe_to_remote_video_track: extern "C" fn(
+            callback_data: *mut c_void,
+            remote_track: *const c_void,
+        ),
+    ) -> *const c_void;
+
+    fn LKRoomCreate(delegate: *const c_void) -> *const c_void;
+    fn LKRoomConnect(
+        room: *const c_void,
+        url: CFStringRef,
+        token: CFStringRef,
+        callback: extern "C" fn(*mut c_void, CFStringRef),
+        callback_data: *mut c_void,
+    );
+    fn LKRoomPublishVideoTrack(
+        room: *const c_void,
+        track: *const c_void,
+        callback: extern "C" fn(*mut c_void, CFStringRef),
+        callback_data: *mut c_void,
+    );
+
+    fn LKVideoRendererCreate(
+        callback_data: *mut c_void,
+        on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef),
+        on_drop: extern "C" fn(callback_data: *mut c_void),
+    ) -> *const c_void;
+
+    fn LKVideoTrackAddRenderer(track: *const c_void, renderer: *const c_void);
+
+    fn LKDisplaySources(
+        callback_data: *mut c_void,
+        callback: extern "C" fn(
+            callback_data: *mut c_void,
+            sources: CFArrayRef,
+            error: CFStringRef,
+        ),
+    );
+    fn LKCreateScreenShareTrackForDisplay(display: *const c_void) -> *const c_void;
+}
+
+pub struct Room {
+    native_room: *const c_void,
+    remote_video_track_subscribers: Mutex<Vec<mpsc::UnboundedSender<Arc<RemoteVideoTrack>>>>,
+    _delegate: RoomDelegate,
+}
+
+impl Room {
+    pub fn new() -> Arc<Self> {
+        Arc::new_cyclic(|weak_room| {
+            let delegate = RoomDelegate::new(weak_room.clone());
+            Self {
+                native_room: unsafe { LKRoomCreate(delegate.native_delegate) },
+                remote_video_track_subscribers: Default::default(),
+                _delegate: delegate,
+            }
+        })
+    }
+
+    pub fn connect(&self, url: &str, token: &str) -> impl Future<Output = Result<()>> {
+        let url = CFString::new(url);
+        let token = CFString::new(token);
+        let (did_connect, tx, rx) = Self::build_done_callback();
+        unsafe {
+            LKRoomConnect(
+                self.native_room,
+                url.as_concrete_TypeRef(),
+                token.as_concrete_TypeRef(),
+                did_connect,
+                tx,
+            )
+        }
+
+        async { rx.await.unwrap().context("error connecting to room") }
+    }
+
+    pub fn publish_video_track(&self, track: &LocalVideoTrack) -> impl Future<Output = Result<()>> {
+        let (did_publish, tx, rx) = Self::build_done_callback();
+        unsafe {
+            LKRoomPublishVideoTrack(self.native_room, track.0, did_publish, tx);
+        }
+        async { rx.await.unwrap().context("error publishing video track") }
+    }
+
+    pub fn remote_video_tracks(&self) -> mpsc::UnboundedReceiver<Arc<RemoteVideoTrack>> {
+        let (tx, rx) = mpsc::unbounded();
+        self.remote_video_track_subscribers.lock().push(tx);
+        rx
+    }
+
+    fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) {
+        let track = Arc::new(track);
+        self.remote_video_track_subscribers
+            .lock()
+            .retain(|tx| tx.unbounded_send(track.clone()).is_ok());
+    }
+
+    fn build_done_callback() -> (
+        extern "C" fn(*mut c_void, CFStringRef),
+        *mut c_void,
+        oneshot::Receiver<Result<()>>,
+    ) {
+        let (tx, rx) = oneshot::channel();
+        extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) {
+            let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender<Result<()>>) };
+            if error.is_null() {
+                let _ = tx.send(Ok(()));
+            } else {
+                let error = unsafe { CFString::wrap_under_get_rule(error).to_string() };
+                let _ = tx.send(Err(anyhow!(error)));
+            }
+        }
+        (
+            done_callback,
+            Box::into_raw(Box::new(tx)) as *mut c_void,
+            rx,
+        )
+    }
+}
+
+impl Drop for Room {
+    fn drop(&mut self) {
+        unsafe { LKRelease(self.native_room) }
+    }
+}
+
+struct RoomDelegate {
+    native_delegate: *const c_void,
+    weak_room: *const Room,
+}
+
+impl RoomDelegate {
+    fn new(weak_room: Weak<Room>) -> Self {
+        let weak_room = Weak::into_raw(weak_room);
+        let native_delegate = unsafe {
+            LKRoomDelegateCreate(
+                weak_room as *mut c_void,
+                Self::on_did_subscribe_to_remote_video_track,
+            )
+        };
+        Self {
+            native_delegate,
+            weak_room,
+        }
+    }
+
+    extern "C" fn on_did_subscribe_to_remote_video_track(room: *mut c_void, track: *const c_void) {
+        let room = unsafe { Weak::from_raw(room as *mut Room) };
+        let track = RemoteVideoTrack(track);
+        if let Some(room) = room.upgrade() {
+            room.did_subscribe_to_remote_video_track(track);
+        }
+        let _ = Weak::into_raw(room);
+    }
+}
+
+impl Drop for RoomDelegate {
+    fn drop(&mut self) {
+        unsafe {
+            LKRelease(self.native_delegate);
+            let _ = Weak::from_raw(self.weak_room);
+        }
+    }
+}
+
+pub struct LocalVideoTrack(*const c_void);
+
+impl LocalVideoTrack {
+    pub fn screen_share_for_display(display: &MacOSDisplay) -> Self {
+        Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) })
+    }
+}
+
+impl Drop for LocalVideoTrack {
+    fn drop(&mut self) {
+        unsafe { LKRelease(self.0) }
+    }
+}
+
+pub struct RemoteVideoTrack(*const c_void);
+
+impl RemoteVideoTrack {
+    pub fn add_renderer<F>(&self, callback: F)
+    where
+        F: 'static + FnMut(CVImageBuffer),
+    {
+        extern "C" fn on_frame<F>(callback_data: *mut c_void, frame: CVImageBufferRef)
+        where
+            F: FnMut(CVImageBuffer),
+        {
+            unsafe {
+                let buffer = CVImageBuffer::wrap_under_get_rule(frame);
+                let callback = &mut *(callback_data as *mut F);
+                callback(buffer);
+            }
+        }
+
+        extern "C" fn on_drop<F>(callback_data: *mut c_void) {
+            unsafe {
+                let _ = Box::from_raw(callback_data as *mut F);
+            }
+        }
+
+        let callback_data = Box::into_raw(Box::new(callback));
+        unsafe {
+            let renderer =
+                LKVideoRendererCreate(callback_data as *mut c_void, on_frame::<F>, on_drop::<F>);
+            LKVideoTrackAddRenderer(self.0, renderer);
+        }
+    }
+}
+
+impl Drop for RemoteVideoTrack {
+    fn drop(&mut self) {
+        unsafe { LKRelease(self.0) }
+    }
+}
+
+pub struct MacOSDisplay(*const c_void);
+
+impl Drop for MacOSDisplay {
+    fn drop(&mut self) {
+        unsafe { LKRelease(self.0) }
+    }
+}
+
+pub fn display_sources() -> impl Future<Output = Result<Vec<MacOSDisplay>>> {
+    extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) {
+        unsafe {
+            let tx = Box::from_raw(tx as *mut oneshot::Sender<Result<Vec<MacOSDisplay>>>);
+
+            if sources.is_null() {
+                let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error))));
+            } else {
+                let sources = CFArray::wrap_under_get_rule(sources);
+                let sources_vec = sources.iter().map(|source| MacOSDisplay(*source)).collect();
+                let _ = tx.send(Ok(sources_vec));
+            }
+        }
+    }
+
+    let (tx, rx) = oneshot::channel();
+
+    unsafe {
+        LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback);
+    }
+
+    async move { rx.await.unwrap() }
+}