diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 853011d8e2..0814350712 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,7 +209,6 @@ jobs: cargo check -p workspace cargo build -p remote_server cargo check -p gpui --examples - script/check-rust-livekit-macos # Since the macOS runners are stateful, so we need to remove the config file to prevent potential bug. - name: Clean CI config file diff --git a/Cargo.lock b/Cargo.lock index f001aaa2d5..71c50e148e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1881,7 +1881,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "lazy_static", "lazycell", "log", @@ -1904,7 +1904,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.10.5", + "itertools 0.12.1", "log", "prettyplease", "proc-macro2", @@ -1993,7 +1993,7 @@ dependencies = [ "ash-window", "bitflags 2.9.0", "bytemuck", - "codespan-reporting", + "codespan-reporting 0.11.1", "glow", "gpu-alloc", "gpu-alloc-ash", @@ -2279,12 +2279,11 @@ dependencies = [ [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] @@ -2299,10 +2298,10 @@ dependencies = [ "fs", "futures 0.3.31", "gpui", + "gpui_tokio", "http_client", "language", "livekit_client", - "livekit_client_macos", "log", "postage", "project", @@ -2438,8 +2437,7 @@ dependencies = [ [[package]] name = "cargo_metadata" version = "0.19.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5eb614ed4c27c5d706420e4320fbe3216ab31fa1c33cd8246ac36dae4479ba" +source = "git+https://github.com/zed-industries/cargo_metadata?rev=ce8171bad673923d61a77b6761d0dc4aff63398a#ce8171bad673923d61a77b6761d0dc4aff63398a" dependencies = [ "camino", "cargo-platform", @@ -2565,6 +2563,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "cgl" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ced0551234e87afee12411d535648dd89d2e7f34c78b753395567aff3d447ff" +dependencies = [ + "libc", +] + [[package]] name = "channel" version = "0.1.0" @@ -2721,7 +2728,7 @@ dependencies = [ "anyhow", "clap", "collections", - "core-foundation 0.9.4", + "core-foundation 0.10.0", "core-services", "exec", "fork", @@ -2874,6 +2881,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "codespan-reporting" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "collab" version = "0.44.0" @@ -2921,6 +2939,7 @@ dependencies = [ "git_ui", "google_ai", "gpui", + "gpui_tokio", "hex", "http_client", "hyper 0.14.32", @@ -2930,7 +2949,6 @@ dependencies = [ "language_model", "livekit_api", "livekit_client", - "livekit_client_macos", "log", "lsp", "menu", @@ -3354,6 +3372,19 @@ dependencies = [ "libc", ] +[[package]] +name = "core-graphics2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4583956b9806b69f73fcb23aee05eb3620efc282972f08f6a6db7504f8334d" +dependencies = [ + "bitflags 2.9.0", + "block", + "cfg-if", + "core-foundation 0.10.0", + "libc", +] + [[package]] name = "core-services" version = "0.2.1" @@ -3365,16 +3396,30 @@ dependencies = [ [[package]] name = "core-text" -version = "20.1.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d2790b5c08465d49f8dc05c8bcae9fea467855947db39b0f8145c091aaced5" +checksum = "a593227b66cbd4007b2a050dfdd9e1d1318311409c8d600dc82ba1b15ca9c130" dependencies = [ - "core-foundation 0.9.4", - "core-graphics 0.23.2", + "core-foundation 0.10.0", + "core-graphics 0.24.0", "foreign-types 0.5.0", "libc", ] +[[package]] +name = "core-video" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45e71d5be22206bed53c3c3cb99315fc4c3d31b8963808c6bc4538168c4f8ef" +dependencies = [ + "block", + "core-foundation 0.10.0", + "core-graphics2", + "io-surface", + "libc", + "metal", +] + [[package]] name = "core_maths" version = "0.1.1" @@ -3793,9 +3838,9 @@ checksum = "96a6ac251f4a2aca6b3f91340350eab87ae57c3f127ffeb585e92bd336717991" [[package]] name = "cxx" -version = "1.0.134" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5a32d755fe20281b46118ee4b507233311fb7a48a0cfd42f554b93640521a2f" +checksum = "fdb3e596b379180315d2f934231e233a2fc745041f88231807774093d8de45f2" dependencies = [ "cc", "cxxbridge-cmd", @@ -3807,12 +3852,12 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.134" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11645536ada5d1c8804312cbffc9ab950f2216154de431de930da47ca6955199" +checksum = "3743fae7f47620cd34ec23bab819db9ee52da93166a058f87ab0ad99d777dc9b" dependencies = [ "cc", - "codespan-reporting", + "codespan-reporting 0.12.0", "proc-macro2", "quote", "scratch", @@ -3821,12 +3866,12 @@ dependencies = [ [[package]] name = "cxxbridge-cmd" -version = "1.0.134" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcc9c78e3c7289665aab921a2b394eaffe8bdb369aa18d81ffc0f534fd49385" +checksum = "aaea0273c049b126a3918df88a1670c9c0168e0738df9370a988ff69070d4fff" dependencies = [ "clap", - "codespan-reporting", + "codespan-reporting 0.12.0", "proc-macro2", "quote", "syn 2.0.100", @@ -3834,15 +3879,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.134" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a22a87bd9e78d7204d793261470a4c9d585154fddd251828d8aefbb5f74c3bf" +checksum = "020a9a3d6b792aab7f30f6e323893ad7f45052e572cde5d014c47fe67c89495f" [[package]] name = "cxxbridge-macro" -version = "1.0.134" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dfdb020ff8787c5daf6e0dca743005cc8782868faeadfbabb8824ede5cb1c72" +checksum = "ee54cd01f94db0328c4c73036d38bd8c3bb88927e953d05ffefe743edbf4eb68" dependencies = [ "proc-macro2", "quote", @@ -5073,12 +5118,12 @@ checksum = "f81ec6369c545a7d40e4589b5597581fa1c441fe1cce96dd1de43159910a36a2" [[package]] name = "font-kit" version = "0.14.1" -source = "git+https://github.com/zed-industries/font-kit?rev=40391b7#40391b7c0041d8a8572af2afa3de32ae088f0120" +source = "git+https://github.com/zed-industries/font-kit?rev=5474cfad4b719a72ec8ed2cb7327b2b01fd10568#5474cfad4b719a72ec8ed2cb7327b2b01fd10568" dependencies = [ "bitflags 2.9.0", "byteorder", - "core-foundation 0.9.4", - "core-graphics 0.23.2", + "core-foundation 0.10.0", + "core-graphics 0.24.0", "core-text", "dirs 5.0.1", "dwrote", @@ -5276,7 +5321,7 @@ name = "fsevent" version = "0.1.0" dependencies = [ "bitflags 2.9.0", - "core-foundation 0.9.4", + "core-foundation 0.10.0", "fsevent-sys 3.1.0", "parking_lot", "tempfile", @@ -5813,10 +5858,11 @@ dependencies = [ "cbindgen 0.28.0", "cocoa 0.26.0", "collections", - "core-foundation 0.9.4", + "core-foundation 0.10.0", "core-foundation-sys", - "core-graphics 0.23.2", + "core-graphics 0.24.0", "core-text", + "core-video", "cosmic-text", "ctor", "derive_more", @@ -6918,6 +6964,19 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" +[[package]] +name = "io-surface" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8283575d5f0b2e7447ec0840363879d71c0fa325d4c699d5b45208ea4a51f45e" +dependencies = [ + "cgl", + "core-foundation 0.10.0", + "core-foundation-sys", + "leaky-cow", + "libc", +] + [[package]] name = "iovec" version = "0.1.4" @@ -7498,6 +7557,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "leak" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd100e01f1154f2908dfa7d02219aeab25d0b9c7fa955164192e3245255a0c73" + +[[package]] +name = "leaky-cow" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40a8225d44241fd324a8af2806ba635fc7c8a7e9a7de4d5cf3ef54e71f5926fc" +dependencies = [ + "leak", +] + [[package]] name = "leb128" version = "0.2.5" @@ -7598,8 +7672,8 @@ dependencies = [ [[package]] name = "libwebrtc" -version = "0.3.7" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.3.10" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ "cxx", "jni", @@ -7633,9 +7707,9 @@ dependencies = [ [[package]] name = "link-cplusplus" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +checksum = "4a6f6da007f968f9def0d65a05b187e2960183de70c160204ecfccf0ee330212" dependencies = [ "cc", ] @@ -7683,12 +7757,13 @@ checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "livekit" -version = "0.7.0" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.7.8" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ "chrono", "futures-util", "lazy_static", + "libloading", "libwebrtc", "livekit-api", "livekit-protocol", @@ -7705,10 +7780,10 @@ dependencies = [ [[package]] name = "livekit-api" -version = "0.4.1" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.4.2" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ - "async-tungstenite", + "base64 0.21.7", "futures-util", "http 0.2.12", "jsonwebtoken", @@ -7716,7 +7791,9 @@ dependencies = [ "livekit-runtime", "log", "parking_lot", + "pbjson-types", "prost 0.12.6", + "rand 0.9.0", "reqwest 0.11.27", "scopeguard", "serde", @@ -7724,14 +7801,14 @@ dependencies = [ "sha2", "thiserror 1.0.69", "tokio", - "tokio-tungstenite 0.20.1", + "tokio-tungstenite 0.26.2", "url", ] [[package]] name = "livekit-protocol" -version = "0.3.6" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.3.9" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ "futures-util", "livekit-runtime", @@ -7747,13 +7824,11 @@ dependencies = [ [[package]] name = "livekit-runtime" -version = "0.3.1" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.4.0" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ - "async-io", - "async-std", - "async-task", - "futures 0.3.31", + "tokio", + "tokio-stream", ] [[package]] @@ -7778,19 +7853,21 @@ dependencies = [ "anyhow", "async-trait", "collections", - "core-foundation 0.9.4", + "core-foundation 0.10.0", + "core-video", "coreaudio-rs 0.12.1", "cpal", "futures 0.3.31", "gpui", - "http 0.2.12", - "http_client", + "gpui_tokio", + "http_client_tls", "image", + "libwebrtc", "livekit", "livekit_api", "log", - "media", "nanoid", + "objc", "parking_lot", "postage", "serde", @@ -7798,32 +7875,10 @@ dependencies = [ "sha2", "simplelog", "smallvec", + "tokio-tungstenite 0.26.2", "util", ] -[[package]] -name = "livekit_client_macos" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-broadcast", - "async-trait", - "collections", - "core-foundation 0.9.4", - "futures 0.3.31", - "gpui", - "livekit_api", - "log", - "media", - "nanoid", - "parking_lot", - "postage", - "serde", - "serde_json", - "sha2", - "simplelog", -] - [[package]] name = "lmdb-master-sys" version = "0.2.4" @@ -8165,7 +8220,8 @@ version = "0.1.0" dependencies = [ "anyhow", "bindgen 0.70.1", - "core-foundation 0.9.4", + "core-foundation 0.10.0", + "core-video", "ctor", "foreign-types 0.5.0", "metal", @@ -8215,9 +8271,9 @@ dependencies = [ [[package]] name = "metal" -version = "0.31.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f569fb946490b5743ad69813cb19629130ce9374034abe31614a36402d18f99e" +checksum = "7ecfd3296f8c56b7c1f6fbac3c71cefa9d78ce009850c45000015f206dc7fa21" dependencies = [ "bitflags 2.9.0", "block", @@ -8370,12 +8426,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "multimap" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" - [[package]] name = "naga" version = "23.1.0" @@ -8386,7 +8436,7 @@ dependencies = [ "bit-set 0.8.0", "bitflags 2.9.0", "cfg_aliases 0.1.1", - "codespan-reporting", + "codespan-reporting 0.11.1", "hexf-parse", "indexmap", "log", @@ -10694,7 +10744,7 @@ dependencies = [ "itertools 0.10.5", "lazy_static", "log", - "multimap 0.8.3", + "multimap", "petgraph", "prost 0.9.0", "prost-types 0.9.0", @@ -10713,7 +10763,7 @@ dependencies = [ "heck 0.4.1", "itertools 0.10.5", "log", - "multimap 0.10.0", + "multimap", "once_cell", "petgraph", "prettyplease", @@ -12132,9 +12182,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scratch" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" +checksum = "9f6280af86e5f559536da57a45ebc84948833b3bee313a7dd25232e09c878a52" [[package]] name = "scrypt" @@ -14093,7 +14143,7 @@ dependencies = [ name = "time_format" version = "0.1.0" dependencies = [ - "core-foundation 0.9.4", + "core-foundation 0.10.0", "core-foundation-sys", "sys-locale", "time", @@ -14317,10 +14367,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", "tokio", - "tokio-rustls 0.24.1", "tungstenite 0.20.1", ] @@ -14336,6 +14383,21 @@ dependencies = [ "tungstenite 0.21.0", ] +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "rustls 0.23.25", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.1", + "tungstenite 0.26.2", +] + [[package]] name = "tokio-util" version = "0.7.13" @@ -14840,7 +14902,6 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls 0.21.12", "sha1", "thiserror 1.0.69", "url", @@ -14884,6 +14945,25 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes 1.10.1", + "data-encoding", + "http 1.2.0", + "httparse", + "log", + "rand 0.9.0", + "rustls 0.23.25", + "rustls-pki-types", + "sha1", + "thiserror 2.0.12", + "utf-8", +] + [[package]] name = "typeid" version = "1.0.2" @@ -16020,8 +16100,8 @@ dependencies = [ [[package]] name = "webrtc-sys" -version = "0.3.5" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.3.7" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ "cc", "cxx", @@ -16033,8 +16113,8 @@ dependencies = [ [[package]] name = "webrtc-sys-build" -version = "0.3.5" -source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=811ceae29fabee455f110c56cd66b3f49a7e5003#811ceae29fabee455f110c56cd66b3f49a7e5003" +version = "0.3.6" +source = "git+https://github.com/zed-industries/livekit-rust-sdks?rev=102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8#102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8" dependencies = [ "fs2", "regex", diff --git a/Cargo.toml b/Cargo.toml index 6b01ae7988..12a03a37fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,7 +87,6 @@ members = [ "crates/languages", "crates/livekit_api", "crates/livekit_client", - "crates/livekit_client_macos", "crates/lmstudio", "crates/lsp", "crates/markdown", @@ -292,7 +291,6 @@ language_tools = { path = "crates/language_tools" } languages = { path = "crates/languages" } livekit_api = { path = "crates/livekit_api" } livekit_client = { path = "crates/livekit_client" } -livekit_client_macos = { path = "crates/livekit_client_macos" } lmstudio = { path = "crates/lmstudio" } lsp = { path = "crates/lsp" } markdown = { path = "crates/markdown" } @@ -413,15 +411,16 @@ blade-util = { git = "https://github.com/kvark/blade", rev = "b16f5c7bd873c7126f naga = { version = "23.1.0", features = ["wgsl-in"] } blake3 = "1.5.3" bytes = "1.0" -cargo_metadata = "0.19" +cargo_metadata = { git = "https://github.com/zed-industries/cargo_metadata", rev = "ce8171bad673923d61a77b6761d0dc4aff63398a"} cargo_toml = "0.21" chrono = { version = "0.4", features = ["serde"] } circular-buffer = "1.0" clap = { version = "4.4", features = ["derive"] } cocoa = "0.26" cocoa-foundation = "0.2.0" +core-video = { version = "0.4.3", features = ["metal"] } convert_case = "0.8.0" -core-foundation = "0.9.3" +core-foundation = "0.10.0" core-foundation-sys = "0.8.6" ctor = "0.4.0" dashmap = "6.0" @@ -459,11 +458,6 @@ libc = "0.2" libsqlite3-sys = { version = "0.30.1", features = ["bundled"] } linkify = "0.10.0" linkme = "0.3.31" -livekit = { git = "https://github.com/zed-industries/livekit-rust-sdks", rev = "811ceae29fabee455f110c56cd66b3f49a7e5003", features = [ - "dispatcher", - "services-dispatcher", - "rustls-tls-native-roots", -], default-features = false } log = { version = "0.4.16", features = ["kv_unstable_serde", "serde"] } markup5ever_rcdom = "0.3.0" mlua = { version = "0.10", features = ["lua54", "vendored", "async", "send"] } @@ -552,6 +546,7 @@ time = { version = "0.3", features = [ tiny_http = "0.8" toml = "0.8" tokio = { version = "1" } +tokio-tungstenite = { version = "0.26", features = ["__rustls-tls"]} tower-http = "0.4.4" tree-sitter = { version = "0.25.3", features = ["wasm"] } tree-sitter-bash = "0.23" @@ -597,7 +592,7 @@ which = "6.0.0" wit-component = "0.221" zed_llm_client = "0.4" zstd = "0.11" -metal = "0.31" +metal = "0.29" [workspace.dependencies.async-stripe] git = "https://github.com/zed-industries/async-stripe" diff --git a/change-sophie.wav b/change-sophie.wav new file mode 100644 index 0000000000..1f0a0bebc8 Binary files /dev/null and b/change-sophie.wav differ diff --git a/crates/assistant_eval/build.rs b/crates/assistant_eval/build.rs index 6268b66f9c..5b955c222a 100644 --- a/crates/assistant_eval/build.rs +++ b/crates/assistant_eval/build.rs @@ -6,15 +6,6 @@ fn main() { if cfg!(target_os = "macos") { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); - println!("cargo:rerun-if-env-changed=ZED_BUNDLE"); - if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { - // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); - } else { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } - // Weakly link ReplayKit to ensure Zed can be used on macOS 10.15+. println!("cargo:rustc-link-arg=-Wl,-weak_framework,ReplayKit"); diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml index 9225472a55..19eda091ee 100644 --- a/crates/call/Cargo.toml +++ b/crates/call/Cargo.toml @@ -18,7 +18,6 @@ test-support = [ "collections/test-support", "gpui/test-support", "livekit_client/test-support", - "livekit_client_macos/test-support", "project/test-support", "util/test-support" ] @@ -41,11 +40,7 @@ serde_derive.workspace = true settings.workspace = true telemetry.workspace = true util.workspace = true - -[target.'cfg(target_os = "macos")'.dependencies] -livekit_client_macos.workspace = true - -[target.'cfg(not(target_os = "macos"))'.dependencies] +gpui_tokio.workspace = true livekit_client.workspace = true [dev-dependencies] @@ -57,9 +52,4 @@ language = { workspace = true, features = ["test-support"] } project = { workspace = true, features = ["test-support"] } util = { workspace = true, features = ["test-support"] } http_client = { workspace = true, features = ["test-support"] } - -[target.'cfg(target_os = "macos")'.dev-dependencies] -livekit_client_macos = { workspace = true, features = ["test-support"] } - -[target.'cfg(not(target_os = "macos"))'.dev-dependencies] livekit_client = { workspace = true, features = ["test-support"] } diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs index 5e212d35b7..5218c3c6b8 100644 --- a/crates/call/src/call.rs +++ b/crates/call/src/call.rs @@ -1,13 +1,5 @@ pub mod call_settings; -#[cfg(target_os = "macos")] -mod macos; +mod call_impl; -#[cfg(target_os = "macos")] -pub use macos::*; - -#[cfg(not(target_os = "macos"))] -mod cross_platform; - -#[cfg(not(target_os = "macos"))] -pub use cross_platform::*; +pub use call_impl::*; diff --git a/crates/call/src/cross_platform/mod.rs b/crates/call/src/call_impl/mod.rs similarity index 98% rename from crates/call/src/cross_platform/mod.rs rename to crates/call/src/call_impl/mod.rs index 2704daf92b..0db63b203c 100644 --- a/crates/call/src/cross_platform/mod.rs +++ b/crates/call/src/call_impl/mod.rs @@ -17,9 +17,7 @@ use room::Event; use settings::Settings; use std::sync::Arc; -pub use livekit_client::{ - track::RemoteVideoTrack, RemoteVideoTrackView, RemoteVideoTrackViewEvent, -}; +pub use livekit_client::{RemoteVideoTrack, RemoteVideoTrackView, RemoteVideoTrackViewEvent}; pub use participant::ParticipantLocation; pub use room::Room; @@ -28,10 +26,6 @@ struct GlobalActiveCall(Entity); impl Global for GlobalActiveCall {} pub fn init(client: Arc, user_store: Entity, cx: &mut App) { - livekit_client::init( - cx.background_executor().dispatcher.clone(), - cx.http_client(), - ); CallSettings::register(cx); let active_call = cx.new(|cx| ActiveCall::new(client, user_store, cx)); diff --git a/crates/call/src/macos/participant.rs b/crates/call/src/call_impl/participant.rs similarity index 83% rename from crates/call/src/macos/participant.rs rename to crates/call/src/call_impl/participant.rs index 1f1a63a10e..e887aeb329 100644 --- a/crates/call/src/macos/participant.rs +++ b/crates/call/src/call_impl/participant.rs @@ -1,13 +1,14 @@ use anyhow::{anyhow, Result}; -use client::ParticipantIndex; -use client::{proto, User}; +use client::{proto, ParticipantIndex, User}; use collections::HashMap; use gpui::WeakEntity; -pub use livekit_client_macos::Frame; -pub use livekit_client_macos::{RemoteAudioTrack, RemoteVideoTrack}; +use livekit_client::AudioStream; use project::Project; use std::sync::Arc; +pub use livekit_client::TrackSid; +pub use livekit_client::{RemoteAudioTrack, RemoteVideoTrack}; + #[derive(Copy, Clone, Debug, Eq, PartialEq)] pub enum ParticipantLocation { SharedProject { project_id: u64 }, @@ -48,7 +49,6 @@ impl LocalParticipant { } } -#[derive(Clone, Debug)] pub struct RemoteParticipant { pub user: Arc, pub peer_id: proto::PeerId, @@ -58,13 +58,13 @@ pub struct RemoteParticipant { pub participant_index: ParticipantIndex, pub muted: bool, pub speaking: bool, - pub video_tracks: HashMap>, - pub audio_tracks: HashMap>, + pub video_tracks: HashMap, + pub audio_tracks: HashMap, } impl RemoteParticipant { pub fn has_video_tracks(&self) -> bool { - !self.video_tracks.is_empty() + return !self.video_tracks.is_empty(); } pub fn can_write(&self) -> bool { diff --git a/crates/call/src/cross_platform/room.rs b/crates/call/src/call_impl/room.rs similarity index 90% rename from crates/call/src/cross_platform/room.rs rename to crates/call/src/call_impl/room.rs index ebed7439cc..44294f8360 100644 --- a/crates/call/src/cross_platform/room.rs +++ b/crates/call/src/call_impl/room.rs @@ -1,5 +1,3 @@ -#![cfg_attr(all(target_os = "windows", target_env = "gnu"), allow(unused))] - use crate::{ call_settings::CallSettings, participant::{LocalParticipant, ParticipantLocation, RemoteParticipant}, @@ -14,20 +12,10 @@ use collections::{BTreeMap, HashMap, HashSet}; use fs::Fs; use futures::{FutureExt, StreamExt}; use gpui::{App, AppContext as _, AsyncApp, Context, Entity, EventEmitter, Task, WeakEntity}; +use gpui_tokio::Tokio; use language::LanguageRegistry; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -use livekit::{ - capture_local_audio_track, capture_local_video_track, - id::ParticipantIdentity, - options::{TrackPublishOptions, VideoCodec}, - play_remote_audio_track, - publication::LocalTrackPublication, - track::{TrackKind, TrackSource}, - RoomEvent, RoomOptions, -}; -#[cfg(all(target_os = "windows", target_env = "gnu"))] -use livekit::{publication::LocalTrackPublication, RoomEvent}; -use livekit_client as livekit; +use livekit::{LocalTrackPublication, ParticipantIdentity, RoomEvent}; +use livekit_client::{self as livekit, TrackSid}; use postage::{sink::Sink, stream::Stream, watch}; use project::Project; use settings::Settings as _; @@ -47,6 +35,9 @@ pub enum Event { RemoteVideoTracksChanged { participant_id: proto::PeerId, }, + RemoteVideoTrackUnsubscribed { + sid: TrackSid, + }, RemoteAudioTracksChanged { participant_id: proto::PeerId, }, @@ -104,11 +95,7 @@ impl Room { !self.shared_projects.is_empty() } - #[cfg(all( - any(test, feature = "test-support"), - not(all(target_os = "windows", target_env = "gnu")) - ))] - pub fn is_connected(&self) -> bool { + pub fn is_connected(&self, _: &App) -> bool { if let Some(live_kit) = self.live_kit.as_ref() { live_kit.room.connection_state() == livekit::ConnectionState::Connected } else { @@ -477,13 +464,15 @@ impl Room { id: worktree.id().to_proto(), scan_id: worktree.completed_scan_id() as u64, }); - for repository in worktree.repositories().iter() { - repositories.push(proto::RejoinRepository { - id: repository.work_directory_id().to_proto(), - scan_id: worktree.completed_scan_id() as u64, - }); - } } + for (entry_id, repository) in project.repositories(cx) { + let repository = repository.read(cx); + repositories.push(proto::RejoinRepository { + id: entry_id.to_proto(), + scan_id: repository.completed_scan_id as u64, + }); + } + rejoined_projects.push(proto::RejoinProject { id: project_id, worktrees, @@ -687,12 +676,6 @@ impl Room { } } - #[cfg(all(target_os = "windows", target_env = "gnu"))] - fn start_room_connection(&self, mut room: proto::Room, cx: &mut Context) -> Task<()> { - Task::ready(()) - } - - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn start_room_connection(&self, mut room: proto::Room, cx: &mut Context) -> Task<()> { // Filter ourselves out from the room's participants. let local_participant_ix = room @@ -845,7 +828,6 @@ impl Room { muted: true, speaking: false, video_tracks: Default::default(), - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] audio_tracks: Default::default(), }, ); @@ -948,7 +930,6 @@ impl Room { ); match event { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::TrackSubscribed { track, participant, @@ -963,18 +944,22 @@ impl Room { ) })?; if self.live_kit.as_ref().map_or(true, |kit| kit.deafened) { - track.rtc_track().set_enabled(false); + if matches!(track, livekit_client::RemoteTrack::Audio(_)) { + track.set_enabled(false, cx); + } } match track { - livekit::track::RemoteTrack::Audio(track) => { + livekit_client::RemoteTrack::Audio(track) => { cx.emit(Event::RemoteAudioTracksChanged { participant_id: participant.peer_id, }); - let stream = play_remote_audio_track(&track, cx.background_executor())?; - participant.audio_tracks.insert(track_id, (track, stream)); - participant.muted = publication.is_muted(); + if let Some(live_kit) = self.live_kit.as_ref() { + let stream = live_kit.room.play_remote_audio_track(&track, cx)?; + participant.audio_tracks.insert(track_id, (track, stream)); + participant.muted = publication.is_muted(); + } } - livekit::track::RemoteTrack::Video(track) => { + livekit_client::RemoteTrack::Video(track) => { cx.emit(Event::RemoteVideoTracksChanged { participant_id: participant.peer_id, }); @@ -983,7 +968,6 @@ impl Room { } } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::TrackUnsubscribed { track, participant, .. } => { @@ -995,23 +979,23 @@ impl Room { ) })?; match track { - livekit::track::RemoteTrack::Audio(track) => { + livekit_client::RemoteTrack::Audio(track) => { participant.audio_tracks.remove(&track.sid()); participant.muted = true; cx.emit(Event::RemoteAudioTracksChanged { participant_id: participant.peer_id, }); } - livekit::track::RemoteTrack::Video(track) => { + livekit_client::RemoteTrack::Video(track) => { participant.video_tracks.remove(&track.sid()); cx.emit(Event::RemoteVideoTracksChanged { participant_id: participant.peer_id, }); + cx.emit(Event::RemoteVideoTrackUnsubscribed { sid: track.sid() }); } } } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::ActiveSpeakersChanged { speakers } => { let mut speaker_ids = speakers .into_iter() @@ -1028,7 +1012,6 @@ impl Room { } } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::TrackMuted { participant, publication, @@ -1053,7 +1036,6 @@ impl Room { } } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::LocalTrackUnpublished { publication, .. } => { log::info!("unpublished track {}", publication.sid()); if let Some(room) = &mut self.live_kit { @@ -1076,12 +1058,10 @@ impl Room { } } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::LocalTrackPublished { publication, .. } => { log::info!("published track {:?}", publication.sid()); } - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] RoomEvent::Disconnected { reason } => { log::info!("disconnected from room: {reason:?}"); self.leave(cx).detach_and_log_err(cx); @@ -1309,13 +1289,6 @@ impl Room { pub fn can_use_microphone(&self) -> bool { use proto::ChannelRole::*; - #[cfg(not(any(test, feature = "test-support")))] - { - if cfg!(all(target_os = "windows", target_env = "gnu")) { - return false; - } - } - match self.local_participant.role { Admin | Member | Talker => true, Guest | Banned => false, @@ -1330,40 +1303,23 @@ impl Room { } } - #[cfg(all(target_os = "windows", target_env = "gnu"))] - pub fn share_microphone(&mut self, cx: &mut Context) -> Task> { - Task::ready(Err(anyhow!("MinGW is not supported yet"))) - } - - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] #[track_caller] pub fn share_microphone(&mut self, cx: &mut Context) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); } - let (participant, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { + let (room, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { let publish_id = post_inc(&mut live_kit.next_publish_id); live_kit.microphone_track = LocalTrack::Pending { publish_id }; cx.notify(); - (live_kit.room.local_participant(), publish_id) + (live_kit.room.clone(), publish_id) } else { return Task::ready(Err(anyhow!("live-kit was not initialized"))); }; cx.spawn(async move |this, cx| { - let (track, stream) = capture_local_audio_track(cx.background_executor())?.await; - - let publication = participant - .publish_track( - livekit::track::LocalTrack::Audio(track), - TrackPublishOptions { - source: TrackSource::Microphone, - ..Default::default() - }, - ) - .await - .map_err(|error| anyhow!("failed to publish track: {error}")); + let publication = room.publish_local_microphone_track(cx).await; this.update(cx, |this, cx| { let live_kit = this .live_kit @@ -1380,15 +1336,15 @@ impl Room { }; match publication { - Ok(publication) => { + Ok((publication, stream)) => { if canceled { - cx.background_spawn(async move { - participant.unpublish_track(&publication.sid()).await + cx.spawn(async move |_, cx| { + room.unpublish_local_track(publication.sid(), cx).await }) .detach_and_log_err(cx) } else { if live_kit.muted_by_user || live_kit.deafened { - publication.mute(); + publication.mute(cx); } live_kit.microphone_track = LocalTrack::Published { track_publication: publication, @@ -1412,12 +1368,6 @@ impl Room { }) } - #[cfg(all(target_os = "windows", target_env = "gnu"))] - pub fn share_screen(&mut self, cx: &mut Context) -> Task> { - Task::ready(Err(anyhow!("MinGW is not supported yet"))) - } - - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] pub fn share_screen(&mut self, cx: &mut Context) -> Task> { if self.status.is_offline() { return Task::ready(Err(anyhow!("room is offline"))); @@ -1441,19 +1391,7 @@ impl Room { let sources = sources.await??; let source = sources.first().ok_or_else(|| anyhow!("no display found"))?; - let (track, stream) = capture_local_video_track(&**source).await?; - - let publication = participant - .publish_track( - livekit::track::LocalTrack::Video(track), - TrackPublishOptions { - source: TrackSource::Screenshare, - video_codec: VideoCodec::H264, - ..Default::default() - }, - ) - .await - .map_err(|error| anyhow!("error publishing screen track {error:?}")); + let publication = participant.publish_screenshare_track(&**source, cx).await; this.update(cx, |this, cx| { let live_kit = this @@ -1471,10 +1409,10 @@ impl Room { }; match publication { - Ok(publication) => { + Ok((publication, stream)) => { if canceled { - cx.background_spawn(async move { - participant.unpublish_track(&publication.sid()).await + cx.spawn(async move |_, cx| { + participant.unpublish_track(publication.sid(), cx).await }) .detach() } else { @@ -1564,14 +1502,11 @@ impl Room { LocalTrack::Published { track_publication, .. } => { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] { let local_participant = live_kit.room.local_participant(); let sid = track_publication.sid(); - cx.background_spawn( - async move { local_participant.unpublish_track(&sid).await }, - ) - .detach_and_log_err(cx); + cx.spawn(async move |_, cx| local_participant.unpublish_track(sid, cx).await) + .detach_and_log_err(cx); cx.notify(); } @@ -1582,14 +1517,13 @@ impl Room { } fn set_deafened(&mut self, deafened: bool, cx: &mut Context) -> Option<()> { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] { let live_kit = self.live_kit.as_mut()?; cx.notify(); for (_, participant) in live_kit.room.remote_participants() { for (_, publication) in participant.track_publications() { - if publication.kind() == TrackKind::Audio { - publication.set_enabled(!deafened); + if publication.is_audio() { + publication.set_enabled(!deafened, cx); } } } @@ -1620,14 +1554,13 @@ impl Room { LocalTrack::Published { track_publication, .. } => { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - { - if should_mute { - track_publication.mute() - } else { - track_publication.unmute() - } + let guard = Tokio::handle(cx); + if should_mute { + track_publication.mute(cx) + } else { + track_publication.unmute(cx) } + drop(guard); None } @@ -1635,30 +1568,19 @@ impl Room { } } -#[cfg(all(target_os = "windows", target_env = "gnu"))] -fn spawn_room_connection( - livekit_connection_info: Option, - cx: &mut Context<'_, Room>, -) { -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn spawn_room_connection( livekit_connection_info: Option, cx: &mut Context<'_, Room>, ) { if let Some(connection_info) = livekit_connection_info { cx.spawn(async move |this, cx| { - let (room, mut events) = livekit::Room::connect( - &connection_info.server_url, - &connection_info.token, - RoomOptions::default(), - ) - .await?; + let (room, mut events) = + livekit::Room::connect(connection_info.server_url, connection_info.token, cx) + .await?; this.update(cx, |this, cx| { let _handle_updates = cx.spawn(async move |this, cx| { - while let Some(event) = events.recv().await { + while let Some(event) = events.next().await { if this .update(cx, |this, cx| { this.livekit_room_updated(event, cx).warn_on_err(); @@ -1707,10 +1629,6 @@ struct LiveKitRoom { } impl LiveKitRoom { - #[cfg(all(target_os = "windows", target_env = "gnu"))] - fn stop_publishing(&mut self, _cx: &mut Context) {} - - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn stop_publishing(&mut self, cx: &mut Context) { let mut tracks_to_unpublish = Vec::new(); if let LocalTrack::Published { @@ -1730,9 +1648,9 @@ impl LiveKitRoom { } let participant = self.room.local_participant(); - cx.background_spawn(async move { + cx.spawn(async move |_, cx| { for sid in tracks_to_unpublish { - participant.unpublish_track(&sid).await.log_err(); + participant.unpublish_track(sid, cx).await.log_err(); } }) .detach(); diff --git a/crates/call/src/cross_platform/participant.rs b/crates/call/src/cross_platform/participant.rs deleted file mode 100644 index de001ea9ca..0000000000 --- a/crates/call/src/cross_platform/participant.rs +++ /dev/null @@ -1,84 +0,0 @@ -#![cfg_attr(all(target_os = "windows", target_env = "gnu"), allow(unused))] - -use anyhow::{anyhow, Result}; -use client::{proto, ParticipantIndex, User}; -use collections::HashMap; -use gpui::WeakEntity; -use livekit_client::AudioStream; -use project::Project; -use std::sync::Arc; - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub use livekit_client::id::TrackSid; -pub use livekit_client::track::{RemoteAudioTrack, RemoteVideoTrack}; - -#[derive(Copy, Clone, Debug, Eq, PartialEq)] -pub enum ParticipantLocation { - SharedProject { project_id: u64 }, - UnsharedProject, - External, -} - -impl ParticipantLocation { - pub fn from_proto(location: Option) -> Result { - match location.and_then(|l| l.variant) { - Some(proto::participant_location::Variant::SharedProject(project)) => { - Ok(Self::SharedProject { - project_id: project.id, - }) - } - Some(proto::participant_location::Variant::UnsharedProject(_)) => { - Ok(Self::UnsharedProject) - } - Some(proto::participant_location::Variant::External(_)) => Ok(Self::External), - None => Err(anyhow!("participant location was not provided")), - } - } -} - -#[derive(Clone, Default)] -pub struct LocalParticipant { - pub projects: Vec, - pub active_project: Option>, - pub role: proto::ChannelRole, -} - -impl LocalParticipant { - pub fn can_write(&self) -> bool { - matches!( - self.role, - proto::ChannelRole::Admin | proto::ChannelRole::Member - ) - } -} - -pub struct RemoteParticipant { - pub user: Arc, - pub peer_id: proto::PeerId, - pub role: proto::ChannelRole, - pub projects: Vec, - pub location: ParticipantLocation, - pub participant_index: ParticipantIndex, - pub muted: bool, - pub speaking: bool, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub video_tracks: HashMap, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub audio_tracks: HashMap, -} - -impl RemoteParticipant { - pub fn has_video_tracks(&self) -> bool { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - return !self.video_tracks.is_empty(); - #[cfg(all(target_os = "windows", target_env = "gnu"))] - return false; - } - - pub fn can_write(&self) -> bool { - matches!( - self.role, - proto::ChannelRole::Admin | proto::ChannelRole::Member - ) - } -} diff --git a/crates/call/src/macos/mod.rs b/crates/call/src/macos/mod.rs deleted file mode 100644 index 49ff6f3ef6..0000000000 --- a/crates/call/src/macos/mod.rs +++ /dev/null @@ -1,521 +0,0 @@ -pub mod participant; -pub mod room; - -use crate::call_settings::CallSettings; -use anyhow::{anyhow, Result}; -use audio::Audio; -use client::{proto, ChannelId, Client, TypedEnvelope, User, UserStore, ZED_ALWAYS_ACTIVE}; -use collections::HashSet; -use futures::{channel::oneshot, future::Shared, Future, FutureExt}; -use gpui::{ - App, AppContext as _, AsyncApp, Context, Entity, EventEmitter, Global, Subscription, Task, - WeakEntity, -}; -use postage::watch; -use project::Project; -use room::Event; -use settings::Settings; -use std::sync::Arc; - -pub use participant::ParticipantLocation; -pub use room::Room; - -struct GlobalActiveCall(Entity); - -impl Global for GlobalActiveCall {} - -pub fn init(client: Arc, user_store: Entity, cx: &mut App) { - CallSettings::register(cx); - - let active_call = cx.new(|cx| ActiveCall::new(client, user_store, cx)); - cx.set_global(GlobalActiveCall(active_call)); -} - -pub struct OneAtATime { - cancel: Option>, -} - -impl OneAtATime { - /// spawn a task in the given context. - /// if another task is spawned before that resolves, or if the OneAtATime itself is dropped, the first task will be cancelled and return Ok(None) - /// otherwise you'll see the result of the task. - fn spawn(&mut self, cx: &mut App, f: F) -> Task>> - where - F: 'static + FnOnce(AsyncApp) -> Fut, - Fut: Future>, - R: 'static, - { - let (tx, rx) = oneshot::channel(); - self.cancel.replace(tx); - cx.spawn(async move |cx| { - futures::select_biased! { - _ = rx.fuse() => Ok(None), - result = f(cx.clone()).fuse() => result.map(Some), - } - }) - } - - fn running(&self) -> bool { - self.cancel - .as_ref() - .is_some_and(|cancel| !cancel.is_canceled()) - } -} - -#[derive(Clone)] -pub struct IncomingCall { - pub room_id: u64, - pub calling_user: Arc, - pub participants: Vec>, - pub initial_project: Option, -} - -/// Singleton global maintaining the user's participation in a room across workspaces. -pub struct ActiveCall { - room: Option<(Entity, Vec)>, - pending_room_creation: Option, Arc>>>>, - location: Option>, - _join_debouncer: OneAtATime, - pending_invites: HashSet, - incoming_call: ( - watch::Sender>, - watch::Receiver>, - ), - client: Arc, - user_store: Entity, - _subscriptions: Vec, -} - -impl EventEmitter for ActiveCall {} - -impl ActiveCall { - fn new(client: Arc, user_store: Entity, cx: &mut Context) -> Self { - Self { - room: None, - pending_room_creation: None, - location: None, - pending_invites: Default::default(), - incoming_call: watch::channel(), - _join_debouncer: OneAtATime { cancel: None }, - _subscriptions: vec![ - client.add_request_handler(cx.weak_entity(), Self::handle_incoming_call), - client.add_message_handler(cx.weak_entity(), Self::handle_call_canceled), - ], - client, - user_store, - } - } - - pub fn channel_id(&self, cx: &App) -> Option { - self.room()?.read(cx).channel_id() - } - - async fn handle_incoming_call( - this: Entity, - envelope: TypedEnvelope, - mut cx: AsyncApp, - ) -> Result { - let user_store = this.update(&mut cx, |this, _| this.user_store.clone())?; - let call = IncomingCall { - room_id: envelope.payload.room_id, - participants: user_store - .update(&mut cx, |user_store, cx| { - user_store.get_users(envelope.payload.participant_user_ids, cx) - })? - .await?, - calling_user: user_store - .update(&mut cx, |user_store, cx| { - user_store.get_user(envelope.payload.calling_user_id, cx) - })? - .await?, - initial_project: envelope.payload.initial_project, - }; - this.update(&mut cx, |this, _| { - *this.incoming_call.0.borrow_mut() = Some(call); - })?; - - Ok(proto::Ack {}) - } - - async fn handle_call_canceled( - this: Entity, - envelope: TypedEnvelope, - mut cx: AsyncApp, - ) -> Result<()> { - this.update(&mut cx, |this, _| { - let mut incoming_call = this.incoming_call.0.borrow_mut(); - if incoming_call - .as_ref() - .map_or(false, |call| call.room_id == envelope.payload.room_id) - { - incoming_call.take(); - } - })?; - Ok(()) - } - - pub fn global(cx: &App) -> Entity { - cx.global::().0.clone() - } - - pub fn try_global(cx: &App) -> Option> { - cx.try_global::() - .map(|call| call.0.clone()) - } - - pub fn invite( - &mut self, - called_user_id: u64, - initial_project: Option>, - cx: &mut Context, - ) -> Task> { - if !self.pending_invites.insert(called_user_id) { - return Task::ready(Err(anyhow!("user was already invited"))); - } - cx.notify(); - - if self._join_debouncer.running() { - return Task::ready(Ok(())); - } - - let room = if let Some(room) = self.room().cloned() { - Some(Task::ready(Ok(room)).shared()) - } else { - self.pending_room_creation.clone() - }; - - let invite = if let Some(room) = room { - cx.spawn(async move |_, cx| { - let room = room.await.map_err(|err| anyhow!("{:?}", err))?; - - let initial_project_id = if let Some(initial_project) = initial_project { - Some( - room.update(cx, |room, cx| room.share_project(initial_project, cx))? - .await?, - ) - } else { - None - }; - - room.update(cx, move |room, cx| { - room.call(called_user_id, initial_project_id, cx) - })? - .await?; - - anyhow::Ok(()) - }) - } else { - let client = self.client.clone(); - let user_store = self.user_store.clone(); - let room = cx - .spawn(async move |this, cx| { - let create_room = async { - let room = cx - .update(|cx| { - Room::create( - called_user_id, - initial_project, - client, - user_store, - cx, - ) - })? - .await?; - - this.update(cx, |this, cx| this.set_room(Some(room.clone()), cx))? - .await?; - - anyhow::Ok(room) - }; - - let room = create_room.await; - this.update(cx, |this, _| this.pending_room_creation = None)?; - room.map_err(Arc::new) - }) - .shared(); - self.pending_room_creation = Some(room.clone()); - cx.background_spawn(async move { - room.await.map_err(|err| anyhow!("{:?}", err))?; - anyhow::Ok(()) - }) - }; - - cx.spawn(async move |this, cx| { - let result = invite.await; - if result.is_ok() { - this.update(cx, |this, cx| { - this.report_call_event("Participant Invited", cx) - })?; - } else { - //TODO: report collaboration error - log::error!("invite failed: {:?}", result); - } - - this.update(cx, |this, cx| { - this.pending_invites.remove(&called_user_id); - cx.notify(); - })?; - result - }) - } - - pub fn cancel_invite( - &mut self, - called_user_id: u64, - cx: &mut Context, - ) -> Task> { - let room_id = if let Some(room) = self.room() { - room.read(cx).id() - } else { - return Task::ready(Err(anyhow!("no active call"))); - }; - - let client = self.client.clone(); - cx.background_spawn(async move { - client - .request(proto::CancelCall { - room_id, - called_user_id, - }) - .await?; - anyhow::Ok(()) - }) - } - - pub fn incoming(&self) -> watch::Receiver> { - self.incoming_call.1.clone() - } - - pub fn accept_incoming(&mut self, cx: &mut Context) -> Task> { - if self.room.is_some() { - return Task::ready(Err(anyhow!("cannot join while on another call"))); - } - - let call = if let Some(call) = self.incoming_call.0.borrow_mut().take() { - call - } else { - return Task::ready(Err(anyhow!("no incoming call"))); - }; - - if self.pending_room_creation.is_some() { - return Task::ready(Ok(())); - } - - let room_id = call.room_id; - let client = self.client.clone(); - let user_store = self.user_store.clone(); - let join = self._join_debouncer.spawn(cx, move |mut cx| async move { - Room::join(room_id, client, user_store, &mut cx).await - }); - - cx.spawn(async move |this, cx| { - let room = join.await?; - this.update(cx, |this, cx| this.set_room(room.clone(), cx))? - .await?; - this.update(cx, |this, cx| { - this.report_call_event("Incoming Call Accepted", cx) - })?; - Ok(()) - }) - } - - pub fn decline_incoming(&mut self, _: &mut Context) -> Result<()> { - let call = self - .incoming_call - .0 - .borrow_mut() - .take() - .ok_or_else(|| anyhow!("no incoming call"))?; - telemetry::event!("Incoming Call Declined", room_id = call.room_id); - self.client.send(proto::DeclineCall { - room_id: call.room_id, - })?; - Ok(()) - } - - pub fn join_channel( - &mut self, - channel_id: ChannelId, - cx: &mut Context, - ) -> Task>>> { - if let Some(room) = self.room().cloned() { - if room.read(cx).channel_id() == Some(channel_id) { - return Task::ready(Ok(Some(room))); - } else { - room.update(cx, |room, cx| room.clear_state(cx)); - } - } - - if self.pending_room_creation.is_some() { - return Task::ready(Ok(None)); - } - - let client = self.client.clone(); - let user_store = self.user_store.clone(); - let join = self._join_debouncer.spawn(cx, move |mut cx| async move { - Room::join_channel(channel_id, client, user_store, &mut cx).await - }); - - cx.spawn(async move |this, cx| { - let room = join.await?; - this.update(cx, |this, cx| this.set_room(room.clone(), cx))? - .await?; - this.update(cx, |this, cx| this.report_call_event("Channel Joined", cx))?; - Ok(room) - }) - } - - pub fn hang_up(&mut self, cx: &mut Context) -> Task> { - cx.notify(); - self.report_call_event("Call Ended", cx); - - Audio::end_call(cx); - - let channel_id = self.channel_id(cx); - if let Some((room, _)) = self.room.take() { - cx.emit(Event::RoomLeft { channel_id }); - room.update(cx, |room, cx| room.leave(cx)) - } else { - Task::ready(Ok(())) - } - } - - pub fn share_project( - &mut self, - project: Entity, - cx: &mut Context, - ) -> Task> { - if let Some((room, _)) = self.room.as_ref() { - self.report_call_event("Project Shared", cx); - room.update(cx, |room, cx| room.share_project(project, cx)) - } else { - Task::ready(Err(anyhow!("no active call"))) - } - } - - pub fn unshare_project( - &mut self, - project: Entity, - cx: &mut Context, - ) -> Result<()> { - if let Some((room, _)) = self.room.as_ref() { - self.report_call_event("Project Unshared", cx); - room.update(cx, |room, cx| room.unshare_project(project, cx)) - } else { - Err(anyhow!("no active call")) - } - } - - pub fn location(&self) -> Option<&WeakEntity> { - self.location.as_ref() - } - - pub fn set_location( - &mut self, - project: Option<&Entity>, - cx: &mut Context, - ) -> Task> { - if project.is_some() || !*ZED_ALWAYS_ACTIVE { - self.location = project.map(|project| project.downgrade()); - if let Some((room, _)) = self.room.as_ref() { - return room.update(cx, |room, cx| room.set_location(project, cx)); - } - } - Task::ready(Ok(())) - } - - fn set_room(&mut self, room: Option>, cx: &mut Context) -> Task> { - if room.as_ref() == self.room.as_ref().map(|room| &room.0) { - Task::ready(Ok(())) - } else { - cx.notify(); - if let Some(room) = room { - if room.read(cx).status().is_offline() { - self.room = None; - Task::ready(Ok(())) - } else { - let subscriptions = vec![ - cx.observe(&room, |this, room, cx| { - if room.read(cx).status().is_offline() { - this.set_room(None, cx).detach_and_log_err(cx); - } - - cx.notify(); - }), - cx.subscribe(&room, |_, _, event, cx| cx.emit(event.clone())), - ]; - self.room = Some((room.clone(), subscriptions)); - let location = self - .location - .as_ref() - .and_then(|location| location.upgrade()); - let channel_id = room.read(cx).channel_id(); - cx.emit(Event::RoomJoined { channel_id }); - room.update(cx, |room, cx| room.set_location(location.as_ref(), cx)) - } - } else { - self.room = None; - Task::ready(Ok(())) - } - } - } - - pub fn room(&self) -> Option<&Entity> { - self.room.as_ref().map(|(room, _)| room) - } - - pub fn client(&self) -> Arc { - self.client.clone() - } - - pub fn pending_invites(&self) -> &HashSet { - &self.pending_invites - } - - pub fn report_call_event(&self, operation: &'static str, cx: &mut App) { - if let Some(room) = self.room() { - let room = room.read(cx); - telemetry::event!( - operation, - room_id = room.id(), - channel_id = room.channel_id() - ); - } - } -} - -#[cfg(test)] -mod test { - use gpui::TestAppContext; - - use crate::OneAtATime; - - #[gpui::test] - async fn test_one_at_a_time(cx: &mut TestAppContext) { - let mut one_at_a_time = OneAtATime { cancel: None }; - - assert_eq!( - cx.update(|cx| one_at_a_time.spawn(cx, |_| async { Ok(1) })) - .await - .unwrap(), - Some(1) - ); - - let (a, b) = cx.update(|cx| { - ( - one_at_a_time.spawn(cx, |_| async { - panic!(""); - }), - one_at_a_time.spawn(cx, |_| async { Ok(3) }), - ) - }); - - assert_eq!(a.await.unwrap(), None::); - assert_eq!(b.await.unwrap(), Some(3)); - - let promise = cx.update(|cx| one_at_a_time.spawn(cx, |_| async { Ok(4) })); - drop(one_at_a_time); - - assert_eq!(promise.await.unwrap(), None); - } -} diff --git a/crates/call/src/macos/room.rs b/crates/call/src/macos/room.rs deleted file mode 100644 index bfddd624fc..0000000000 --- a/crates/call/src/macos/room.rs +++ /dev/null @@ -1,1707 +0,0 @@ -use crate::{ - call_settings::CallSettings, - participant::{LocalParticipant, ParticipantLocation, RemoteParticipant}, -}; -use anyhow::{anyhow, Result}; -use audio::{Audio, Sound}; -use client::{ - proto::{self, PeerId}, - ChannelId, Client, ParticipantIndex, TypedEnvelope, User, UserStore, -}; -use collections::{BTreeMap, HashMap, HashSet}; -use fs::Fs; -use futures::{FutureExt, StreamExt}; -use gpui::{App, AppContext as _, AsyncApp, Context, Entity, EventEmitter, Task, WeakEntity}; -use language::LanguageRegistry; -use livekit_client_macos::{LocalAudioTrack, LocalTrackPublication, LocalVideoTrack, RoomUpdate}; -use postage::{sink::Sink, stream::Stream, watch}; -use project::Project; -use settings::Settings as _; -use std::{future::Future, mem, sync::Arc, time::Duration}; -use util::{post_inc, ResultExt, TryFutureExt}; - -pub const RECONNECT_TIMEOUT: Duration = Duration::from_secs(30); - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Event { - RoomJoined { - channel_id: Option, - }, - ParticipantLocationChanged { - participant_id: proto::PeerId, - }, - RemoteVideoTracksChanged { - participant_id: proto::PeerId, - }, - RemoteAudioTracksChanged { - participant_id: proto::PeerId, - }, - RemoteProjectShared { - owner: Arc, - project_id: u64, - worktree_root_names: Vec, - }, - RemoteProjectUnshared { - project_id: u64, - }, - RemoteProjectJoined { - project_id: u64, - }, - RemoteProjectInvitationDiscarded { - project_id: u64, - }, - RoomLeft { - channel_id: Option, - }, -} - -pub struct Room { - id: u64, - channel_id: Option, - live_kit: Option, - status: RoomStatus, - shared_projects: HashSet>, - joined_projects: HashSet>, - local_participant: LocalParticipant, - remote_participants: BTreeMap, - pending_participants: Vec>, - participant_user_ids: HashSet, - pending_call_count: usize, - leave_when_empty: bool, - client: Arc, - user_store: Entity, - follows_by_leader_id_project_id: HashMap<(PeerId, u64), Vec>, - client_subscriptions: Vec, - _subscriptions: Vec, - room_update_completed_tx: watch::Sender>, - room_update_completed_rx: watch::Receiver>, - pending_room_update: Option>, - maintain_connection: Option>>, -} - -impl EventEmitter for Room {} - -impl Room { - pub fn channel_id(&self) -> Option { - self.channel_id - } - - pub fn is_sharing_project(&self) -> bool { - !self.shared_projects.is_empty() - } - - #[cfg(any(test, feature = "test-support"))] - pub fn is_connected(&self) -> bool { - if let Some(live_kit) = self.live_kit.as_ref() { - matches!( - *live_kit.room.status().borrow(), - livekit_client_macos::ConnectionState::Connected { .. } - ) - } else { - false - } - } - - fn new( - id: u64, - channel_id: Option, - live_kit_connection_info: Option, - client: Arc, - user_store: Entity, - cx: &mut Context, - ) -> Self { - let live_kit_room = if let Some(connection_info) = live_kit_connection_info { - let room = livekit_client_macos::Room::new(); - let mut status = room.status(); - // Consume the initial status of the room. - let _ = status.try_recv(); - let _maintain_room = cx.spawn(async move |this, cx| { - while let Some(status) = status.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - if status == livekit_client_macos::ConnectionState::Disconnected { - this.update(cx, |this, cx| this.leave(cx).log_err()).ok(); - break; - } - } - }); - - let _handle_updates = cx.spawn({ - let room = room.clone(); - async move |this, cx| { - let mut updates = room.updates(); - while let Some(update) = updates.next().await { - let this = if let Some(this) = this.upgrade() { - this - } else { - break; - }; - - this.update(cx, |this, cx| { - this.live_kit_room_updated(update, cx).log_err() - }) - .ok(); - } - } - }); - - let connect = room.connect(&connection_info.server_url, &connection_info.token); - cx.spawn(async move |this, cx| { - connect.await?; - this.update(cx, |this, cx| { - if this.can_use_microphone() { - if let Some(live_kit) = &this.live_kit { - if !live_kit.muted_by_user && !live_kit.deafened { - return this.share_microphone(cx); - } - } - } - Task::ready(Ok(())) - })? - .await - }) - .detach_and_log_err(cx); - - Some(LiveKitRoom { - room, - screen_track: LocalTrack::None, - microphone_track: LocalTrack::None, - next_publish_id: 0, - muted_by_user: Self::mute_on_join(cx), - deafened: false, - speaking: false, - _maintain_room, - _handle_updates, - }) - } else { - None - }; - - let maintain_connection = cx.spawn({ - let client = client.clone(); - async move |this, cx| { - Self::maintain_connection(this, client.clone(), cx) - .log_err() - .await - } - }); - - Audio::play_sound(Sound::Joined, cx); - - let (room_update_completed_tx, room_update_completed_rx) = watch::channel(); - - Self { - id, - channel_id, - live_kit: live_kit_room, - status: RoomStatus::Online, - shared_projects: Default::default(), - joined_projects: Default::default(), - participant_user_ids: Default::default(), - local_participant: Default::default(), - remote_participants: Default::default(), - pending_participants: Default::default(), - pending_call_count: 0, - client_subscriptions: vec![ - client.add_message_handler(cx.weak_entity(), Self::handle_room_updated) - ], - _subscriptions: vec![ - cx.on_release(Self::released), - cx.on_app_quit(Self::app_will_quit), - ], - leave_when_empty: false, - pending_room_update: None, - client, - user_store, - follows_by_leader_id_project_id: Default::default(), - maintain_connection: Some(maintain_connection), - room_update_completed_tx, - room_update_completed_rx, - } - } - - pub(crate) fn create( - called_user_id: u64, - initial_project: Option>, - client: Arc, - user_store: Entity, - cx: &mut App, - ) -> Task>> { - cx.spawn(async move |cx| { - let response = client.request(proto::CreateRoom {}).await?; - let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; - let room = cx.new(|cx| { - let mut room = Self::new( - room_proto.id, - None, - response.live_kit_connection_info, - client, - user_store, - cx, - ); - if let Some(participant) = room_proto.participants.first() { - room.local_participant.role = participant.role() - } - room - })?; - - let initial_project_id = if let Some(initial_project) = initial_project { - let initial_project_id = room - .update(cx, |room, cx| { - room.share_project(initial_project.clone(), cx) - })? - .await?; - Some(initial_project_id) - } else { - None - }; - - let did_join = room - .update(cx, |room, cx| { - room.leave_when_empty = true; - room.call(called_user_id, initial_project_id, cx) - })? - .await; - match did_join { - Ok(()) => Ok(room), - Err(error) => Err(error.context("room creation failed")), - } - }) - } - - pub(crate) async fn join_channel( - channel_id: ChannelId, - client: Arc, - user_store: Entity, - cx: &mut AsyncApp, - ) -> Result> { - Self::from_join_response( - client - .request(proto::JoinChannel { - channel_id: channel_id.0, - }) - .await?, - client, - user_store, - cx, - ) - } - - pub(crate) async fn join( - room_id: u64, - client: Arc, - user_store: Entity, - cx: &mut AsyncApp, - ) -> Result> { - Self::from_join_response( - client.request(proto::JoinRoom { id: room_id }).await?, - client, - user_store, - cx, - ) - } - - fn released(&mut self, cx: &mut App) { - if self.status.is_online() { - self.leave_internal(cx).detach_and_log_err(cx); - } - } - - fn app_will_quit(&mut self, cx: &mut Context) -> impl Future { - let task = if self.status.is_online() { - let leave = self.leave_internal(cx); - Some(cx.background_spawn(async move { - leave.await.log_err(); - })) - } else { - None - }; - - async move { - if let Some(task) = task { - task.await; - } - } - } - - pub fn mute_on_join(cx: &App) -> bool { - CallSettings::get_global(cx).mute_on_join || client::IMPERSONATE_LOGIN.is_some() - } - - fn from_join_response( - response: proto::JoinRoomResponse, - client: Arc, - user_store: Entity, - cx: &mut AsyncApp, - ) -> Result> { - let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; - let room = cx.new(|cx| { - Self::new( - room_proto.id, - response.channel_id.map(ChannelId), - response.live_kit_connection_info, - client, - user_store, - cx, - ) - })?; - room.update(cx, |room, cx| { - room.leave_when_empty = room.channel_id.is_none(); - room.apply_room_update(room_proto, cx)?; - anyhow::Ok(()) - })??; - Ok(room) - } - - fn should_leave(&self) -> bool { - self.leave_when_empty - && self.pending_room_update.is_none() - && self.pending_participants.is_empty() - && self.remote_participants.is_empty() - && self.pending_call_count == 0 - } - - pub(crate) fn leave(&mut self, cx: &mut Context) -> Task> { - cx.notify(); - self.leave_internal(cx) - } - - fn leave_internal(&mut self, cx: &mut App) -> Task> { - if self.status.is_offline() { - return Task::ready(Err(anyhow!("room is offline"))); - } - - log::info!("leaving room"); - Audio::play_sound(Sound::Leave, cx); - - self.clear_state(cx); - - let leave_room = self.client.request(proto::LeaveRoom {}); - cx.background_spawn(async move { - leave_room.await?; - anyhow::Ok(()) - }) - } - - pub(crate) fn clear_state(&mut self, cx: &mut App) { - for project in self.shared_projects.drain() { - if let Some(project) = project.upgrade() { - project.update(cx, |project, cx| { - project.unshare(cx).log_err(); - }); - } - } - for project in self.joined_projects.drain() { - if let Some(project) = project.upgrade() { - project.update(cx, |project, cx| { - project.disconnected_from_host(cx); - project.close(cx); - }); - } - } - - self.status = RoomStatus::Offline; - self.remote_participants.clear(); - self.pending_participants.clear(); - self.participant_user_ids.clear(); - self.client_subscriptions.clear(); - self.live_kit.take(); - self.pending_room_update.take(); - self.maintain_connection.take(); - } - - async fn maintain_connection( - this: WeakEntity, - client: Arc, - cx: &mut AsyncApp, - ) -> Result<()> { - let mut client_status = client.status(); - loop { - let _ = client_status.try_recv(); - let is_connected = client_status.borrow().is_connected(); - // Even if we're initially connected, any future change of the status means we momentarily disconnected. - if !is_connected || client_status.next().await.is_some() { - log::info!("detected client disconnection"); - - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(cx, |this, cx| { - this.status = RoomStatus::Rejoining; - cx.notify(); - })?; - - // Wait for client to re-establish a connection to the server. - { - let mut reconnection_timeout = - cx.background_executor().timer(RECONNECT_TIMEOUT).fuse(); - let client_reconnection = async { - let mut remaining_attempts = 3; - while remaining_attempts > 0 { - if client_status.borrow().is_connected() { - log::info!("client reconnected, attempting to rejoin room"); - - let Some(this) = this.upgrade() else { break }; - match this.update(cx, |this, cx| this.rejoin(cx)) { - Ok(task) => { - if task.await.log_err().is_some() { - return true; - } else { - remaining_attempts -= 1; - } - } - Err(_app_dropped) => return false, - } - } else if client_status.borrow().is_signed_out() { - return false; - } - - log::info!( - "waiting for client status change, remaining attempts {}", - remaining_attempts - ); - client_status.next().await; - } - false - } - .fuse(); - futures::pin_mut!(client_reconnection); - - futures::select_biased! { - reconnected = client_reconnection => { - if reconnected { - log::info!("successfully reconnected to room"); - // If we successfully joined the room, go back around the loop - // waiting for future connection status changes. - continue; - } - } - _ = reconnection_timeout => { - log::info!("room reconnection timeout expired"); - } - } - } - - break; - } - } - - // The client failed to re-establish a connection to the server - // or an error occurred while trying to re-join the room. Either way - // we leave the room and return an error. - if let Some(this) = this.upgrade() { - log::info!("reconnection failed, leaving room"); - this.update(cx, |this, cx| this.leave(cx))?.await?; - } - Err(anyhow!( - "can't reconnect to room: client failed to re-establish connection" - )) - } - - fn rejoin(&mut self, cx: &mut Context) -> Task> { - let mut projects = HashMap::default(); - let mut reshared_projects = Vec::new(); - let mut rejoined_projects = Vec::new(); - self.shared_projects.retain(|project| { - if let Some(handle) = project.upgrade() { - let project = handle.read(cx); - if let Some(project_id) = project.remote_id() { - projects.insert(project_id, handle.clone()); - reshared_projects.push(proto::UpdateProject { - project_id, - worktrees: project.worktree_metadata_protos(cx), - }); - return true; - } - } - false - }); - self.joined_projects.retain(|project| { - if let Some(handle) = project.upgrade() { - let project = handle.read(cx); - if let Some(project_id) = project.remote_id() { - projects.insert(project_id, handle.clone()); - let mut worktrees = Vec::new(); - let mut repositories = Vec::new(); - for worktree in project.worktrees(cx) { - let worktree = worktree.read(cx); - worktrees.push(proto::RejoinWorktree { - id: worktree.id().to_proto(), - scan_id: worktree.completed_scan_id() as u64, - }); - } - for (entry_id, repository) in project.repositories(cx) { - let repository = repository.read(cx); - repositories.push(proto::RejoinRepository { - id: entry_id.to_proto(), - scan_id: repository.completed_scan_id as u64, - }); - } - - rejoined_projects.push(proto::RejoinProject { - id: project_id, - worktrees, - repositories, - }); - } - return true; - } - false - }); - - let response = self.client.request_envelope(proto::RejoinRoom { - id: self.id, - reshared_projects, - rejoined_projects, - }); - - cx.spawn(async move |this, cx| { - let response = response.await?; - let message_id = response.message_id; - let response = response.payload; - let room_proto = response.room.ok_or_else(|| anyhow!("invalid room"))?; - this.update(cx, |this, cx| { - this.status = RoomStatus::Online; - this.apply_room_update(room_proto, cx)?; - - for reshared_project in response.reshared_projects { - if let Some(project) = projects.get(&reshared_project.id) { - project.update(cx, |project, cx| { - project.reshared(reshared_project, cx).log_err(); - }); - } - } - - for rejoined_project in response.rejoined_projects { - if let Some(project) = projects.get(&rejoined_project.id) { - project.update(cx, |project, cx| { - project.rejoined(rejoined_project, message_id, cx).log_err(); - }); - } - } - - anyhow::Ok(()) - })? - }) - } - - pub fn id(&self) -> u64 { - self.id - } - - pub fn status(&self) -> RoomStatus { - self.status - } - - pub fn local_participant(&self) -> &LocalParticipant { - &self.local_participant - } - - pub fn local_participant_user(&self, cx: &App) -> Option> { - self.user_store.read(cx).current_user() - } - - pub fn remote_participants(&self) -> &BTreeMap { - &self.remote_participants - } - - pub fn remote_participant_for_peer_id(&self, peer_id: PeerId) -> Option<&RemoteParticipant> { - self.remote_participants - .values() - .find(|p| p.peer_id == peer_id) - } - - pub fn role_for_user(&self, user_id: u64) -> Option { - self.remote_participants - .get(&user_id) - .map(|participant| participant.role) - } - - pub fn contains_guests(&self) -> bool { - self.local_participant.role == proto::ChannelRole::Guest - || self - .remote_participants - .values() - .any(|p| p.role == proto::ChannelRole::Guest) - } - - pub fn local_participant_is_admin(&self) -> bool { - self.local_participant.role == proto::ChannelRole::Admin - } - - pub fn local_participant_is_guest(&self) -> bool { - self.local_participant.role == proto::ChannelRole::Guest - } - - pub fn set_participant_role( - &mut self, - user_id: u64, - role: proto::ChannelRole, - cx: &Context, - ) -> Task> { - let client = self.client.clone(); - let room_id = self.id; - let role = role.into(); - cx.spawn(async move |_, _| { - client - .request(proto::SetRoomParticipantRole { - room_id, - user_id, - role, - }) - .await - .map(|_| ()) - }) - } - - pub fn pending_participants(&self) -> &[Arc] { - &self.pending_participants - } - - pub fn contains_participant(&self, user_id: u64) -> bool { - self.participant_user_ids.contains(&user_id) - } - - pub fn followers_for(&self, leader_id: PeerId, project_id: u64) -> &[PeerId] { - self.follows_by_leader_id_project_id - .get(&(leader_id, project_id)) - .map_or(&[], |v| v.as_slice()) - } - - /// Returns the most 'active' projects, defined as most people in the project - pub fn most_active_project(&self, cx: &App) -> Option<(u64, u64)> { - let mut project_hosts_and_guest_counts = HashMap::, u32)>::default(); - for participant in self.remote_participants.values() { - match participant.location { - ParticipantLocation::SharedProject { project_id } => { - project_hosts_and_guest_counts - .entry(project_id) - .or_default() - .1 += 1; - } - ParticipantLocation::External | ParticipantLocation::UnsharedProject => {} - } - for project in &participant.projects { - project_hosts_and_guest_counts - .entry(project.id) - .or_default() - .0 = Some(participant.user.id); - } - } - - if let Some(user) = self.user_store.read(cx).current_user() { - for project in &self.local_participant.projects { - project_hosts_and_guest_counts - .entry(project.id) - .or_default() - .0 = Some(user.id); - } - } - - project_hosts_and_guest_counts - .into_iter() - .filter_map(|(id, (host, guest_count))| Some((id, host?, guest_count))) - .max_by_key(|(_, _, guest_count)| *guest_count) - .map(|(id, host, _)| (id, host)) - } - - async fn handle_room_updated( - this: Entity, - envelope: TypedEnvelope, - mut cx: AsyncApp, - ) -> Result<()> { - let room = envelope - .payload - .room - .ok_or_else(|| anyhow!("invalid room"))?; - this.update(&mut cx, |this, cx| this.apply_room_update(room, cx))? - } - - fn apply_room_update(&mut self, mut room: proto::Room, cx: &mut Context) -> Result<()> { - // Filter ourselves out from the room's participants. - let local_participant_ix = room - .participants - .iter() - .position(|participant| Some(participant.user_id) == self.client.user_id()); - let local_participant = local_participant_ix.map(|ix| room.participants.swap_remove(ix)); - - let pending_participant_user_ids = room - .pending_participants - .iter() - .map(|p| p.user_id) - .collect::>(); - - let remote_participant_user_ids = room - .participants - .iter() - .map(|p| p.user_id) - .collect::>(); - - let (remote_participants, pending_participants) = - self.user_store.update(cx, move |user_store, cx| { - ( - user_store.get_users(remote_participant_user_ids, cx), - user_store.get_users(pending_participant_user_ids, cx), - ) - }); - - self.pending_room_update = Some(cx.spawn(async move |this, cx| { - let (remote_participants, pending_participants) = - futures::join!(remote_participants, pending_participants); - - this.update(cx, |this, cx| { - this.participant_user_ids.clear(); - - if let Some(participant) = local_participant { - let role = participant.role(); - this.local_participant.projects = participant.projects; - if this.local_participant.role != role { - this.local_participant.role = role; - - if role == proto::ChannelRole::Guest { - for project in mem::take(&mut this.shared_projects) { - if let Some(project) = project.upgrade() { - this.unshare_project(project, cx).log_err(); - } - } - this.local_participant.projects.clear(); - if let Some(live_kit_room) = &mut this.live_kit { - live_kit_room.stop_publishing(cx); - } - } - - this.joined_projects.retain(|project| { - if let Some(project) = project.upgrade() { - project.update(cx, |project, cx| project.set_role(role, cx)); - true - } else { - false - } - }); - } - } else { - this.local_participant.projects.clear(); - } - - if let Some(participants) = remote_participants.log_err() { - for (participant, user) in room.participants.into_iter().zip(participants) { - let Some(peer_id) = participant.peer_id else { - continue; - }; - let participant_index = ParticipantIndex(participant.participant_index); - this.participant_user_ids.insert(participant.user_id); - - let old_projects = this - .remote_participants - .get(&participant.user_id) - .into_iter() - .flat_map(|existing| &existing.projects) - .map(|project| project.id) - .collect::>(); - let new_projects = participant - .projects - .iter() - .map(|project| project.id) - .collect::>(); - - for project in &participant.projects { - if !old_projects.contains(&project.id) { - cx.emit(Event::RemoteProjectShared { - owner: user.clone(), - project_id: project.id, - worktree_root_names: project.worktree_root_names.clone(), - }); - } - } - - for unshared_project_id in old_projects.difference(&new_projects) { - this.joined_projects.retain(|project| { - if let Some(project) = project.upgrade() { - project.update(cx, |project, cx| { - if project.remote_id() == Some(*unshared_project_id) { - project.disconnected_from_host(cx); - false - } else { - true - } - }) - } else { - false - } - }); - cx.emit(Event::RemoteProjectUnshared { - project_id: *unshared_project_id, - }); - } - - let role = participant.role(); - let location = ParticipantLocation::from_proto(participant.location) - .unwrap_or(ParticipantLocation::External); - if let Some(remote_participant) = - this.remote_participants.get_mut(&participant.user_id) - { - remote_participant.peer_id = peer_id; - remote_participant.projects = participant.projects; - remote_participant.participant_index = participant_index; - if location != remote_participant.location - || role != remote_participant.role - { - remote_participant.location = location; - remote_participant.role = role; - cx.emit(Event::ParticipantLocationChanged { - participant_id: peer_id, - }); - } - } else { - this.remote_participants.insert( - participant.user_id, - RemoteParticipant { - user: user.clone(), - participant_index, - peer_id, - projects: participant.projects, - location, - role, - muted: true, - speaking: false, - video_tracks: Default::default(), - audio_tracks: Default::default(), - }, - ); - - Audio::play_sound(Sound::Joined, cx); - - if let Some(live_kit) = this.live_kit.as_ref() { - let video_tracks = - live_kit.room.remote_video_tracks(&user.id.to_string()); - let audio_tracks = - live_kit.room.remote_audio_tracks(&user.id.to_string()); - let publications = live_kit - .room - .remote_audio_track_publications(&user.id.to_string()); - - for track in video_tracks { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteVideoTrack(track), - cx, - ) - .log_err(); - } - - for (track, publication) in - audio_tracks.iter().zip(publications.iter()) - { - this.live_kit_room_updated( - RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - ), - cx, - ) - .log_err(); - } - } - } - } - - this.remote_participants.retain(|user_id, participant| { - if this.participant_user_ids.contains(user_id) { - true - } else { - for project in &participant.projects { - cx.emit(Event::RemoteProjectUnshared { - project_id: project.id, - }); - } - false - } - }); - } - - if let Some(pending_participants) = pending_participants.log_err() { - this.pending_participants = pending_participants; - for participant in &this.pending_participants { - this.participant_user_ids.insert(participant.id); - } - } - - this.follows_by_leader_id_project_id.clear(); - for follower in room.followers { - let project_id = follower.project_id; - let (leader, follower) = match (follower.leader_id, follower.follower_id) { - (Some(leader), Some(follower)) => (leader, follower), - - _ => { - log::error!("Follower message {follower:?} missing some state"); - continue; - } - }; - - let list = this - .follows_by_leader_id_project_id - .entry((leader, project_id)) - .or_default(); - if !list.contains(&follower) { - list.push(follower); - } - } - - this.pending_room_update.take(); - if this.should_leave() { - log::info!("room is empty, leaving"); - this.leave(cx).detach(); - } - - this.user_store.update(cx, |user_store, cx| { - let participant_indices_by_user_id = this - .remote_participants - .iter() - .map(|(user_id, participant)| (*user_id, participant.participant_index)) - .collect(); - user_store.set_participant_indices(participant_indices_by_user_id, cx); - }); - - this.check_invariants(); - this.room_update_completed_tx.try_send(Some(())).ok(); - cx.notify(); - }) - .ok(); - })); - - cx.notify(); - Ok(()) - } - - pub fn room_update_completed(&mut self) -> impl Future { - let mut done_rx = self.room_update_completed_rx.clone(); - async move { - while let Some(result) = done_rx.next().await { - if result.is_some() { - break; - } - } - } - } - - fn live_kit_room_updated(&mut self, update: RoomUpdate, cx: &mut Context) -> Result<()> { - match update { - RoomUpdate::SubscribedToRemoteVideoTrack(track) => { - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.video_tracks.insert(track_id.clone(), track); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, - } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.video_tracks.remove(&track_id); - cx.emit(Event::RemoteVideoTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::ActiveSpeakersChanged { speakers } => { - let mut speaker_ids = speakers - .into_iter() - .filter_map(|speaker_sid| speaker_sid.parse().ok()) - .collect::>(); - speaker_ids.sort_unstable(); - for (sid, participant) in &mut self.remote_participants { - participant.speaking = speaker_ids.binary_search(sid).is_ok(); - } - if let Some(id) = self.client.user_id() { - if let Some(room) = &mut self.live_kit { - room.speaking = speaker_ids.binary_search(&id).is_ok(); - } - } - } - - RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } => { - let mut found = false; - for participant in &mut self.remote_participants.values_mut() { - for track in participant.audio_tracks.values() { - if track.sid() == track_id { - found = true; - break; - } - } - if found { - participant.muted = muted; - break; - } - } - } - - RoomUpdate::SubscribedToRemoteAudioTrack(track, publication) => { - if let Some(live_kit) = &self.live_kit { - if live_kit.deafened { - track.stop(); - cx.foreground_executor() - .spawn(publication.set_enabled(false)) - .detach(); - } - } - - let user_id = track.publisher_id().parse()?; - let track_id = track.sid().to_string(); - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("subscribed to track by unknown participant"))?; - participant.audio_tracks.insert(track_id.clone(), track); - participant.muted = publication.is_muted(); - - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } => { - let user_id = publisher_id.parse()?; - let participant = self - .remote_participants - .get_mut(&user_id) - .ok_or_else(|| anyhow!("unsubscribed from track by unknown participant"))?; - participant.audio_tracks.remove(&track_id); - cx.emit(Event::RemoteAudioTracksChanged { - participant_id: participant.peer_id, - }); - } - - RoomUpdate::LocalAudioTrackUnpublished { publication } => { - log::info!("unpublished audio track {}", publication.sid()); - if let Some(room) = &mut self.live_kit { - room.microphone_track = LocalTrack::None; - } - } - - RoomUpdate::LocalVideoTrackUnpublished { publication } => { - log::info!("unpublished video track {}", publication.sid()); - if let Some(room) = &mut self.live_kit { - room.screen_track = LocalTrack::None; - } - } - - RoomUpdate::LocalAudioTrackPublished { publication } => { - log::info!("published audio track {}", publication.sid()); - } - - RoomUpdate::LocalVideoTrackPublished { publication } => { - log::info!("published video track {}", publication.sid()); - } - } - - cx.notify(); - Ok(()) - } - - fn check_invariants(&self) { - #[cfg(any(test, feature = "test-support"))] - { - for participant in self.remote_participants.values() { - assert!(self.participant_user_ids.contains(&participant.user.id)); - assert_ne!(participant.user.id, self.client.user_id().unwrap()); - } - - for participant in &self.pending_participants { - assert!(self.participant_user_ids.contains(&participant.id)); - assert_ne!(participant.id, self.client.user_id().unwrap()); - } - - assert_eq!( - self.participant_user_ids.len(), - self.remote_participants.len() + self.pending_participants.len() - ); - } - } - - pub(crate) fn call( - &mut self, - called_user_id: u64, - initial_project_id: Option, - cx: &mut Context, - ) -> Task> { - if self.status.is_offline() { - return Task::ready(Err(anyhow!("room is offline"))); - } - - cx.notify(); - let client = self.client.clone(); - let room_id = self.id; - self.pending_call_count += 1; - cx.spawn(async move |this, cx| { - let result = client - .request(proto::Call { - room_id, - called_user_id, - initial_project_id, - }) - .await; - this.update(cx, |this, cx| { - this.pending_call_count -= 1; - if this.should_leave() { - this.leave(cx).detach_and_log_err(cx); - } - })?; - result?; - Ok(()) - }) - } - - pub fn join_project( - &mut self, - id: u64, - language_registry: Arc, - fs: Arc, - cx: &mut Context, - ) -> Task>> { - let client = self.client.clone(); - let user_store = self.user_store.clone(); - cx.emit(Event::RemoteProjectJoined { project_id: id }); - cx.spawn(async move |this, cx| { - let project = - Project::in_room(id, client, user_store, language_registry, fs, cx.clone()).await?; - - this.update(cx, |this, cx| { - this.joined_projects.retain(|project| { - if let Some(project) = project.upgrade() { - !project.read(cx).is_disconnected(cx) - } else { - false - } - }); - this.joined_projects.insert(project.downgrade()); - })?; - Ok(project) - }) - } - - pub fn share_project( - &mut self, - project: Entity, - cx: &mut Context, - ) -> Task> { - if let Some(project_id) = project.read(cx).remote_id() { - return Task::ready(Ok(project_id)); - } - - let request = self.client.request(proto::ShareProject { - room_id: self.id(), - worktrees: project.read(cx).worktree_metadata_protos(cx), - is_ssh_project: project.read(cx).is_via_ssh(), - }); - - cx.spawn(async move |this, cx| { - let response = request.await?; - - project.update(cx, |project, cx| project.shared(response.project_id, cx))??; - - // If the user's location is in this project, it changes from UnsharedProject to SharedProject. - this.update(cx, |this, cx| { - this.shared_projects.insert(project.downgrade()); - let active_project = this.local_participant.active_project.as_ref(); - if active_project.map_or(false, |location| *location == project) { - this.set_location(Some(&project), cx) - } else { - Task::ready(Ok(())) - } - })? - .await?; - - Ok(response.project_id) - }) - } - - pub(crate) fn unshare_project( - &mut self, - project: Entity, - cx: &mut Context, - ) -> Result<()> { - let project_id = match project.read(cx).remote_id() { - Some(project_id) => project_id, - None => return Ok(()), - }; - - self.client.send(proto::UnshareProject { project_id })?; - project.update(cx, |this, cx| this.unshare(cx))?; - - if self.local_participant.active_project == Some(project.downgrade()) { - self.set_location(Some(&project), cx).detach_and_log_err(cx); - } - Ok(()) - } - - pub(crate) fn set_location( - &mut self, - project: Option<&Entity>, - cx: &mut Context, - ) -> Task> { - if self.status.is_offline() { - return Task::ready(Err(anyhow!("room is offline"))); - } - - let client = self.client.clone(); - let room_id = self.id; - let location = if let Some(project) = project { - self.local_participant.active_project = Some(project.downgrade()); - if let Some(project_id) = project.read(cx).remote_id() { - proto::participant_location::Variant::SharedProject( - proto::participant_location::SharedProject { id: project_id }, - ) - } else { - proto::participant_location::Variant::UnsharedProject( - proto::participant_location::UnsharedProject {}, - ) - } - } else { - self.local_participant.active_project = None; - proto::participant_location::Variant::External(proto::participant_location::External {}) - }; - - cx.notify(); - cx.background_spawn(async move { - client - .request(proto::UpdateParticipantLocation { - room_id, - location: Some(proto::ParticipantLocation { - variant: Some(location), - }), - }) - .await?; - Ok(()) - }) - } - - pub fn is_screen_sharing(&self) -> bool { - self.live_kit.as_ref().map_or(false, |live_kit| { - !matches!(live_kit.screen_track, LocalTrack::None) - }) - } - - pub fn is_sharing_mic(&self) -> bool { - self.live_kit.as_ref().map_or(false, |live_kit| { - !matches!(live_kit.microphone_track, LocalTrack::None) - }) - } - - pub fn is_muted(&self) -> bool { - self.live_kit.as_ref().map_or(false, |live_kit| { - matches!(live_kit.microphone_track, LocalTrack::None) - || live_kit.muted_by_user - || live_kit.deafened - }) - } - - pub fn muted_by_user(&self) -> bool { - self.live_kit - .as_ref() - .map_or(false, |live_kit| live_kit.muted_by_user) - } - - pub fn is_speaking(&self) -> bool { - self.live_kit - .as_ref() - .map_or(false, |live_kit| live_kit.speaking) - } - - pub fn is_deafened(&self) -> Option { - self.live_kit.as_ref().map(|live_kit| live_kit.deafened) - } - - pub fn can_use_microphone(&self) -> bool { - use proto::ChannelRole::*; - match self.local_participant.role { - Admin | Member | Talker => true, - Guest | Banned => false, - } - } - - pub fn can_share_projects(&self) -> bool { - use proto::ChannelRole::*; - match self.local_participant.role { - Admin | Member => true, - Guest | Banned | Talker => false, - } - } - - #[track_caller] - pub fn share_microphone(&mut self, cx: &mut Context) -> Task> { - if self.status.is_offline() { - return Task::ready(Err(anyhow!("room is offline"))); - } - - let publish_id = if let Some(live_kit) = self.live_kit.as_mut() { - let publish_id = post_inc(&mut live_kit.next_publish_id); - live_kit.microphone_track = LocalTrack::Pending { publish_id }; - cx.notify(); - publish_id - } else { - return Task::ready(Err(anyhow!("live-kit was not initialized"))); - }; - - cx.spawn(async move |this, cx| { - let publish_track = async { - let track = LocalAudioTrack::create(); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_audio_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.microphone_track - { - *cur_publish_id != publish_id - } else { - true - }; - - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - if live_kit.muted_by_user || live_kit.deafened { - cx.background_spawn(publication.set_mute(true)).detach(); - } - live_kit.microphone_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); - } - Ok(()) - } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.microphone_track = LocalTrack::None; - cx.notify(); - Err(error) - } - } - } - })? - }) - } - - pub fn share_screen(&mut self, cx: &mut Context) -> Task> { - if self.status.is_offline() { - return Task::ready(Err(anyhow!("room is offline"))); - } else if self.is_screen_sharing() { - return Task::ready(Err(anyhow!("screen was already shared"))); - } - - let (displays, publish_id) = if let Some(live_kit) = self.live_kit.as_mut() { - let publish_id = post_inc(&mut live_kit.next_publish_id); - live_kit.screen_track = LocalTrack::Pending { publish_id }; - cx.notify(); - (live_kit.room.display_sources(), publish_id) - } else { - return Task::ready(Err(anyhow!("live-kit was not initialized"))); - }; - - cx.spawn(async move |this, cx| { - let publish_track = async { - let displays = displays.await?; - let display = displays - .first() - .ok_or_else(|| anyhow!("no display found"))?; - let track = LocalVideoTrack::screen_share_for_display(display); - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(cx, |this, _| { - this.live_kit - .as_ref() - .map(|live_kit| live_kit.room.publish_video_track(track)) - })? - .ok_or_else(|| anyhow!("live-kit was not initialized"))? - .await - }; - - let publication = publish_track.await; - this.upgrade() - .ok_or_else(|| anyhow!("room was dropped"))? - .update(cx, |this, cx| { - let live_kit = this - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - - let canceled = if let LocalTrack::Pending { - publish_id: cur_publish_id, - } = &live_kit.screen_track - { - *cur_publish_id != publish_id - } else { - true - }; - - match publication { - Ok(publication) => { - if canceled { - live_kit.room.unpublish_track(publication); - } else { - live_kit.screen_track = LocalTrack::Published { - track_publication: publication, - }; - cx.notify(); - } - - Audio::play_sound(Sound::StartScreenshare, cx); - - Ok(()) - } - Err(error) => { - if canceled { - Ok(()) - } else { - live_kit.screen_track = LocalTrack::None; - cx.notify(); - Err(error) - } - } - } - })? - }) - } - - pub fn toggle_mute(&mut self, cx: &mut Context) { - if let Some(live_kit) = self.live_kit.as_mut() { - // When unmuting, undeafen if the user was deafened before. - let was_deafened = live_kit.deafened; - if live_kit.muted_by_user - || live_kit.deafened - || matches!(live_kit.microphone_track, LocalTrack::None) - { - live_kit.muted_by_user = false; - live_kit.deafened = false; - } else { - live_kit.muted_by_user = true; - } - let muted = live_kit.muted_by_user; - let should_undeafen = was_deafened && !live_kit.deafened; - - if let Some(task) = self.set_mute(muted, cx) { - task.detach_and_log_err(cx); - } - - if should_undeafen { - if let Some(task) = self.set_deafened(false, cx) { - task.detach_and_log_err(cx); - } - } - } - } - - pub fn toggle_deafen(&mut self, cx: &mut Context) { - if let Some(live_kit) = self.live_kit.as_mut() { - // When deafening, mute the microphone if it was not already muted. - // When un-deafening, unmute the microphone, unless it was explicitly muted. - let deafened = !live_kit.deafened; - live_kit.deafened = deafened; - let should_change_mute = !live_kit.muted_by_user; - - if let Some(task) = self.set_deafened(deafened, cx) { - task.detach_and_log_err(cx); - } - - if should_change_mute { - if let Some(task) = self.set_mute(deafened, cx) { - task.detach_and_log_err(cx); - } - } - } - } - - pub fn unshare_screen(&mut self, cx: &mut Context) -> Result<()> { - if self.status.is_offline() { - return Err(anyhow!("room is offline")); - } - - let live_kit = self - .live_kit - .as_mut() - .ok_or_else(|| anyhow!("live-kit was not initialized"))?; - match mem::take(&mut live_kit.screen_track) { - LocalTrack::None => Err(anyhow!("screen was not shared")), - LocalTrack::Pending { .. } => { - cx.notify(); - Ok(()) - } - LocalTrack::Published { - track_publication, .. - } => { - live_kit.room.unpublish_track(track_publication); - cx.notify(); - - Audio::play_sound(Sound::StopScreenshare, cx); - Ok(()) - } - } - } - - fn set_deafened(&mut self, deafened: bool, cx: &mut Context) -> Option>> { - let live_kit = self.live_kit.as_mut()?; - cx.notify(); - - let mut track_updates = Vec::new(); - for participant in self.remote_participants.values() { - for publication in live_kit - .room - .remote_audio_track_publications(&participant.user.id.to_string()) - { - track_updates.push(publication.set_enabled(!deafened)); - } - - for track in participant.audio_tracks.values() { - if deafened { - track.stop(); - } else { - track.start(); - } - } - } - - Some(cx.foreground_executor().spawn(async move { - for result in futures::future::join_all(track_updates).await { - result?; - } - Ok(()) - })) - } - - fn set_mute(&mut self, should_mute: bool, cx: &mut Context) -> Option>> { - let live_kit = self.live_kit.as_mut()?; - cx.notify(); - - if should_mute { - Audio::play_sound(Sound::Mute, cx); - } else { - Audio::play_sound(Sound::Unmute, cx); - } - - match &mut live_kit.microphone_track { - LocalTrack::None => { - if should_mute { - None - } else { - Some(self.share_microphone(cx)) - } - } - LocalTrack::Pending { .. } => None, - LocalTrack::Published { track_publication } => Some( - cx.foreground_executor() - .spawn(track_publication.set_mute(should_mute)), - ), - } - } - - #[cfg(any(test, feature = "test-support"))] - pub fn set_display_sources(&self, sources: Vec) { - self.live_kit - .as_ref() - .unwrap() - .room - .set_display_sources(sources); - } -} - -struct LiveKitRoom { - room: Arc, - screen_track: LocalTrack, - microphone_track: LocalTrack, - /// Tracks whether we're currently in a muted state due to auto-mute from deafening or manual mute performed by user. - muted_by_user: bool, - deafened: bool, - speaking: bool, - next_publish_id: usize, - _maintain_room: Task<()>, - _handle_updates: Task<()>, -} - -impl LiveKitRoom { - fn stop_publishing(&mut self, cx: &mut Context) { - if let LocalTrack::Published { - track_publication, .. - } = mem::replace(&mut self.microphone_track, LocalTrack::None) - { - self.room.unpublish_track(track_publication); - cx.notify(); - } - - if let LocalTrack::Published { - track_publication, .. - } = mem::replace(&mut self.screen_track, LocalTrack::None) - { - self.room.unpublish_track(track_publication); - cx.notify(); - } - } -} - -enum LocalTrack { - None, - Pending { - publish_id: usize, - }, - Published { - track_publication: LocalTrackPublication, - }, -} - -impl Default for LocalTrack { - fn default() -> Self { - Self::None - } -} - -#[derive(Copy, Clone, PartialEq, Eq)] -pub enum RoomStatus { - Online, - Rejoining, - Offline, -} - -impl RoomStatus { - pub fn is_offline(&self) -> bool { - matches!(self, RoomStatus::Offline) - } - - pub fn is_online(&self) -> bool { - matches!(self, RoomStatus::Online) - } -} diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index f18db8bac9..3914b4ae2f 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -729,10 +729,11 @@ mod mac_os { use anyhow::{anyhow, Context as _, Result}; use core_foundation::{ array::{CFArray, CFIndex}, + base::TCFType as _, string::kCFStringEncodingUTF8, url::{CFURLCreateWithBytes, CFURL}, }; - use core_services::{kLSLaunchDefaults, LSLaunchURLSpec, LSOpenFromURLSpec, TCFType}; + use core_services::{kLSLaunchDefaults, LSLaunchURLSpec, LSOpenFromURLSpec}; use serde::Deserialize; use std::{ ffi::OsStr, @@ -759,7 +760,6 @@ mod mac_os { }, LocalPath { executable: PathBuf, - plist: InfoPlist, }, } @@ -796,34 +796,16 @@ mod mac_os { plist, }) } - _ => { - println!("Bundle path {bundle_path:?} has no *.app extension, attempting to locate a dev build"); - let plist_path = bundle_path - .parent() - .with_context(|| format!("Bundle path {bundle_path:?} has no parent"))? - .join("WebRTC.framework/Resources/Info.plist"); - let plist = - plist::from_file::<_, InfoPlist>(&plist_path).with_context(|| { - format!("Reading dev bundle plist file at {plist_path:?}") - })?; - Ok(Bundle::LocalPath { - executable: bundle_path, - plist, - }) - } + _ => Ok(Bundle::LocalPath { + executable: bundle_path, + }), } } } impl InstalledApp for Bundle { fn zed_version_string(&self) -> String { - let is_dev = matches!(self, Self::LocalPath { .. }); - format!( - "Zed {}{} – {}", - self.plist().bundle_short_version_string, - if is_dev { " (dev)" } else { "" }, - self.path().display(), - ) + format!("Zed {} – {}", self.version(), self.path().display(),) } fn launch(&self, url: String) -> anyhow::Result<()> { @@ -909,10 +891,10 @@ mod mac_os { } impl Bundle { - fn plist(&self) -> &InfoPlist { + fn version(&self) -> String { match self { - Self::App { plist, .. } => plist, - Self::LocalPath { plist, .. } => plist, + Self::App { plist, .. } => plist.bundle_short_version_string.clone(), + Self::LocalPath { .. } => "".to_string(), } } diff --git a/crates/collab/Cargo.toml b/crates/collab/Cargo.toml index eb65e992a1..0f14afae97 100644 --- a/crates/collab/Cargo.toml +++ b/crates/collab/Cargo.toml @@ -100,13 +100,15 @@ extension.workspace = true file_finder.workspace = true fs = { workspace = true, features = ["test-support"] } git = { workspace = true, features = ["test-support"] } -git_ui = { workspace = true, features = ["test-support"] } git_hosting_providers.workspace = true +git_ui = { workspace = true, features = ["test-support"] } gpui = { workspace = true, features = ["test-support"] } +gpui_tokio.workspace = true hyper.workspace = true indoc.workspace = true language = { workspace = true, features = ["test-support"] } language_model = { workspace = true, features = ["test-support"] } +livekit_client = { workspace = true, features = ["test-support"] } lsp = { workspace = true, features = ["test-support"] } menu.workspace = true multi_buffer = { workspace = true, features = ["test-support"] } @@ -131,11 +133,5 @@ util.workspace = true workspace = { workspace = true, features = ["test-support"] } worktree = { workspace = true, features = ["test-support"] } -[target.'cfg(target_os = "macos")'.dev-dependencies] -livekit_client_macos = { workspace = true, features = ["test-support"] } - -[target.'cfg(not(target_os = "macos"))'.dev-dependencies] -livekit_client = { workspace = true, features = ["test-support"] } - [package.metadata.cargo-machete] ignored = ["async-stripe"] diff --git a/crates/collab/src/tests/channel_tests.rs b/crates/collab/src/tests/channel_tests.rs index 74b3f79d64..43f616dc57 100644 --- a/crates/collab/src/tests/channel_tests.rs +++ b/crates/collab/src/tests/channel_tests.rs @@ -387,7 +387,7 @@ async fn test_channel_room( executor.run_until_parked(); let room_a = cx_a.read(|cx| active_call_a.read_with(cx, |call, _| call.room().unwrap().clone())); - cx_a.read(|cx| room_a.read_with(cx, |room, _| assert!(room.is_connected()))); + cx_a.read(|cx| room_a.read_with(cx, |room, cx| assert!(room.is_connected(cx)))); cx_a.read(|cx| { client_a.channel_store().read_with(cx, |channels, _| { @@ -461,7 +461,7 @@ async fn test_channel_room( let room_a = cx_a.read(|cx| active_call_a.read_with(cx, |call, _| call.room().unwrap().clone())); - cx_a.read(|cx| room_a.read_with(cx, |room, _| assert!(room.is_connected()))); + cx_a.read(|cx| room_a.read_with(cx, |room, cx| assert!(room.is_connected(cx)))); assert_eq!( room_participants(&room_a, cx_a), RoomParticipants { @@ -472,7 +472,7 @@ async fn test_channel_room( let room_b = cx_b.read(|cx| active_call_b.read_with(cx, |call, _| call.room().unwrap().clone())); - cx_b.read(|cx| room_b.read_with(cx, |room, _| assert!(room.is_connected()))); + cx_b.read(|cx| room_b.read_with(cx, |room, cx| assert!(room.is_connected(cx)))); assert_eq!( room_participants(&room_b, cx_b), RoomParticipants { @@ -556,7 +556,7 @@ async fn test_channel_room( let room_a = cx_a.read(|cx| active_call_a.read_with(cx, |call, _| call.room().unwrap().clone())); - cx_a.read(|cx| room_a.read_with(cx, |room, _| assert!(room.is_connected()))); + cx_a.read(|cx| room_a.read_with(cx, |room, cx| assert!(room.is_connected(cx)))); assert_eq!( room_participants(&room_a, cx_a), RoomParticipants { @@ -567,7 +567,7 @@ async fn test_channel_room( let room_b = cx_b.read(|cx| active_call_b.read_with(cx, |call, _| call.room().unwrap().clone())); - cx_b.read(|cx| room_b.read_with(cx, |room, _| assert!(room.is_connected()))); + cx_b.read(|cx| room_b.read_with(cx, |room, cx| assert!(room.is_connected(cx)))); assert_eq!( room_participants(&room_b, cx_b), RoomParticipants { diff --git a/crates/collab/src/tests/following_tests.rs b/crates/collab/src/tests/following_tests.rs index a9ee27bff1..bc372d16fe 100644 --- a/crates/collab/src/tests/following_tests.rs +++ b/crates/collab/src/tests/following_tests.rs @@ -436,9 +436,6 @@ async fn test_basic_following( editor_a1.item_id() ); - // TODO: Re-enable this test once we can replace our swift Livekit SDK with the rust SDK - // todo(windows) - // Fix this on Windows #[cfg(all(not(target_os = "macos"), not(target_os = "windows")))] { use crate::rpc::RECONNECT_TIMEOUT; @@ -463,8 +460,9 @@ async fn test_basic_following( .update(cx, |room, cx| room.share_screen(cx)) }) .await - .unwrap(); // This is what breaks + .unwrap(); executor.run_until_parked(); + let shared_screen = workspace_a.update(cx_a, |workspace, cx| { workspace .active_item(cx) diff --git a/crates/collab/src/tests/integration_tests.rs b/crates/collab/src/tests/integration_tests.rs index ee4c7cb3ec..c21b1a8dca 100644 --- a/crates/collab/src/tests/integration_tests.rs +++ b/crates/collab/src/tests/integration_tests.rs @@ -244,60 +244,56 @@ async fn test_basic_calls( } ); - // TODO: Re-enable this test once we can replace our swift Livekit SDK with the rust SDK - #[cfg(not(target_os = "macos"))] - { - // User A shares their screen - let display = gpui::TestScreenCaptureSource::new(); - let events_b = active_call_events(cx_b); - let events_c = active_call_events(cx_c); - cx_a.set_screen_capture_sources(vec![display]); - active_call_a - .update(cx_a, |call, cx| { - call.room() - .unwrap() - .update(cx, |room, cx| room.share_screen(cx)) - }) - .await - .unwrap(); + // User A shares their screen + let display = gpui::TestScreenCaptureSource::new(); + let events_b = active_call_events(cx_b); + let events_c = active_call_events(cx_c); + cx_a.set_screen_capture_sources(vec![display]); + active_call_a + .update(cx_a, |call, cx| { + call.room() + .unwrap() + .update(cx, |room, cx| room.share_screen(cx)) + }) + .await + .unwrap(); - executor.run_until_parked(); + executor.run_until_parked(); - // User B observes the remote screen sharing track. - assert_eq!(events_b.borrow().len(), 1); - let event_b = events_b.borrow().first().unwrap().clone(); - if let call::room::Event::RemoteVideoTracksChanged { participant_id } = event_b { - assert_eq!(participant_id, client_a.peer_id().unwrap()); + // User B observes the remote screen sharing track. + assert_eq!(events_b.borrow().len(), 1); + let event_b = events_b.borrow().first().unwrap().clone(); + if let call::room::Event::RemoteVideoTracksChanged { participant_id } = event_b { + assert_eq!(participant_id, client_a.peer_id().unwrap()); - room_b.read_with(cx_b, |room, _| { - assert_eq!( - room.remote_participants()[&client_a.user_id().unwrap()] - .video_tracks - .len(), - 1 - ); - }); - } else { - panic!("unexpected event") - } + room_b.read_with(cx_b, |room, _| { + assert_eq!( + room.remote_participants()[&client_a.user_id().unwrap()] + .video_tracks + .len(), + 1 + ); + }); + } else { + panic!("unexpected event") + } - // User C observes the remote screen sharing track. - assert_eq!(events_c.borrow().len(), 1); - let event_c = events_c.borrow().first().unwrap().clone(); - if let call::room::Event::RemoteVideoTracksChanged { participant_id } = event_c { - assert_eq!(participant_id, client_a.peer_id().unwrap()); + // User C observes the remote screen sharing track. + assert_eq!(events_c.borrow().len(), 1); + let event_c = events_c.borrow().first().unwrap().clone(); + if let call::room::Event::RemoteVideoTracksChanged { participant_id } = event_c { + assert_eq!(participant_id, client_a.peer_id().unwrap()); - room_c.read_with(cx_c, |room, _| { - assert_eq!( - room.remote_participants()[&client_a.user_id().unwrap()] - .video_tracks - .len(), - 1 - ); - }); - } else { - panic!("unexpected event") - } + room_c.read_with(cx_c, |room, _| { + assert_eq!( + room.remote_participants()[&client_a.user_id().unwrap()] + .video_tracks + .len(), + 1 + ); + }); + } else { + panic!("unexpected event") } // User A leaves the room. @@ -2091,17 +2087,7 @@ async fn test_mute_deafen( audio_tracks_playing: participant .audio_tracks .values() - .map({ - #[cfg(target_os = "macos")] - { - |track| track.is_playing() - } - - #[cfg(not(target_os = "macos"))] - { - |(track, _)| track.rtc_track().enabled() - } - }) + .map(|(track, _)| track.enabled()) .collect(), }) .collect::>() @@ -6238,8 +6224,6 @@ async fn test_contact_requests( } } -// TODO: Re-enable this test once we can replace our swift Livekit SDK with the rust SDK -#[cfg(not(target_os = "macos"))] #[gpui::test(iterations = 10)] async fn test_join_call_after_screen_was_shared( executor: BackgroundExecutor, diff --git a/crates/collab/src/tests/test_server.rs b/crates/collab/src/tests/test_server.rs index 1653782dd9..f139beaa09 100644 --- a/crates/collab/src/tests/test_server.rs +++ b/crates/collab/src/tests/test_server.rs @@ -47,12 +47,8 @@ use std::{ use util::path; use workspace::{Workspace, WorkspaceStore}; -#[cfg(not(target_os = "macos"))] use livekit_client::test::TestServer as LivekitTestServer; -#[cfg(target_os = "macos")] -use livekit_client_macos::TestServer as LivekitTestServer; - pub struct TestServer { pub app_state: Arc, pub test_livekit_server: Arc, @@ -167,6 +163,7 @@ impl TestServer { let fs = FakeFs::new(cx.executor()); cx.update(|cx| { + gpui_tokio::init(cx); if cx.has_global::() { panic!("Same cx used to create two test clients") } diff --git a/crates/evals/build.rs b/crates/evals/build.rs index 8175d90494..8c93f4fb27 100644 --- a/crates/evals/build.rs +++ b/crates/evals/build.rs @@ -1,14 +1,5 @@ fn main() { if cfg!(target_os = "macos") { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); - - println!("cargo:rerun-if-env-changed=ZED_BUNDLE"); - if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { - // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); - } else { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } } } diff --git a/crates/gpui/Cargo.toml b/crates/gpui/Cargo.toml index d3201d61d2..c1de8d9945 100644 --- a/crates/gpui/Cargo.toml +++ b/crates/gpui/Cargo.toml @@ -12,7 +12,7 @@ license = "Apache-2.0" workspace = true [features] -default = ["http_client", "font-kit", "wayland", "x11"] +default = ["macos-blade", "http_client", "font-kit", "wayland", "x11"] test-support = [ "leak-detection", "collections/test-support", @@ -123,10 +123,11 @@ lyon = "1.0" block = "0.1" cocoa.workspace = true core-foundation.workspace = true -core-foundation-sys = "0.8" -core-graphics = "0.23" -core-text = "20.1" -font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "40391b7", optional = true } +core-foundation-sys.workspace = true +core-graphics = "0.24" +core-video.workspace = true +core-text = "21" +font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "5474cfad4b719a72ec8ed2cb7327b2b01fd10568", optional = true } foreign-types = "0.5" log.workspace = true media.workspace = true @@ -154,9 +155,10 @@ blade-macros = { workspace = true, optional = true } blade-util = { workspace = true, optional = true } bytemuck = { version = "1", optional = true } cosmic-text = { version = "0.13.2", optional = true } -font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "40391b7", features = [ +font-kit = { git = "https://github.com/zed-industries/font-kit", rev = "5474cfad4b719a72ec8ed2cb7327b2b01fd10568", features = [ "source-fontconfig-dlopen", ], optional = true } + calloop = { version = "0.13.0" } filedescriptor = { version = "0.8.2", optional = true } open = { version = "5.2.0", optional = true } diff --git a/crates/gpui/src/elements/surface.rs b/crates/gpui/src/elements/surface.rs index 2d25de2c93..707271f2ec 100644 --- a/crates/gpui/src/elements/surface.rs +++ b/crates/gpui/src/elements/surface.rs @@ -3,7 +3,7 @@ use crate::{ Style, StyleRefinement, Styled, Window, }; #[cfg(target_os = "macos")] -use media::core_video::CVImageBuffer; +use core_video::pixel_buffer::CVPixelBuffer; use refineable::Refineable; /// A source of a surface's content. @@ -11,12 +11,12 @@ use refineable::Refineable; pub enum SurfaceSource { /// A macOS image buffer from CoreVideo #[cfg(target_os = "macos")] - Surface(CVImageBuffer), + Surface(CVPixelBuffer), } #[cfg(target_os = "macos")] -impl From for SurfaceSource { - fn from(value: CVImageBuffer) -> Self { +impl From for SurfaceSource { + fn from(value: CVPixelBuffer) -> Self { SurfaceSource::Surface(value) } } @@ -87,7 +87,7 @@ impl Element for Surface { match &self.source { #[cfg(target_os = "macos")] SurfaceSource::Surface(surface) => { - let size = crate::size(surface.width().into(), surface.height().into()); + let size = crate::size(surface.get_width().into(), surface.get_height().into()); let new_bounds = self.object_fit.get_bounds(bounds, size); // TODO: Add support for corner_radii window.paint_surface(new_bounds, surface.clone()); diff --git a/crates/gpui/src/platform/blade/blade_renderer.rs b/crates/gpui/src/platform/blade/blade_renderer.rs index 63b1b6d65e..f8129449b5 100644 --- a/crates/gpui/src/platform/blade/blade_renderer.rs +++ b/crates/gpui/src/platform/blade/blade_renderer.rs @@ -725,8 +725,8 @@ impl BladeRenderer { use std::ptr; assert_eq!( - surface.image_buffer.pixel_format_type(), - media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + surface.image_buffer.get_pixel_format(), + core_video::pixel_buffer::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ); let y_texture = self @@ -735,8 +735,8 @@ impl BladeRenderer { surface.image_buffer.as_concrete_TypeRef(), ptr::null(), metal::MTLPixelFormat::R8Unorm, - surface.image_buffer.plane_width(0), - surface.image_buffer.plane_height(0), + surface.image_buffer.get_width_of_plane(0), + surface.image_buffer.get_height_of_plane(0), 0, ) .unwrap(); @@ -746,8 +746,8 @@ impl BladeRenderer { surface.image_buffer.as_concrete_TypeRef(), ptr::null(), metal::MTLPixelFormat::RG8Unorm, - surface.image_buffer.plane_width(1), - surface.image_buffer.plane_height(1), + surface.image_buffer.get_width_of_plane(1), + surface.image_buffer.get_height_of_plane(1), 1, ) .unwrap(); diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs index bd3d8f35ac..145e6c9edf 100644 --- a/crates/gpui/src/platform/mac.rs +++ b/crates/gpui/src/platform/mac.rs @@ -11,7 +11,7 @@ mod metal_atlas; #[cfg(not(feature = "macos-blade"))] pub mod metal_renderer; -use media::core_video::CVImageBuffer; +use core_video::image_buffer::CVImageBuffer; #[cfg(not(feature = "macos-blade"))] use metal_renderer as renderer; diff --git a/crates/gpui/src/platform/mac/metal_renderer.rs b/crates/gpui/src/platform/mac/metal_renderer.rs index 56109d2ff6..389f97fd99 100644 --- a/crates/gpui/src/platform/mac/metal_renderer.rs +++ b/crates/gpui/src/platform/mac/metal_renderer.rs @@ -13,8 +13,11 @@ use cocoa::{ }; use collections::HashMap; use core_foundation::base::TCFType; -use foreign_types::ForeignType; -use media::core_video::CVMetalTextureCache; +use core_video::{ + metal_texture::CVMetalTextureGetTexture, metal_texture_cache::CVMetalTextureCache, + pixel_buffer::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, +}; +use foreign_types::{ForeignType, ForeignTypeRef}; use metal::{CAMetalLayer, CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange}; use objc::{self, msg_send, sel, sel_impl}; use parking_lot::Mutex; @@ -107,7 +110,7 @@ pub(crate) struct MetalRenderer { #[allow(clippy::arc_with_non_send_sync)] instance_buffer_pool: Arc>, sprite_atlas: Arc, - core_video_texture_cache: CVMetalTextureCache, + core_video_texture_cache: core_video::metal_texture_cache::CVMetalTextureCache, } impl MetalRenderer { @@ -235,7 +238,7 @@ impl MetalRenderer { let command_queue = device.new_command_queue(); let sprite_atlas = Arc::new(MetalAtlas::new(device.clone(), PATH_SAMPLE_COUNT)); let core_video_texture_cache = - unsafe { CVMetalTextureCache::new(device.as_ptr()).unwrap() }; + CVMetalTextureCache::new(None, device.clone(), None).unwrap(); Self { device, @@ -1054,39 +1057,37 @@ impl MetalRenderer { for surface in surfaces { let texture_size = size( - DevicePixels::from(surface.image_buffer.width() as i32), - DevicePixels::from(surface.image_buffer.height() as i32), + DevicePixels::from(surface.image_buffer.get_width() as i32), + DevicePixels::from(surface.image_buffer.get_height() as i32), ); assert_eq!( - surface.image_buffer.pixel_format_type(), - media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + surface.image_buffer.get_pixel_format(), + kCVPixelFormatType_420YpCbCr8BiPlanarFullRange ); - let y_texture = unsafe { - self.core_video_texture_cache - .create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), - ptr::null(), - MTLPixelFormat::R8Unorm, - surface.image_buffer.plane_width(0), - surface.image_buffer.plane_height(0), - 0, - ) - .unwrap() - }; - let cb_cr_texture = unsafe { - self.core_video_texture_cache - .create_texture_from_image( - surface.image_buffer.as_concrete_TypeRef(), - ptr::null(), - MTLPixelFormat::RG8Unorm, - surface.image_buffer.plane_width(1), - surface.image_buffer.plane_height(1), - 1, - ) - .unwrap() - }; + let y_texture = self + .core_video_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + None, + MTLPixelFormat::R8Unorm, + surface.image_buffer.get_width_of_plane(0), + surface.image_buffer.get_height_of_plane(0), + 0, + ) + .unwrap(); + let cb_cr_texture = self + .core_video_texture_cache + .create_texture_from_image( + surface.image_buffer.as_concrete_TypeRef(), + None, + MTLPixelFormat::RG8Unorm, + surface.image_buffer.get_width_of_plane(1), + surface.image_buffer.get_height_of_plane(1), + 1, + ) + .unwrap(); align_offset(instance_offset); let next_offset = *instance_offset + mem::size_of::(); @@ -1104,14 +1105,15 @@ impl MetalRenderer { mem::size_of_val(&texture_size) as u64, &texture_size as *const Size as *const _, ); - command_encoder.set_fragment_texture( - SurfaceInputIndex::YTexture as u64, - Some(y_texture.as_texture_ref()), - ); - command_encoder.set_fragment_texture( - SurfaceInputIndex::CbCrTexture as u64, - Some(cb_cr_texture.as_texture_ref()), - ); + // let y_texture = y_texture.get_texture().unwrap(). + command_encoder.set_fragment_texture(SurfaceInputIndex::YTexture as u64, unsafe { + let texture = CVMetalTextureGetTexture(y_texture.as_concrete_TypeRef()); + Some(metal::TextureRef::from_ptr(texture as *mut _)) + }); + command_encoder.set_fragment_texture(SurfaceInputIndex::CbCrTexture as u64, unsafe { + let texture = CVMetalTextureGetTexture(cb_cr_texture.as_concrete_TypeRef()); + Some(metal::TextureRef::from_ptr(texture as *mut _)) + }); unsafe { let buffer_contents = (instance_buffer.metal_buffer.contents() as *mut u8) diff --git a/crates/gpui/src/platform/mac/screen_capture.rs b/crates/gpui/src/platform/mac/screen_capture.rs index a2b535996f..eaf58ae972 100644 --- a/crates/gpui/src/platform/mac/screen_capture.rs +++ b/crates/gpui/src/platform/mac/screen_capture.rs @@ -9,6 +9,10 @@ use cocoa::{ foundation::NSArray, }; use core_foundation::base::TCFType; +use core_graphics::display::{ + CGDirectDisplayID, CGDisplayCopyDisplayMode, CGDisplayModeGetPixelHeight, + CGDisplayModeGetPixelWidth, CGDisplayModeRelease, +}; use ctor::ctor; use futures::channel::oneshot; use media::core_media::{CMSampleBuffer, CMSampleBufferRef}; @@ -45,8 +49,12 @@ const SCStreamOutputTypeScreen: NSInteger = 0; impl ScreenCaptureSource for MacScreenCaptureSource { fn resolution(&self) -> Result> { unsafe { - let width: i64 = msg_send![self.sc_display, width]; - let height: i64 = msg_send![self.sc_display, height]; + let display_id: CGDirectDisplayID = msg_send![self.sc_display, displayID]; + let display_mode_ref = CGDisplayCopyDisplayMode(display_id); + let width = CGDisplayModeGetPixelWidth(display_mode_ref); + let height = CGDisplayModeGetPixelHeight(display_mode_ref); + CGDisplayModeRelease(display_mode_ref); + Ok(size(px(width as f32), px(height as f32))) } } @@ -65,6 +73,10 @@ impl ScreenCaptureSource for MacScreenCaptureSource { let excluded_windows = NSArray::array(nil); let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows]; let configuration: id = msg_send![configuration, init]; + let _: id = msg_send![configuration, setScalesToFit: true]; + let _: id = msg_send![configuration, setPixelFormat: 0x42475241]; + // let _: id = msg_send![configuration, setShowsCursor: false]; + // let _: id = msg_send![configuration, setCaptureResolution: 3]; let delegate: id = msg_send![delegate, init]; let output: id = msg_send![output, init]; @@ -73,6 +85,9 @@ impl ScreenCaptureSource for MacScreenCaptureSource { Box::into_raw(Box::new(frame_callback)) as *mut c_void, ); + let resolution = self.resolution().unwrap(); + let _: id = msg_send![configuration, setWidth: resolution.width.0 as i64]; + let _: id = msg_send![configuration, setHeight: resolution.height.0 as i64]; let stream: id = msg_send![stream, initWithFilter:filter configuration:configuration delegate:delegate]; let (mut tx, rx) = oneshot::channel(); diff --git a/crates/gpui/src/scene.rs b/crates/gpui/src/scene.rs index 03862bc149..6271d6e88b 100644 --- a/crates/gpui/src/scene.rs +++ b/crates/gpui/src/scene.rs @@ -662,7 +662,7 @@ pub(crate) struct PaintSurface { pub bounds: Bounds, pub content_mask: ContentMask, #[cfg(target_os = "macos")] - pub image_buffer: media::core_video::CVImageBuffer, + pub image_buffer: core_video::pixel_buffer::CVPixelBuffer, } impl From for Primitive { diff --git a/crates/gpui/src/window.rs b/crates/gpui/src/window.rs index 4f2d9fb01e..4c9e968237 100644 --- a/crates/gpui/src/window.rs +++ b/crates/gpui/src/window.rs @@ -17,11 +17,11 @@ use crate::{ }; use anyhow::{anyhow, Context as _, Result}; use collections::{FxHashMap, FxHashSet}; +#[cfg(target_os = "macos")] +use core_video::pixel_buffer::CVPixelBuffer; use derive_more::{Deref, DerefMut}; use futures::channel::oneshot; use futures::FutureExt; -#[cfg(target_os = "macos")] -use media::core_video::CVImageBuffer; use parking_lot::RwLock; use raw_window_handle::{HandleError, HasWindowHandle}; use refineable::Refineable; @@ -2658,7 +2658,7 @@ impl Window { /// /// This method should only be called as part of the paint phase of element drawing. #[cfg(target_os = "macos")] - pub fn paint_surface(&mut self, bounds: Bounds, image_buffer: CVImageBuffer) { + pub fn paint_surface(&mut self, bounds: Bounds, image_buffer: CVPixelBuffer) { use crate::PaintSurface; self.invalidator.debug_assert_paint(); diff --git a/crates/gpui_tokio/src/gpui_tokio.rs b/crates/gpui_tokio/src/gpui_tokio.rs index d6fc4868b7..fffe18a616 100644 --- a/crates/gpui_tokio/src/gpui_tokio.rs +++ b/crates/gpui_tokio/src/gpui_tokio.rs @@ -32,7 +32,7 @@ pub struct Tokio {} impl Tokio { /// Spawns the given future on Tokio's thread pool, and returns it via a GPUI task /// Note that the Tokio task will be cancelled if the GPUI task is dropped - pub fn spawn(cx: &mut C, f: Fut) -> C::Result>> + pub fn spawn(cx: &C, f: Fut) -> C::Result>> where C: AppContext, Fut: Future + Send + 'static, @@ -52,7 +52,7 @@ impl Tokio { }) } - pub fn handle(cx: &mut App) -> tokio::runtime::Handle { + pub fn handle(cx: &App) -> tokio::runtime::Handle { GlobalTokio::global(cx).runtime.handle().clone() } } diff --git a/crates/livekit_client/Cargo.toml b/crates/livekit_client/Cargo.toml index 8d492984ba..674f6900fa 100644 --- a/crates/livekit_client/Cargo.toml +++ b/crates/livekit_client/Cargo.toml @@ -10,46 +10,47 @@ license = "GPL-3.0-or-later" workspace = true [lib] -path = "src/livekit_client.rs" +path = "src/lib.rs" doctest = false [[example]] name = "test_app" [features] -no-webrtc = [] -test-support = ["collections/test-support", "gpui/test-support", "nanoid"] +test-support = ["collections/test-support", "gpui/test-support"] [dependencies] +gpui_tokio.workspace = true anyhow.workspace = true async-trait.workspace = true collections.workspace = true cpal = "0.15" futures.workspace = true gpui.workspace = true -http_2 = { package = "http", version = "0.2.1" } livekit_api.workspace = true log.workspace = true -media.workspace = true -nanoid = { workspace = true, optional = true } +nanoid.workspace = true parking_lot.workspace = true postage.workspace = true util.workspace = true -http_client.workspace = true smallvec.workspace = true image.workspace = true +tokio-tungstenite.workspace = true +http_client_tls.workspace = true [target.'cfg(not(all(target_os = "windows", target_env = "gnu")))'.dependencies] -livekit.workspace = true +livekit = { rev = "102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8", git = "https://github.com/zed-industries/livekit-rust-sdks", features = ["__rustls-tls"]} +libwebrtc = { rev = "102ebbb1ccfbdbcb7332d86dc30b1b1c8c01e4f8", git = "https://github.com/zed-industries/livekit-rust-sdks"} [target.'cfg(target_os = "macos")'.dependencies] core-foundation.workspace = true coreaudio-rs = "0.12.1" +objc = "0.2" +core-video.workspace = true [dev-dependencies] collections = { workspace = true, features = ["test-support"] } gpui = { workspace = true, features = ["test-support"] } -nanoid.workspace = true sha2.workspace = true simplelog.workspace = true diff --git a/crates/livekit_client/examples/test_app.rs b/crates/livekit_client/examples/test_app.rs index ff680cad19..c04d442592 100644 --- a/crates/livekit_client/examples/test_app.rs +++ b/crates/livekit_client/examples/test_app.rs @@ -1,8 +1,6 @@ -#![cfg_attr(windows, allow(unused))] -// TODO: For some reason mac build complains about import of postage::stream::Stream, but removal of -// it causes compile errors. -#![cfg_attr(target_os = "macos", allow(unused_imports))] +use std::sync::Arc; +use futures::StreamExt; use gpui::{ actions, bounds, div, point, prelude::{FluentBuilder as _, IntoElement}, @@ -11,26 +9,9 @@ use gpui::{ StatefulInteractiveElement as _, Styled, Task, Window, WindowBounds, WindowHandle, WindowOptions, }; -#[cfg(not(target_os = "windows"))] use livekit_client::{ - capture_local_audio_track, capture_local_video_track, - id::ParticipantIdentity, - options::{TrackPublishOptions, VideoCodec}, - participant::{Participant, RemoteParticipant}, - play_remote_audio_track, - publication::{LocalTrackPublication, RemoteTrackPublication}, - track::{LocalTrack, RemoteTrack, RemoteVideoTrack, TrackSource}, - AudioStream, RemoteVideoTrackView, Room, RoomEvent, RoomOptions, -}; -#[cfg(not(target_os = "windows"))] -use postage::stream::Stream; - -#[cfg(target_os = "windows")] -use livekit_client::{ - participant::{Participant, RemoteParticipant}, - publication::{LocalTrackPublication, RemoteTrackPublication}, - track::{LocalTrack, RemoteTrack, RemoteVideoTrack}, - AudioStream, RemoteVideoTrackView, Room, RoomEvent, + AudioStream, LocalTrackPublication, Participant, ParticipantIdentity, RemoteParticipant, + RemoteTrackPublication, RemoteVideoTrack, RemoteVideoTrackView, Room, RoomEvent, }; use livekit_api::token::{self, VideoGrant}; @@ -39,25 +20,18 @@ use simplelog::SimpleLogger; actions!(livekit_client, [Quit]); -#[cfg(windows)] -fn main() {} - -#[cfg(not(windows))] fn main() { SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); gpui::Application::new().run(|cx| { - livekit_client::init( - cx.background_executor().dispatcher.clone(), - cx.http_client(), - ); - #[cfg(any(test, feature = "test-support"))] println!("USING TEST LIVEKIT"); #[cfg(not(any(test, feature = "test-support")))] println!("USING REAL LIVEKIT"); + gpui_tokio::init(cx); + cx.activate(true); cx.on_action(quit); cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); @@ -83,14 +57,12 @@ fn main() { &livekit_key, &livekit_secret, Some(&format!("test-participant-{i}")), - VideoGrant::to_join("test-room"), + VideoGrant::to_join("wtej-trty"), ) .unwrap(); let bounds = bounds(point(width * i, px(0.0)), size(width, height)); - let window = - LivekitWindow::new(livekit_url.as_str(), token.as_str(), bounds, cx.clone()) - .await; + let window = LivekitWindow::new(livekit_url.clone(), token, bounds, cx).await; windows.push(window); } }) @@ -103,12 +75,11 @@ fn quit(_: &Quit, cx: &mut gpui::App) { } struct LivekitWindow { - room: Room, + room: Arc, microphone_track: Option, screen_share_track: Option, - microphone_stream: Option, + microphone_stream: Option, screen_share_stream: Option>, - #[cfg(not(target_os = "windows"))] remote_participants: Vec<(ParticipantIdentity, ParticipantState)>, _events_task: Task<()>, } @@ -121,17 +92,23 @@ struct ParticipantState { speaking: bool, } -#[cfg(not(windows))] impl LivekitWindow { async fn new( - url: &str, - token: &str, + url: String, + token: String, bounds: Bounds, - cx: AsyncApp, + cx: &mut AsyncApp, ) -> WindowHandle { - let (room, mut events) = Room::connect(url, token, RoomOptions::default()) - .await - .unwrap(); + let (room, mut events) = + Room::connect(url.clone(), token, cx) + .await + .unwrap_or_else(|err| { + eprintln!( + "Failed to connect to {url}: {err}.\nTry `foreman start` to run the livekit server" + ); + + std::process::exit(1) + }); cx.update(|cx| { cx.open_window( @@ -142,7 +119,7 @@ impl LivekitWindow { |window, cx| { cx.new(|cx| { let _events_task = cx.spawn_in(window, async move |this, cx| { - while let Some(event) = events.recv().await { + while let Some(event) = events.next().await { cx.update(|window, cx| { this.update(cx, |this: &mut LivekitWindow, cx| { this.handle_room_event(event, window, cx) @@ -153,7 +130,7 @@ impl LivekitWindow { }); Self { - room, + room: Arc::new(room), microphone_track: None, microphone_stream: None, screen_share_track: None, @@ -201,15 +178,16 @@ impl LivekitWindow { participant, track, } => { + let room = self.room.clone(); let output = self.remote_participant(participant); match track { - RemoteTrack::Audio(track) => { + livekit_client::RemoteTrack::Audio(track) => { output.audio_output_stream = Some(( publication.clone(), - play_remote_audio_track(&track, cx.background_executor()).unwrap(), + room.play_remote_audio_track(&track, cx).unwrap(), )); } - RemoteTrack::Video(track) => { + livekit_client::RemoteTrack::Video(track) => { output.screen_share_output_view = Some(( track.clone(), cx.new(|cx| RemoteVideoTrackView::new(track, window, cx)), @@ -269,25 +247,15 @@ impl LivekitWindow { fn toggle_mute(&mut self, window: &mut Window, cx: &mut Context) { if let Some(track) = &self.microphone_track { if track.is_muted() { - track.unmute(); + track.unmute(cx); } else { - track.mute(); + track.mute(cx); } cx.notify(); } else { - let participant = self.room.local_participant(); + let room = self.room.clone(); cx.spawn_in(window, async move |this, cx| { - let (track, stream) = capture_local_audio_track(cx.background_executor())?.await; - let publication = participant - .publish_track( - LocalTrack::Audio(track), - TrackPublishOptions { - source: TrackSource::Microphone, - ..Default::default() - }, - ) - .await - .unwrap(); + let (publication, stream) = room.publish_local_microphone_track(cx).await.unwrap(); this.update(cx, |this, cx| { this.microphone_track = Some(publication); this.microphone_stream = Some(stream); @@ -302,8 +270,8 @@ impl LivekitWindow { if let Some(track) = self.screen_share_track.take() { self.screen_share_stream.take(); let participant = self.room.local_participant(); - cx.background_spawn(async move { - participant.unpublish_track(&track.sid()).await.unwrap(); + cx.spawn(async move |_, cx| { + participant.unpublish_track(track.sid(), cx).await.unwrap(); }) .detach(); cx.notify(); @@ -313,16 +281,9 @@ impl LivekitWindow { cx.spawn_in(window, async move |this, cx| { let sources = sources.await.unwrap()?; let source = sources.into_iter().next().unwrap(); - let (track, stream) = capture_local_video_track(&*source).await?; - let publication = participant - .publish_track( - LocalTrack::Video(track), - TrackPublishOptions { - source: TrackSource::Screenshare, - video_codec: VideoCodec::H264, - ..Default::default() - }, - ) + + let (publication, stream) = participant + .publish_screenshare_track(&*source, cx) .await .unwrap(); this.update(cx, |this, cx| { @@ -338,7 +299,6 @@ impl LivekitWindow { fn toggle_remote_audio_for_participant( &mut self, identity: &ParticipantIdentity, - cx: &mut Context, ) -> Option<()> { let participant = self.remote_participants.iter().find_map(|(id, state)| { @@ -349,13 +309,12 @@ impl LivekitWindow { } })?; let publication = &participant.audio_output_stream.as_ref()?.0; - publication.set_enabled(!publication.is_enabled()); + publication.set_enabled(!publication.is_enabled(), cx); cx.notify(); Some(()) } } -#[cfg(not(windows))] impl Render for LivekitWindow { fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { fn button() -> gpui::Div { @@ -407,7 +366,7 @@ impl Render for LivekitWindow { .flex_grow() .children(self.remote_participants.iter().map(|(identity, state)| { div() - .h(px(300.0)) + .h(px(1080.0)) .flex() .flex_col() .m_2() diff --git a/crates/livekit_client/src/lib.rs b/crates/livekit_client/src/lib.rs new file mode 100644 index 0000000000..c35c83f228 --- /dev/null +++ b/crates/livekit_client/src/lib.rs @@ -0,0 +1,165 @@ +use collections::HashMap; + +mod remote_video_track_view; +pub use remote_video_track_view::{RemoteVideoTrackView, RemoteVideoTrackViewEvent}; + +#[cfg(not(any( + test, + feature = "test-support", + all(target_os = "windows", target_env = "gnu") +)))] +mod livekit_client; +#[cfg(not(any( + test, + feature = "test-support", + all(target_os = "windows", target_env = "gnu") +)))] +pub use livekit_client::*; + +#[cfg(any( + test, + feature = "test-support", + all(target_os = "windows", target_env = "gnu") +))] +mod mock_client; +#[cfg(any( + test, + feature = "test-support", + all(target_os = "windows", target_env = "gnu") +))] +pub mod test; +#[cfg(any( + test, + feature = "test-support", + all(target_os = "windows", target_env = "gnu") +))] +pub use mock_client::*; + +#[derive(Debug, Clone)] +pub enum Participant { + Local(LocalParticipant), + Remote(RemoteParticipant), +} + +#[derive(Debug, Clone)] +pub enum TrackPublication { + Local(LocalTrackPublication), + Remote(RemoteTrackPublication), +} + +impl TrackPublication { + pub fn sid(&self) -> TrackSid { + match self { + TrackPublication::Local(local) => local.sid(), + TrackPublication::Remote(remote) => remote.sid(), + } + } + + pub fn is_muted(&self) -> bool { + match self { + TrackPublication::Local(local) => local.is_muted(), + TrackPublication::Remote(remote) => remote.is_muted(), + } + } +} + +#[derive(Clone, Debug)] +pub enum RemoteTrack { + Audio(RemoteAudioTrack), + Video(RemoteVideoTrack), +} + +impl RemoteTrack { + pub fn sid(&self) -> TrackSid { + match self { + RemoteTrack::Audio(remote_audio_track) => remote_audio_track.sid(), + RemoteTrack::Video(remote_video_track) => remote_video_track.sid(), + } + } +} + +#[derive(Clone, Debug)] +pub enum LocalTrack { + Audio(LocalAudioTrack), + Video(LocalVideoTrack), +} + +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum RoomEvent { + ParticipantConnected(RemoteParticipant), + ParticipantDisconnected(RemoteParticipant), + LocalTrackPublished { + publication: LocalTrackPublication, + track: LocalTrack, + participant: LocalParticipant, + }, + LocalTrackUnpublished { + publication: LocalTrackPublication, + participant: LocalParticipant, + }, + LocalTrackSubscribed { + track: LocalTrack, + }, + TrackSubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnsubscribed { + track: RemoteTrack, + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackSubscriptionFailed { + participant: RemoteParticipant, + // error: livekit::track::TrackError, + track_sid: TrackSid, + }, + TrackPublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackUnpublished { + publication: RemoteTrackPublication, + participant: RemoteParticipant, + }, + TrackMuted { + participant: Participant, + publication: TrackPublication, + }, + TrackUnmuted { + participant: Participant, + publication: TrackPublication, + }, + RoomMetadataChanged { + old_metadata: String, + metadata: String, + }, + ParticipantMetadataChanged { + participant: Participant, + old_metadata: String, + metadata: String, + }, + ParticipantNameChanged { + participant: Participant, + old_name: String, + name: String, + }, + ParticipantAttributesChanged { + participant: Participant, + changed_attributes: HashMap, + }, + ActiveSpeakersChanged { + speakers: Vec, + }, + ConnectionStateChanged(ConnectionState), + Connected { + participants_with_tracks: Vec<(RemoteParticipant, Vec)>, + }, + Disconnected { + reason: &'static str, + }, + Reconnecting, + Reconnected, +} diff --git a/crates/livekit_client/src/livekit_client.rs b/crates/livekit_client/src/livekit_client.rs index c47cd48fa9..ddab660678 100644 --- a/crates/livekit_client/src/livekit_client.rs +++ b/crates/livekit_client/src/livekit_client.rs @@ -1,679 +1,497 @@ -#![cfg_attr(all(target_os = "windows", target_env = "gnu"), allow(unused))] +use std::sync::Arc; -mod remote_video_track_view; -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu") -))] -pub mod test; +use anyhow::Result; +use collections::HashMap; +use futures::{channel::mpsc, SinkExt}; +use gpui::{App, AsyncApp, ScreenCaptureSource, ScreenCaptureStream, Task}; +use gpui_tokio::Tokio; +use playback::capture_local_video_track; -use anyhow::{anyhow, Context as _, Result}; -use cpal::traits::{DeviceTrait, HostTrait, StreamTrait as _}; -use futures::{io, Stream, StreamExt as _}; -use gpui::{ - BackgroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task, -}; -use parking_lot::Mutex; -use std::{borrow::Cow, collections::VecDeque, future::Future, pin::Pin, sync::Arc, thread}; -use util::{debug_panic, ResultExt as _}; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -use webrtc::{ - audio_frame::AudioFrame, - audio_source::{native::NativeAudioSource, AudioSourceOptions, RtcAudioSource}, - audio_stream::native::NativeAudioStream, - video_frame::{VideoBuffer, VideoFrame, VideoRotation}, - video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, - video_stream::native::NativeVideoStream, -}; +mod playback; -#[cfg(all( - not(any(test, feature = "test-support")), - not(all(target_os = "windows", target_env = "gnu")) -))] -use livekit::track::RemoteAudioTrack; -#[cfg(all( - not(any(test, feature = "test-support")), - not(all(target_os = "windows", target_env = "gnu")) -))] -pub use livekit::*; -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu") -))] -use test::track::RemoteAudioTrack; -#[cfg(any( - test, - feature = "test-support", - all(target_os = "windows", target_env = "gnu") -))] -pub use test::*; +use crate::{LocalTrack, Participant, RemoteTrack, RoomEvent, TrackPublication}; +pub use playback::AudioStream; +pub(crate) use playback::{play_remote_video_track, RemoteVideoFrame}; -pub use remote_video_track_view::{RemoteVideoTrackView, RemoteVideoTrackViewEvent}; +#[derive(Clone, Debug)] +pub struct RemoteVideoTrack(livekit::track::RemoteVideoTrack); +#[derive(Clone, Debug)] +pub struct RemoteAudioTrack(livekit::track::RemoteAudioTrack); +#[derive(Clone, Debug)] +pub struct RemoteTrackPublication(livekit::publication::RemoteTrackPublication); +#[derive(Clone, Debug)] +pub struct RemoteParticipant(livekit::participant::RemoteParticipant); -pub enum AudioStream { - Input { - _thread_handle: std::sync::mpsc::Sender<()>, - _transmit_task: Task<()>, - }, - Output { - _task: Task<()>, - }, +#[derive(Clone, Debug)] +pub struct LocalVideoTrack(livekit::track::LocalVideoTrack); +#[derive(Clone, Debug)] +pub struct LocalAudioTrack(livekit::track::LocalAudioTrack); +#[derive(Clone, Debug)] +pub struct LocalTrackPublication(livekit::publication::LocalTrackPublication); +#[derive(Clone, Debug)] +pub struct LocalParticipant(livekit::participant::LocalParticipant); + +pub struct Room { + room: livekit::Room, + _task: Task<()>, + playback: playback::AudioStack, } -struct Dispatcher(Arc); +pub type TrackSid = livekit::id::TrackSid; +pub type ConnectionState = livekit::ConnectionState; +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct ParticipantIdentity(pub String); -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl livekit::dispatcher::Dispatcher for Dispatcher { - fn dispatch(&self, runnable: livekit::dispatcher::Runnable) { - self.0.dispatch(runnable, None); - } - - fn dispatch_after( - &self, - duration: std::time::Duration, - runnable: livekit::dispatcher::Runnable, - ) { - self.0.dispatch_after(duration, runnable); - } -} - -struct HttpClientAdapter(Arc); - -fn http_2_status(status: http_client::http::StatusCode) -> http_2::StatusCode { - http_2::StatusCode::from_u16(status.as_u16()) - .expect("valid status code to status code conversion") -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl livekit::dispatcher::HttpClient for HttpClientAdapter { - fn get( - &self, - url: &str, - ) -> Pin> + Send>> { - let http_client = self.0.clone(); - let url = url.to_string(); - Box::pin(async move { - let response = http_client - .get(&url, http_client::AsyncBody::empty(), false) - .await - .map_err(io::Error::other)?; - Ok(livekit::dispatcher::Response { - status: http_2_status(response.status()), - body: Box::pin(response.into_body()), - }) - }) - } - - fn send_async( - &self, - request: http_2::Request>, - ) -> Pin> + Send>> { - let http_client = self.0.clone(); - let mut builder = http_client::http::Request::builder() - .method(request.method().as_str()) - .uri(request.uri().to_string()); - - for (key, value) in request.headers().iter() { - builder = builder.header(key.as_str(), value.as_bytes()); - } - - if !request.extensions().is_empty() { - debug_panic!( - "Livekit sent an HTTP request with a protocol extension that Zed doesn't support!" - ); - } - - let request = builder - .body(http_client::AsyncBody::from_bytes( - request.into_body().into(), - )) - .unwrap(); - - Box::pin(async move { - let response = http_client.send(request).await.map_err(io::Error::other)?; - Ok(livekit::dispatcher::Response { - status: http_2_status(response.status()), - body: Box::pin(response.into_body()), - }) - }) - } -} - -#[cfg(all(target_os = "windows", target_env = "gnu"))] -pub fn init( - dispatcher: Arc, - http_client: Arc, -) { -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub fn init( - dispatcher: Arc, - http_client: Arc, -) { - livekit::dispatcher::set_dispatcher(Dispatcher(dispatcher)); - livekit::dispatcher::set_http_client(HttpClientAdapter(http_client)); -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub async fn capture_local_video_track( - capture_source: &dyn ScreenCaptureSource, -) -> Result<(track::LocalVideoTrack, Box)> { - let resolution = capture_source.resolution()?; - let track_source = NativeVideoSource::new(VideoResolution { - width: resolution.width.0 as u32, - height: resolution.height.0 as u32, - }); - - let capture_stream = capture_source - .stream({ - let track_source = track_source.clone(); - Box::new(move |frame| { - if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { - track_source.capture_frame(&VideoFrame { - rotation: VideoRotation::VideoRotation0, - timestamp_us: 0, - buffer, - }); - } - }) - }) +impl Room { + pub async fn connect( + url: String, + token: String, + cx: &mut AsyncApp, + ) -> Result<(Self, mpsc::UnboundedReceiver)> { + let connector = + tokio_tungstenite::Connector::Rustls(Arc::new(http_client_tls::tls_config())); + let mut config = livekit::RoomOptions::default(); + config.connector = Some(connector); + let (room, mut events) = Tokio::spawn(cx, async move { + livekit::Room::connect(&url, &token, config).await + })? .await??; - Ok(( - track::LocalVideoTrack::create_video_track( - "screen share", - RtcVideoSource::Native(track_source), - ), - capture_stream, - )) -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub fn capture_local_audio_track( - background_executor: &BackgroundExecutor, -) -> Result> { - use util::maybe; - - let (frame_tx, mut frame_rx) = futures::channel::mpsc::unbounded(); - let (thread_handle, thread_kill_rx) = std::sync::mpsc::channel::<()>(); - let sample_rate; - let channels; - - if cfg!(any(test, feature = "test-support")) { - sample_rate = 2; - channels = 1; - } else { - let (device, config) = default_device(true)?; - sample_rate = config.sample_rate().0; - channels = config.channels() as u32; - thread::spawn(move || { - maybe!({ - if let Some(name) = device.name().ok() { - log::info!("Using microphone: {}", name) - } else { - log::info!("Using microphone: "); + let (mut tx, rx) = mpsc::unbounded(); + let task = cx.background_executor().spawn(async move { + while let Some(event) = events.recv().await { + if let Some(event) = room_event_from_livekit(event) { + tx.send(event.into()).await.ok(); } - - let stream = device - .build_input_stream_raw( - &config.config(), - cpal::SampleFormat::I16, - move |data, _: &_| { - frame_tx - .unbounded_send(AudioFrame { - data: Cow::Owned(data.as_slice::().unwrap().to_vec()), - sample_rate, - num_channels: channels, - samples_per_channel: data.len() as u32 / channels, - }) - .ok(); - }, - |err| log::error!("error capturing audio track: {:?}", err), - None, - ) - .context("failed to build input stream")?; - - stream.play()?; - // Keep the thread alive and holding onto the `stream` - thread_kill_rx.recv().ok(); - anyhow::Ok(Some(())) - }) - .log_err(); - }); - } - - Ok(background_executor.spawn({ - let background_executor = background_executor.clone(); - async move { - let source = NativeAudioSource::new( - AudioSourceOptions { - echo_cancellation: true, - noise_suppression: true, - auto_gain_control: true, - }, - sample_rate, - channels, - 100, - ); - let transmit_task = background_executor.spawn({ - let source = source.clone(); - async move { - while let Some(frame) = frame_rx.next().await { - source.capture_frame(&frame).await.log_err(); - } - } - }); - - let track = track::LocalAudioTrack::create_audio_track( - "microphone", - RtcAudioSource::Native(source), - ); - - ( - track, - AudioStream::Input { - _thread_handle: thread_handle, - _transmit_task: transmit_task, - }, - ) - } - })) -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub fn play_remote_audio_track( - track: &RemoteAudioTrack, - background_executor: &BackgroundExecutor, -) -> Result { - let track = track.clone(); - // We track device changes in our output because Livekit has a resampler built in, - // and it's easy to create a new native audio stream when the device changes. - if cfg!(any(test, feature = "test-support")) { - Ok(AudioStream::Output { - _task: background_executor.spawn(async {}), - }) - } else { - let mut default_change_listener = DeviceChangeListener::new(false)?; - let (output_device, output_config) = default_device(false)?; - - let _task = background_executor.spawn({ - let background_executor = background_executor.clone(); - async move { - let (mut _receive_task, mut _thread) = - start_output_stream(output_config, output_device, &track, &background_executor); - - while let Some(_) = default_change_listener.next().await { - let Some((output_device, output_config)) = get_default_output().log_err() - else { - continue; - }; - - if let Ok(name) = output_device.name() { - log::info!("Using speaker: {}", name) - } else { - log::info!("Using speaker: ") - } - - (_receive_task, _thread) = start_output_stream( - output_config, - output_device, - &track, - &background_executor, - ); - } - - futures::future::pending::<()>().await; } }); - Ok(AudioStream::Output { _task }) - } -} - -fn default_device(input: bool) -> anyhow::Result<(cpal::Device, cpal::SupportedStreamConfig)> { - let device; - let config; - if input { - device = cpal::default_host() - .default_input_device() - .ok_or_else(|| anyhow!("no audio input device available"))?; - config = device - .default_input_config() - .context("failed to get default input config")?; - } else { - device = cpal::default_host() - .default_output_device() - .ok_or_else(|| anyhow!("no audio output device available"))?; - config = device - .default_output_config() - .context("failed to get default output config")?; - } - Ok((device, config)) -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -fn get_default_output() -> anyhow::Result<(cpal::Device, cpal::SupportedStreamConfig)> { - let host = cpal::default_host(); - let output_device = host - .default_output_device() - .context("failed to read default output device")?; - let output_config = output_device.default_output_config()?; - Ok((output_device, output_config)) -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -fn start_output_stream( - output_config: cpal::SupportedStreamConfig, - output_device: cpal::Device, - track: &track::RemoteAudioTrack, - background_executor: &BackgroundExecutor, -) -> (Task<()>, std::sync::mpsc::Sender<()>) { - let buffer = Arc::new(Mutex::new(VecDeque::::new())); - let sample_rate = output_config.sample_rate(); - - let mut stream = NativeAudioStream::new( - track.rtc_track(), - sample_rate.0 as i32, - output_config.channels() as i32, - ); - - let receive_task = background_executor.spawn({ - let buffer = buffer.clone(); - async move { - const MS_OF_BUFFER: u32 = 100; - const MS_IN_SEC: u32 = 1000; - while let Some(frame) = stream.next().await { - let frame_size = frame.samples_per_channel * frame.num_channels; - debug_assert!(frame.data.len() == frame_size as usize); - - let buffer_size = - ((frame.sample_rate * frame.num_channels) / MS_IN_SEC * MS_OF_BUFFER) as usize; - - let mut buffer = buffer.lock(); - let new_size = buffer.len() + frame.data.len(); - if new_size > buffer_size { - let overflow = new_size - buffer_size; - buffer.drain(0..overflow); - } - - buffer.extend(frame.data.iter()); - } - } - }); - - // The _output_stream needs to be on it's own thread because it's !Send - // and we experienced a deadlock when it's created on the main thread. - let (thread, end_on_drop_rx) = std::sync::mpsc::channel::<()>(); - thread::spawn(move || { - if cfg!(any(test, feature = "test-support")) { - // Can't play audio in tests - return; - } - - let output_stream = output_device.build_output_stream( - &output_config.config(), - { - let buffer = buffer.clone(); - move |data, _info| { - let mut buffer = buffer.lock(); - if buffer.len() < data.len() { - // Instead of partially filling a buffer, output silence. If a partial - // buffer was outputted then this could lead to a perpetual state of - // outputting partial buffers as it never gets filled enough for a full - // frame. - data.fill(0); - } else { - // SAFETY: We know that buffer has at least data.len() values in it. - // because we just checked - let mut drain = buffer.drain(..data.len()); - data.fill_with(|| unsafe { drain.next().unwrap_unchecked() }); - } - } + Ok(( + Self { + room, + _task: task, + playback: playback::AudioStack::new(cx.background_executor().clone()), }, - |error| log::error!("error playing audio track: {:?}", error), - None, - ); - - let Some(output_stream) = output_stream.log_err() else { - return; - }; - - output_stream.play().log_err(); - // Block forever to keep the output stream alive - end_on_drop_rx.recv().ok(); - }); - - (receive_task, thread) -} - -#[cfg(all(target_os = "windows", target_env = "gnu"))] -pub fn play_remote_video_track( - track: &track::RemoteVideoTrack, -) -> impl Stream { - futures::stream::empty() -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub fn play_remote_video_track( - track: &track::RemoteVideoTrack, -) -> impl Stream { - NativeVideoStream::new(track.rtc_track()) - .filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) }) -} - -#[cfg(target_os = "macos")] -pub type RemoteVideoFrame = media::core_video::CVImageBuffer; - -#[cfg(target_os = "macos")] -fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { - use core_foundation::base::TCFType as _; - use media::core_video::CVImageBuffer; - - let buffer = buffer.as_native()?; - let pixel_buffer = buffer.get_cv_pixel_buffer(); - if pixel_buffer.is_null() { - return None; + rx, + )) } - unsafe { Some(CVImageBuffer::wrap_under_get_rule(pixel_buffer as _)) } + pub fn local_participant(&self) -> LocalParticipant { + LocalParticipant(self.room.local_participant()) + } + + pub fn remote_participants(&self) -> HashMap { + self.room + .remote_participants() + .into_iter() + .map(|(k, v)| (ParticipantIdentity(k.0), RemoteParticipant(v))) + .collect() + } + + pub fn connection_state(&self) -> ConnectionState { + self.room.connection_state() + } + + pub async fn publish_local_microphone_track( + &self, + cx: &mut AsyncApp, + ) -> Result<(LocalTrackPublication, playback::AudioStream)> { + let (track, stream) = self.playback.capture_local_microphone_track()?; + let publication = self + .local_participant() + .publish_track( + livekit::track::LocalTrack::Audio(track.0), + livekit::options::TrackPublishOptions { + source: livekit::track::TrackSource::Microphone, + ..Default::default() + }, + cx, + ) + .await?; + + Ok((publication, stream)) + } + + // pub async fn publish_local_wav_track( + // &self, + // cx: &mut AsyncApp, + // ) -> Result<(LocalTrackPublication, playback::AudioStream)> { + // let apm = self.apm.clone(); + // let executor = cx.background_executor().clone(); + // let (track, stream) = + // Tokio::spawn( + // cx, + // async move { capture_local_wav_track(apm, &executor).await }, + // )? + // .await??; + // let publication = self + // .local_participant() + // .publish_track( + // livekit::track::LocalTrack::Audio(track.0), + // livekit::options::TrackPublishOptions { + // source: livekit::track::TrackSource::Microphone, + // ..Default::default() + // }, + // cx, + // ) + // .await?; + + // Ok((publication, stream)) + // } + + pub async fn unpublish_local_track( + &self, + sid: TrackSid, + cx: &mut AsyncApp, + ) -> Result { + self.local_participant().unpublish_track(sid, cx).await + } + + pub fn play_remote_audio_track( + &self, + track: &RemoteAudioTrack, + _cx: &App, + ) -> Result { + Ok(self.playback.play_remote_audio_track(&track.0)) + } } -#[cfg(not(target_os = "macos"))] -pub type RemoteVideoFrame = Arc; +impl LocalParticipant { + pub async fn publish_screenshare_track( + &self, + source: &dyn ScreenCaptureSource, + cx: &mut AsyncApp, + ) -> Result<(LocalTrackPublication, Box)> { + let (track, stream) = capture_local_video_track(&*source, cx).await?; + let options = livekit::options::TrackPublishOptions { + source: livekit::track::TrackSource::Screenshare, + video_codec: livekit::options::VideoCodec::VP8, + ..Default::default() + }; + let publication = self + .publish_track(livekit::track::LocalTrack::Video(track.0), options, cx) + .await?; -#[cfg(not(any(target_os = "macos", all(target_os = "windows", target_env = "gnu"))))] -fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { - use gpui::RenderImage; - use image::{Frame, RgbaImage}; - use livekit::webrtc::prelude::VideoFormatType; - use smallvec::SmallVec; - use std::alloc::{alloc, Layout}; + Ok((publication, stream)) + } - let width = buffer.width(); - let height = buffer.height(); - let stride = width * 4; - let byte_len = (stride * height) as usize; - let argb_image = unsafe { - // Motivation for this unsafe code is to avoid initializing the frame data, since to_argb - // will write all bytes anyway. - let start_ptr = alloc(Layout::array::(byte_len).log_err()?); - if start_ptr.is_null() { + async fn publish_track( + &self, + track: livekit::track::LocalTrack, + options: livekit::options::TrackPublishOptions, + cx: &mut AsyncApp, + ) -> Result { + let participant = self.0.clone(); + Tokio::spawn(cx, async move { + participant.publish_track(track, options).await + })? + .await? + .map(|p| LocalTrackPublication(p)) + .map_err(|error| anyhow::anyhow!("failed to publish track: {error}")) + } + + pub async fn unpublish_track( + &self, + sid: TrackSid, + cx: &mut AsyncApp, + ) -> Result { + let participant = self.0.clone(); + Tokio::spawn(cx, async move { participant.unpublish_track(&sid).await })? + .await? + .map(|p| LocalTrackPublication(p)) + .map_err(|error| anyhow::anyhow!("failed to unpublish track: {error}")) + } +} + +impl LocalTrackPublication { + pub fn mute(&self, cx: &App) { + let track = self.0.clone(); + Tokio::spawn(cx, async move { + track.mute(); + }) + .detach(); + } + + pub fn unmute(&self, cx: &App) { + let track = self.0.clone(); + Tokio::spawn(cx, async move { + track.unmute(); + }) + .detach(); + } + + pub fn sid(&self) -> TrackSid { + self.0.sid() + } + + pub fn is_muted(&self) -> bool { + self.0.is_muted() + } +} + +impl RemoteParticipant { + pub fn identity(&self) -> ParticipantIdentity { + ParticipantIdentity(self.0.identity().0) + } + + pub fn track_publications(&self) -> HashMap { + self.0 + .track_publications() + .into_iter() + .map(|(sid, publication)| (sid, RemoteTrackPublication(publication))) + .collect() + } +} + +impl RemoteAudioTrack { + pub fn sid(&self) -> TrackSid { + self.0.sid() + } +} + +impl RemoteVideoTrack { + pub fn sid(&self) -> TrackSid { + self.0.sid() + } +} + +impl RemoteTrackPublication { + pub fn is_muted(&self) -> bool { + self.0.is_muted() + } + + pub fn is_enabled(&self) -> bool { + self.0.is_enabled() + } + + pub fn track(&self) -> Option { + self.0.track().map(remote_track_from_livekit) + } + + pub fn is_audio(&self) -> bool { + self.0.kind() == livekit::track::TrackKind::Audio + } + + pub fn set_enabled(&self, enabled: bool, cx: &App) { + let track = self.0.clone(); + Tokio::spawn(cx, async move { track.set_enabled(enabled) }).detach(); + } + + pub fn sid(&self) -> TrackSid { + self.0.sid() + } +} + +impl RemoteTrack { + pub fn set_enabled(&self, enabled: bool, cx: &App) { + let this = self.clone(); + Tokio::spawn(cx, async move { + match this { + RemoteTrack::Audio(remote_audio_track) => { + remote_audio_track.0.rtc_track().set_enabled(enabled) + } + RemoteTrack::Video(remote_video_track) => { + remote_video_track.0.rtc_track().set_enabled(enabled) + } + } + }) + .detach(); + } +} + +impl Participant { + pub fn identity(&self) -> ParticipantIdentity { + match self { + Participant::Local(local_participant) => { + ParticipantIdentity(local_participant.0.identity().0) + } + Participant::Remote(remote_participant) => { + ParticipantIdentity(remote_participant.0.identity().0) + } + } + } +} + +fn participant_from_livekit(participant: livekit::participant::Participant) -> Participant { + match participant { + livekit::participant::Participant::Local(local) => { + Participant::Local(LocalParticipant(local)) + } + livekit::participant::Participant::Remote(remote) => { + Participant::Remote(RemoteParticipant(remote)) + } + } +} + +fn publication_from_livekit( + publication: livekit::publication::TrackPublication, +) -> TrackPublication { + match publication { + livekit::publication::TrackPublication::Local(local) => { + TrackPublication::Local(LocalTrackPublication(local)) + } + livekit::publication::TrackPublication::Remote(remote) => { + TrackPublication::Remote(RemoteTrackPublication(remote)) + } + } +} + +fn remote_track_from_livekit(track: livekit::track::RemoteTrack) -> RemoteTrack { + match track { + livekit::track::RemoteTrack::Audio(audio) => RemoteTrack::Audio(RemoteAudioTrack(audio)), + livekit::track::RemoteTrack::Video(video) => RemoteTrack::Video(RemoteVideoTrack(video)), + } +} + +fn local_track_from_livekit(track: livekit::track::LocalTrack) -> LocalTrack { + match track { + livekit::track::LocalTrack::Audio(audio) => LocalTrack::Audio(LocalAudioTrack(audio)), + livekit::track::LocalTrack::Video(video) => LocalTrack::Video(LocalVideoTrack(video)), + } +} +fn room_event_from_livekit(event: livekit::RoomEvent) -> Option { + let event = match event { + livekit::RoomEvent::ParticipantConnected(remote_participant) => { + RoomEvent::ParticipantConnected(RemoteParticipant(remote_participant)) + } + livekit::RoomEvent::ParticipantDisconnected(remote_participant) => { + RoomEvent::ParticipantDisconnected(RemoteParticipant(remote_participant)) + } + livekit::RoomEvent::LocalTrackPublished { + publication, + track, + participant, + } => RoomEvent::LocalTrackPublished { + publication: LocalTrackPublication(publication), + track: local_track_from_livekit(track), + participant: LocalParticipant(participant), + }, + livekit::RoomEvent::LocalTrackUnpublished { + publication, + participant, + } => RoomEvent::LocalTrackUnpublished { + publication: LocalTrackPublication(publication), + participant: LocalParticipant(participant), + }, + livekit::RoomEvent::LocalTrackSubscribed { track } => RoomEvent::LocalTrackSubscribed { + track: local_track_from_livekit(track), + }, + livekit::RoomEvent::TrackSubscribed { + track, + publication, + participant, + } => RoomEvent::TrackSubscribed { + track: remote_track_from_livekit(track), + publication: RemoteTrackPublication(publication), + participant: RemoteParticipant(participant), + }, + livekit::RoomEvent::TrackUnsubscribed { + track, + publication, + participant, + } => RoomEvent::TrackUnsubscribed { + track: remote_track_from_livekit(track), + publication: RemoteTrackPublication(publication), + participant: RemoteParticipant(participant), + }, + livekit::RoomEvent::TrackSubscriptionFailed { + participant, + error: _, + track_sid, + } => RoomEvent::TrackSubscriptionFailed { + participant: RemoteParticipant(participant), + track_sid, + }, + livekit::RoomEvent::TrackPublished { + publication, + participant, + } => RoomEvent::TrackPublished { + publication: RemoteTrackPublication(publication), + participant: RemoteParticipant(participant), + }, + livekit::RoomEvent::TrackUnpublished { + publication, + participant, + } => RoomEvent::TrackUnpublished { + publication: RemoteTrackPublication(publication), + participant: RemoteParticipant(participant), + }, + livekit::RoomEvent::TrackMuted { + participant, + publication, + } => RoomEvent::TrackMuted { + publication: publication_from_livekit(publication), + participant: participant_from_livekit(participant), + }, + livekit::RoomEvent::TrackUnmuted { + participant, + publication, + } => RoomEvent::TrackUnmuted { + publication: publication_from_livekit(publication), + participant: participant_from_livekit(participant), + }, + livekit::RoomEvent::RoomMetadataChanged { + old_metadata, + metadata, + } => RoomEvent::RoomMetadataChanged { + old_metadata, + metadata, + }, + livekit::RoomEvent::ParticipantMetadataChanged { + participant, + old_metadata, + metadata, + } => RoomEvent::ParticipantMetadataChanged { + participant: participant_from_livekit(participant), + old_metadata, + metadata, + }, + livekit::RoomEvent::ParticipantNameChanged { + participant, + old_name, + name, + } => RoomEvent::ParticipantNameChanged { + participant: participant_from_livekit(participant), + old_name, + name, + }, + livekit::RoomEvent::ParticipantAttributesChanged { + participant, + changed_attributes, + } => RoomEvent::ParticipantAttributesChanged { + participant: participant_from_livekit(participant), + changed_attributes: changed_attributes.into_iter().collect(), + }, + livekit::RoomEvent::ActiveSpeakersChanged { speakers } => { + RoomEvent::ActiveSpeakersChanged { + speakers: speakers.into_iter().map(participant_from_livekit).collect(), + } + } + livekit::RoomEvent::Connected { + participants_with_tracks, + } => RoomEvent::Connected { + participants_with_tracks: participants_with_tracks + .into_iter() + .map({ + |(p, t)| { + ( + RemoteParticipant(p), + t.into_iter().map(|t| RemoteTrackPublication(t)).collect(), + ) + } + }) + .collect(), + }, + livekit::RoomEvent::Disconnected { reason } => RoomEvent::Disconnected { + reason: reason.as_str_name(), + }, + livekit::RoomEvent::Reconnecting => RoomEvent::Reconnecting, + livekit::RoomEvent::Reconnected => RoomEvent::Reconnected, + _ => { + log::trace!("dropping livekit event: {:?}", event); return None; } - let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len); - buffer.to_argb( - VideoFormatType::ARGB, // For some reason, this displays correctly while RGBA (the correct format) does not - bgra_frame_slice, - stride, - width as i32, - height as i32, - ); - Vec::from_raw_parts(start_ptr, byte_len, byte_len) }; - Some(Arc::new(RenderImage::new(SmallVec::from_elem( - Frame::new( - RgbaImage::from_raw(width, height, argb_image) - .with_context(|| "Bug: not enough bytes allocated for image.") - .log_err()?, - ), - 1, - )))) + Some(event) } - -#[cfg(target_os = "macos")] -fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option> { - use core_foundation::base::TCFType as _; - - let pixel_buffer = frame.0.as_concrete_TypeRef(); - std::mem::forget(frame.0); - unsafe { - Some(webrtc::video_frame::native::NativeBuffer::from_cv_pixel_buffer(pixel_buffer as _)) - } -} - -#[cfg(not(any(target_os = "macos", all(target_os = "windows", target_env = "gnu"))))] -fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { - None as Option> -} - -trait DeviceChangeListenerApi: Stream + Sized { - fn new(input: bool) -> Result; -} - -#[cfg(target_os = "macos")] -mod macos { - - use coreaudio::sys::{ - kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice, - kAudioObjectPropertyElementMaster, kAudioObjectPropertyScopeGlobal, - kAudioObjectSystemObject, AudioObjectAddPropertyListener, AudioObjectID, - AudioObjectPropertyAddress, AudioObjectRemovePropertyListener, OSStatus, - }; - use futures::{channel::mpsc::UnboundedReceiver, StreamExt}; - - use crate::DeviceChangeListenerApi; - - /// Implementation from: https://github.com/zed-industries/cpal/blob/fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50/src/host/coreaudio/macos/property_listener.rs#L15 - pub struct CoreAudioDefaultDeviceChangeListener { - rx: UnboundedReceiver<()>, - callback: Box, - input: bool, - } - - trait _AssertSend: Send {} - impl _AssertSend for CoreAudioDefaultDeviceChangeListener {} - - struct PropertyListenerCallbackWrapper(Box); - - unsafe extern "C" fn property_listener_handler_shim( - _: AudioObjectID, - _: u32, - _: *const AudioObjectPropertyAddress, - callback: *mut ::std::os::raw::c_void, - ) -> OSStatus { - let wrapper = callback as *mut PropertyListenerCallbackWrapper; - (*wrapper).0(); - 0 - } - - impl DeviceChangeListenerApi for CoreAudioDefaultDeviceChangeListener { - fn new(input: bool) -> gpui::Result { - let (tx, rx) = futures::channel::mpsc::unbounded(); - - let callback = Box::new(PropertyListenerCallbackWrapper(Box::new(move || { - tx.unbounded_send(()).ok(); - }))); - - unsafe { - coreaudio::Error::from_os_status(AudioObjectAddPropertyListener( - kAudioObjectSystemObject, - &AudioObjectPropertyAddress { - mSelector: if input { - kAudioHardwarePropertyDefaultInputDevice - } else { - kAudioHardwarePropertyDefaultOutputDevice - }, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMaster, - }, - Some(property_listener_handler_shim), - &*callback as *const _ as *mut _, - ))?; - } - - Ok(Self { - rx, - callback, - input, - }) - } - } - - impl Drop for CoreAudioDefaultDeviceChangeListener { - fn drop(&mut self) { - unsafe { - AudioObjectRemovePropertyListener( - kAudioObjectSystemObject, - &AudioObjectPropertyAddress { - mSelector: if self.input { - kAudioHardwarePropertyDefaultInputDevice - } else { - kAudioHardwarePropertyDefaultOutputDevice - }, - mScope: kAudioObjectPropertyScopeGlobal, - mElement: kAudioObjectPropertyElementMaster, - }, - Some(property_listener_handler_shim), - &*self.callback as *const _ as *mut _, - ); - } - } - } - - impl futures::Stream for CoreAudioDefaultDeviceChangeListener { - type Item = (); - - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.rx.poll_next_unpin(cx) - } - } -} - -#[cfg(target_os = "macos")] -type DeviceChangeListener = macos::CoreAudioDefaultDeviceChangeListener; - -#[cfg(not(target_os = "macos"))] -mod noop_change_listener { - use std::task::Poll; - - use crate::DeviceChangeListenerApi; - - pub struct NoopOutputDeviceChangelistener {} - - impl DeviceChangeListenerApi for NoopOutputDeviceChangelistener { - fn new(_input: bool) -> anyhow::Result { - Ok(NoopOutputDeviceChangelistener {}) - } - } - - impl futures::Stream for NoopOutputDeviceChangelistener { - type Item = (); - - fn poll_next( - self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - ) -> Poll> { - Poll::Pending - } - } -} - -#[cfg(not(target_os = "macos"))] -type DeviceChangeListener = noop_change_listener::NoopOutputDeviceChangelistener; diff --git a/crates/livekit_client/src/livekit_client/playback.rs b/crates/livekit_client/src/livekit_client/playback.rs new file mode 100644 index 0000000000..480b499757 --- /dev/null +++ b/crates/livekit_client/src/livekit_client/playback.rs @@ -0,0 +1,763 @@ +use anyhow::{anyhow, Context as _, Result}; + +use cpal::traits::{DeviceTrait, HostTrait, StreamTrait as _}; +use futures::channel::mpsc::UnboundedSender; +use futures::{Stream, StreamExt as _}; +use gpui::{ + BackgroundExecutor, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task, +}; +use libwebrtc::native::{apm, audio_mixer, audio_resampler}; +use livekit::track; + +use livekit::webrtc::{ + audio_frame::AudioFrame, + audio_source::{native::NativeAudioSource, AudioSourceOptions, RtcAudioSource}, + audio_stream::native::NativeAudioStream, + video_frame::{VideoBuffer, VideoFrame, VideoRotation}, + video_source::{native::NativeVideoSource, RtcVideoSource, VideoResolution}, + video_stream::native::NativeVideoStream, +}; +use parking_lot::Mutex; +use std::cell::RefCell; +use std::sync::atomic::{self, AtomicI32}; +use std::sync::Weak; +use std::time::Duration; +use std::{borrow::Cow, collections::VecDeque, sync::Arc, thread}; +use util::{maybe, ResultExt as _}; + +pub(crate) struct AudioStack { + executor: BackgroundExecutor, + apm: Arc>, + mixer: Arc>, + _output_task: RefCell>>, + next_ssrc: AtomicI32, +} + +// NOTE: We use WebRTC's mixer which only supports +// 16kHz, 32kHz and 48kHz. As 48 is the most common "next step up" +// for audio output devices like speakers/bluetooth, we just hard-code +// this; and downsample when we need to. +const SAMPLE_RATE: u32 = 48000; +const NUM_CHANNELS: u32 = 2; + +impl AudioStack { + pub(crate) fn new(executor: BackgroundExecutor) -> Self { + let apm = Arc::new(Mutex::new(apm::AudioProcessingModule::new( + true, true, true, true, + ))); + let mixer = Arc::new(Mutex::new(audio_mixer::AudioMixer::new())); + Self { + executor, + apm, + mixer, + _output_task: RefCell::new(Weak::new()), + next_ssrc: AtomicI32::new(1), + } + } + + pub(crate) fn play_remote_audio_track( + &self, + track: &livekit::track::RemoteAudioTrack, + ) -> AudioStream { + let output_task = self.start_output(); + + let next_ssrc = self.next_ssrc.fetch_add(1, atomic::Ordering::Relaxed); + let source = AudioMixerSource { + ssrc: next_ssrc, + sample_rate: SAMPLE_RATE, + num_channels: NUM_CHANNELS, + buffer: Arc::default(), + }; + self.mixer.lock().add_source(source.clone()); + + let mut stream = NativeAudioStream::new( + track.rtc_track(), + source.sample_rate as i32, + source.num_channels as i32, + ); + + let receive_task = self.executor.spawn({ + let source = source.clone(); + async move { + while let Some(frame) = stream.next().await { + source.receive(frame); + } + } + }); + + let mixer = self.mixer.clone(); + let on_drop = util::defer(move || { + mixer.lock().remove_source(source.ssrc); + drop(receive_task); + drop(output_task); + }); + + AudioStream::Output { + _drop: Box::new(on_drop), + } + } + + pub(crate) fn capture_local_microphone_track( + &self, + ) -> Result<(crate::LocalAudioTrack, AudioStream)> { + let source = NativeAudioSource::new( + // n.b. this struct's options are always ignored, noise cancellation is provided by apm. + AudioSourceOptions::default(), + SAMPLE_RATE, + NUM_CHANNELS, + 10, + ); + + let track = track::LocalAudioTrack::create_audio_track( + "microphone", + RtcAudioSource::Native(source.clone()), + ); + + let apm = self.apm.clone(); + + let (frame_tx, mut frame_rx) = futures::channel::mpsc::unbounded(); + let transmit_task = self.executor.spawn({ + let source = source.clone(); + async move { + while let Some(frame) = frame_rx.next().await { + source.capture_frame(&frame).await.log_err(); + } + } + }); + let capture_task = self.executor.spawn(async move { + Self::capture_input(apm, frame_tx, SAMPLE_RATE, NUM_CHANNELS).await + }); + + let on_drop = util::defer(|| { + drop(transmit_task); + drop(capture_task); + }); + return Ok(( + super::LocalAudioTrack(track), + AudioStream::Output { + _drop: Box::new(on_drop), + }, + )); + } + + fn start_output(&self) -> Arc> { + if let Some(task) = self._output_task.borrow().upgrade() { + return task; + } + let task = Arc::new(self.executor.spawn({ + let apm = self.apm.clone(); + let mixer = self.mixer.clone(); + async move { + Self::play_output(apm, mixer, SAMPLE_RATE, NUM_CHANNELS) + .await + .log_err(); + } + })); + *self._output_task.borrow_mut() = Arc::downgrade(&task); + task + } + + async fn play_output( + apm: Arc>, + mixer: Arc>, + sample_rate: u32, + num_channels: u32, + ) -> Result<()> { + let mut default_change_listener = DeviceChangeListener::new(false)?; + + loop { + let (output_device, output_config) = default_device(false)?; + let (end_on_drop_tx, end_on_drop_rx) = std::sync::mpsc::channel::<()>(); + let mixer = mixer.clone(); + let apm = apm.clone(); + let mut resampler = audio_resampler::AudioResampler::default(); + let mut buf = Vec::new(); + + thread::spawn(move || { + let output_stream = output_device.build_output_stream( + &output_config.config(), + { + move |mut data, _info| { + while data.len() > 0 { + if data.len() <= buf.len() { + let rest = buf.split_off(data.len()); + data.copy_from_slice(&buf); + buf = rest; + return; + } + if buf.len() > 0 { + let (prefix, suffix) = data.split_at_mut(buf.len()); + prefix.copy_from_slice(&buf); + data = suffix; + } + + let mut mixer = mixer.lock(); + let mixed = mixer.mix(output_config.channels() as usize); + let sampled = resampler.remix_and_resample( + mixed, + sample_rate / 100, + num_channels, + sample_rate, + output_config.channels() as u32, + output_config.sample_rate().0, + ); + buf = sampled.to_vec(); + apm.lock() + .process_reverse_stream( + &mut buf, + output_config.sample_rate().0 as i32, + output_config.channels() as i32, + ) + .ok(); + } + } + }, + |error| log::error!("error playing audio track: {:?}", error), + Some(Duration::from_millis(100)), + ); + + let Some(output_stream) = output_stream.log_err() else { + return; + }; + + output_stream.play().log_err(); + // Block forever to keep the output stream alive + end_on_drop_rx.recv().ok(); + }); + + default_change_listener.next().await; + drop(end_on_drop_tx) + } + } + + async fn capture_input( + apm: Arc>, + frame_tx: UnboundedSender>, + sample_rate: u32, + num_channels: u32, + ) -> Result<()> { + let mut default_change_listener = DeviceChangeListener::new(true)?; + loop { + let (device, config) = default_device(true)?; + let (end_on_drop_tx, end_on_drop_rx) = std::sync::mpsc::channel::<()>(); + let apm = apm.clone(); + let frame_tx = frame_tx.clone(); + let mut resampler = audio_resampler::AudioResampler::default(); + + thread::spawn(move || { + maybe!({ + if let Some(name) = device.name().ok() { + log::info!("Using microphone: {}", name) + } else { + log::info!("Using microphone: "); + } + + let ten_ms_buffer_size = + (config.channels() as u32 * config.sample_rate().0 / 100) as usize; + let mut buf: Vec = Vec::with_capacity(ten_ms_buffer_size); + + let stream = device + .build_input_stream_raw( + &config.config(), + cpal::SampleFormat::I16, + move |data, _: &_| { + let mut data = data.as_slice::().unwrap(); + while data.len() > 0 { + let remainder = (buf.capacity() - buf.len()).min(data.len()); + buf.extend_from_slice(&data[..remainder]); + data = &data[remainder..]; + + if buf.capacity() == buf.len() { + let mut sampled = resampler + .remix_and_resample( + buf.as_slice(), + config.sample_rate().0 as u32 / 100, + config.channels() as u32, + config.sample_rate().0 as u32, + num_channels, + sample_rate, + ) + .to_owned(); + apm.lock() + .process_stream( + &mut sampled, + sample_rate as i32, + num_channels as i32, + ) + .log_err(); + buf.clear(); + frame_tx + .unbounded_send(AudioFrame { + data: Cow::Owned(sampled), + sample_rate, + num_channels, + samples_per_channel: sample_rate / 100, + }) + .ok(); + } + } + }, + |err| log::error!("error capturing audio track: {:?}", err), + Some(Duration::from_millis(100)), + ) + .context("failed to build input stream")?; + + stream.play()?; + // Keep the thread alive and holding onto the `stream` + end_on_drop_rx.recv().ok(); + anyhow::Ok(Some(())) + }) + .log_err(); + }); + + default_change_listener.next().await; + drop(end_on_drop_tx) + } + } +} + +use super::LocalVideoTrack; + +pub enum AudioStream { + Input { _task: Task<()> }, + Output { _drop: Box }, +} + +pub(crate) async fn capture_local_video_track( + capture_source: &dyn ScreenCaptureSource, + cx: &mut gpui::AsyncApp, +) -> Result<(crate::LocalVideoTrack, Box)> { + let resolution = capture_source.resolution()?; + let track_source = gpui_tokio::Tokio::spawn(cx, async move { + NativeVideoSource::new(VideoResolution { + width: resolution.width.0 as u32, + height: resolution.height.0 as u32, + }) + })? + .await?; + + let capture_stream = capture_source + .stream({ + let track_source = track_source.clone(); + Box::new(move |frame| { + if let Some(buffer) = video_frame_buffer_to_webrtc(frame) { + track_source.capture_frame(&VideoFrame { + rotation: VideoRotation::VideoRotation0, + timestamp_us: 0, + buffer, + }); + } + }) + }) + .await??; + + Ok(( + LocalVideoTrack(track::LocalVideoTrack::create_video_track( + "screen share", + RtcVideoSource::Native(track_source), + )), + capture_stream, + )) +} + +fn default_device(input: bool) -> Result<(cpal::Device, cpal::SupportedStreamConfig)> { + let device; + let config; + if input { + device = cpal::default_host() + .default_input_device() + .ok_or_else(|| anyhow!("no audio input device available"))?; + config = device + .default_input_config() + .context("failed to get default input config")?; + } else { + device = cpal::default_host() + .default_output_device() + .ok_or_else(|| anyhow!("no audio output device available"))?; + config = device + .default_output_config() + .context("failed to get default output config")?; + } + Ok((device, config)) +} + +#[derive(Clone)] +struct AudioMixerSource { + ssrc: i32, + sample_rate: u32, + num_channels: u32, + buffer: Arc>>>, +} + +impl AudioMixerSource { + fn receive(&self, frame: AudioFrame) { + assert_eq!( + frame.data.len() as u32, + self.sample_rate * self.num_channels / 100 + ); + + let mut buffer = self.buffer.lock(); + buffer.push_back(frame.data.to_vec()); + while buffer.len() > 10 { + buffer.pop_front(); + } + } +} + +impl libwebrtc::native::audio_mixer::AudioMixerSource for AudioMixerSource { + fn ssrc(&self) -> i32 { + self.ssrc + } + + fn preferred_sample_rate(&self) -> u32 { + self.sample_rate + } + + fn get_audio_frame_with_info<'a>(&self, target_sample_rate: u32) -> Option { + assert_eq!(self.sample_rate, target_sample_rate); + let buf = self.buffer.lock().pop_front()?; + Some(AudioFrame { + data: Cow::Owned(buf), + sample_rate: self.sample_rate, + num_channels: self.num_channels, + samples_per_channel: self.sample_rate / 100, + }) + } +} + +pub fn play_remote_video_track( + track: &crate::RemoteVideoTrack, +) -> impl Stream { + #[cfg(target_os = "macos")] + { + let mut pool = None; + let most_recent_frame_size = (0, 0); + NativeVideoStream::new(track.0.rtc_track()).filter_map(move |frame| { + if pool == None + || most_recent_frame_size != (frame.buffer.width(), frame.buffer.height()) + { + pool = create_buffer_pool(frame.buffer.width(), frame.buffer.height()).log_err(); + } + let pool = pool.clone(); + async move { + if frame.buffer.width() < 10 && frame.buffer.height() < 10 { + // when the remote stops sharing, we get an 8x8 black image. + // In a lil bit, the unpublish will come through and close the view, + // but until then, don't flash black. + return None; + } + + video_frame_buffer_from_webrtc(pool?, frame.buffer) + } + }) + } + #[cfg(not(target_os = "macos"))] + { + NativeVideoStream::new(track.0.rtc_track()) + .filter_map(|frame| async move { video_frame_buffer_from_webrtc(frame.buffer) }) + } +} + +#[cfg(target_os = "macos")] +fn create_buffer_pool( + width: u32, + height: u32, +) -> Result { + use core_foundation::{base::TCFType, number::CFNumber, string::CFString}; + use core_video::pixel_buffer; + use core_video::{ + pixel_buffer::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + pixel_buffer_io_surface::kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey, + pixel_buffer_pool::{self}, + }; + + let width_key: CFString = + unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferWidthKey) }; + let height_key: CFString = + unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferHeightKey) }; + let animation_key: CFString = unsafe { + CFString::wrap_under_get_rule(kCVPixelBufferIOSurfaceCoreAnimationCompatibilityKey) + }; + let format_key: CFString = + unsafe { CFString::wrap_under_get_rule(pixel_buffer::kCVPixelBufferPixelFormatTypeKey) }; + + let yes: CFNumber = 1.into(); + let width: CFNumber = (width as i32).into(); + let height: CFNumber = (height as i32).into(); + let format: CFNumber = (kCVPixelFormatType_420YpCbCr8BiPlanarFullRange as i64).into(); + + let buffer_attributes = core_foundation::dictionary::CFDictionary::from_CFType_pairs(&[ + (width_key, width.into_CFType()), + (height_key, height.into_CFType()), + (animation_key, yes.into_CFType()), + (format_key, format.into_CFType()), + ]); + + pixel_buffer_pool::CVPixelBufferPool::new(None, Some(&buffer_attributes)).map_err(|cv_return| { + anyhow!( + "failed to create pixel buffer pool: CVReturn({})", + cv_return + ) + }) +} + +#[cfg(target_os = "macos")] +pub type RemoteVideoFrame = core_video::pixel_buffer::CVPixelBuffer; + +#[cfg(target_os = "macos")] +fn video_frame_buffer_from_webrtc( + pool: core_video::pixel_buffer_pool::CVPixelBufferPool, + buffer: Box, +) -> Option { + use core_foundation::base::TCFType; + use core_video::{pixel_buffer::CVPixelBuffer, r#return::kCVReturnSuccess}; + use livekit::webrtc::native::yuv_helper::i420_to_nv12; + + if let Some(native) = buffer.as_native() { + let pixel_buffer = native.get_cv_pixel_buffer(); + if pixel_buffer.is_null() { + return None; + } + return unsafe { Some(CVPixelBuffer::wrap_under_get_rule(pixel_buffer as _)) }; + } + + let i420_buffer = buffer.as_i420()?; + let pixel_buffer = pool.create_pixel_buffer().log_err()?; + + let image_buffer = unsafe { + if pixel_buffer.lock_base_address(0) != kCVReturnSuccess { + return None; + } + + let dst_y = pixel_buffer.get_base_address_of_plane(0); + let dst_y_stride = pixel_buffer.get_bytes_per_row_of_plane(0); + let dst_y_len = pixel_buffer.get_height_of_plane(0) * dst_y_stride; + let dst_uv = pixel_buffer.get_base_address_of_plane(1); + let dst_uv_stride = pixel_buffer.get_bytes_per_row_of_plane(1); + let dst_uv_len = pixel_buffer.get_height_of_plane(1) * dst_uv_stride; + let width = pixel_buffer.get_width(); + let height = pixel_buffer.get_height(); + let dst_y_buffer = std::slice::from_raw_parts_mut(dst_y as *mut u8, dst_y_len); + let dst_uv_buffer = std::slice::from_raw_parts_mut(dst_uv as *mut u8, dst_uv_len); + + let (stride_y, stride_u, stride_v) = i420_buffer.strides(); + let (src_y, src_u, src_v) = i420_buffer.data(); + i420_to_nv12( + src_y, + stride_y, + src_u, + stride_u, + src_v, + stride_v, + dst_y_buffer, + dst_y_stride as u32, + dst_uv_buffer, + dst_uv_stride as u32, + width as i32, + height as i32, + ); + + if pixel_buffer.unlock_base_address(0) != kCVReturnSuccess { + return None; + } + + pixel_buffer + }; + + Some(image_buffer) +} + +#[cfg(not(target_os = "macos"))] +pub type RemoteVideoFrame = Arc; + +#[cfg(not(target_os = "macos"))] +fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { + use gpui::RenderImage; + use image::{Frame, RgbaImage}; + use livekit::webrtc::prelude::VideoFormatType; + use smallvec::SmallVec; + use std::alloc::{alloc, Layout}; + + let width = buffer.width(); + let height = buffer.height(); + let stride = width * 4; + let byte_len = (stride * height) as usize; + let argb_image = unsafe { + // Motivation for this unsafe code is to avoid initializing the frame data, since to_argb + // will write all bytes anyway. + let start_ptr = alloc(Layout::array::(byte_len).log_err()?); + if start_ptr.is_null() { + return None; + } + let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len); + buffer.to_argb( + VideoFormatType::ARGB, // For some reason, this displays correctly while RGBA (the correct format) does not + bgra_frame_slice, + stride, + width as i32, + height as i32, + ); + Vec::from_raw_parts(start_ptr, byte_len, byte_len) + }; + + Some(Arc::new(RenderImage::new(SmallVec::from_elem( + Frame::new( + RgbaImage::from_raw(width, height, argb_image) + .with_context(|| "Bug: not enough bytes allocated for image.") + .log_err()?, + ), + 1, + )))) +} + +#[cfg(target_os = "macos")] +fn video_frame_buffer_to_webrtc(frame: ScreenCaptureFrame) -> Option> { + use livekit::webrtc; + + let pixel_buffer = frame.0.as_concrete_TypeRef(); + std::mem::forget(frame.0); + unsafe { + Some(webrtc::video_frame::native::NativeBuffer::from_cv_pixel_buffer(pixel_buffer as _)) + } +} + +#[cfg(not(target_os = "macos"))] +fn video_frame_buffer_to_webrtc(_frame: ScreenCaptureFrame) -> Option> { + None as Option> +} + +trait DeviceChangeListenerApi: Stream + Sized { + fn new(input: bool) -> Result; +} + +#[cfg(target_os = "macos")] +mod macos { + + use coreaudio::sys::{ + kAudioHardwarePropertyDefaultInputDevice, kAudioHardwarePropertyDefaultOutputDevice, + kAudioObjectPropertyElementMaster, kAudioObjectPropertyScopeGlobal, + kAudioObjectSystemObject, AudioObjectAddPropertyListener, AudioObjectID, + AudioObjectPropertyAddress, AudioObjectRemovePropertyListener, OSStatus, + }; + use futures::{channel::mpsc::UnboundedReceiver, StreamExt}; + + /// Implementation from: https://github.com/zed-industries/cpal/blob/fd8bc2fd39f1f5fdee5a0690656caff9a26d9d50/src/host/coreaudio/macos/property_listener.rs#L15 + pub struct CoreAudioDefaultDeviceChangeListener { + rx: UnboundedReceiver<()>, + callback: Box, + input: bool, + } + + trait _AssertSend: Send {} + impl _AssertSend for CoreAudioDefaultDeviceChangeListener {} + + struct PropertyListenerCallbackWrapper(Box); + + unsafe extern "C" fn property_listener_handler_shim( + _: AudioObjectID, + _: u32, + _: *const AudioObjectPropertyAddress, + callback: *mut ::std::os::raw::c_void, + ) -> OSStatus { + let wrapper = callback as *mut PropertyListenerCallbackWrapper; + (*wrapper).0(); + 0 + } + + impl super::DeviceChangeListenerApi for CoreAudioDefaultDeviceChangeListener { + fn new(input: bool) -> gpui::Result { + let (tx, rx) = futures::channel::mpsc::unbounded(); + + let callback = Box::new(PropertyListenerCallbackWrapper(Box::new(move || { + tx.unbounded_send(()).ok(); + }))); + + unsafe { + coreaudio::Error::from_os_status(AudioObjectAddPropertyListener( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: if input { + kAudioHardwarePropertyDefaultInputDevice + } else { + kAudioHardwarePropertyDefaultOutputDevice + }, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }, + Some(property_listener_handler_shim), + &*callback as *const _ as *mut _, + ))?; + } + + Ok(Self { + rx, + callback, + input, + }) + } + } + + impl Drop for CoreAudioDefaultDeviceChangeListener { + fn drop(&mut self) { + unsafe { + AudioObjectRemovePropertyListener( + kAudioObjectSystemObject, + &AudioObjectPropertyAddress { + mSelector: if self.input { + kAudioHardwarePropertyDefaultInputDevice + } else { + kAudioHardwarePropertyDefaultOutputDevice + }, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMaster, + }, + Some(property_listener_handler_shim), + &*self.callback as *const _ as *mut _, + ); + } + } + } + + impl futures::Stream for CoreAudioDefaultDeviceChangeListener { + type Item = (); + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.rx.poll_next_unpin(cx) + } + } +} + +#[cfg(target_os = "macos")] +type DeviceChangeListener = macos::CoreAudioDefaultDeviceChangeListener; + +#[cfg(not(target_os = "macos"))] +mod noop_change_listener { + use std::task::Poll; + + use super::DeviceChangeListenerApi; + + pub struct NoopOutputDeviceChangelistener {} + + impl DeviceChangeListenerApi for NoopOutputDeviceChangelistener { + fn new(_input: bool) -> anyhow::Result { + Ok(NoopOutputDeviceChangelistener {}) + } + } + + impl futures::Stream for NoopOutputDeviceChangelistener { + type Item = (); + + fn poll_next( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> Poll> { + Poll::Pending + } + } +} + +#[cfg(not(target_os = "macos"))] +type DeviceChangeListener = noop_change_listener::NoopOutputDeviceChangelistener; diff --git a/crates/livekit_client/src/mock_client.rs b/crates/livekit_client/src/mock_client.rs new file mode 100644 index 0000000000..d1594edff8 --- /dev/null +++ b/crates/livekit_client/src/mock_client.rs @@ -0,0 +1,38 @@ +use crate::test; + +pub(crate) mod participant; +pub(crate) mod publication; +pub(crate) mod track; + +pub type RemoteVideoTrack = track::RemoteVideoTrack; +pub type RemoteAudioTrack = track::RemoteAudioTrack; +pub type RemoteTrackPublication = publication::RemoteTrackPublication; +pub type RemoteParticipant = participant::RemoteParticipant; + +pub type LocalVideoTrack = track::LocalVideoTrack; +pub type LocalAudioTrack = track::LocalAudioTrack; +pub type LocalTrackPublication = publication::LocalTrackPublication; +pub type LocalParticipant = participant::LocalParticipant; + +pub type Room = test::Room; +pub use test::{ConnectionState, ParticipantIdentity, TrackSid}; + +pub struct AudioStream {} + +#[cfg(not(target_os = "macos"))] +pub type RemoteVideoFrame = std::sync::Arc; + +#[cfg(target_os = "macos")] +#[derive(Clone)] +pub(crate) struct RemoteVideoFrame {} +#[cfg(target_os = "macos")] +impl Into for RemoteVideoFrame { + fn into(self) -> gpui::SurfaceSource { + unimplemented!() + } +} +pub(crate) fn play_remote_video_track( + _track: &crate::RemoteVideoTrack, +) -> impl futures::Stream { + futures::stream::pending() +} diff --git a/crates/livekit_client/src/test/participant.rs b/crates/livekit_client/src/mock_client/participant.rs similarity index 55% rename from crates/livekit_client/src/test/participant.rs rename to crates/livekit_client/src/mock_client/participant.rs index f8df22ff5d..3eb6c56a99 100644 --- a/crates/livekit_client/src/test/participant.rs +++ b/crates/livekit_client/src/mock_client/participant.rs @@ -1,26 +1,24 @@ -use super::*; - -#[derive(Clone, Debug)] -pub enum Participant { - Local(LocalParticipant), - Remote(RemoteParticipant), -} +use crate::{ + test::{Room, WeakRoom}, + AudioStream, LocalAudioTrack, LocalTrackPublication, LocalVideoTrack, Participant, + ParticipantIdentity, RemoteTrack, RemoteTrackPublication, TrackSid, +}; +use anyhow::Result; +use collections::HashMap; +use gpui::{AsyncApp, ScreenCaptureSource, ScreenCaptureStream}; #[derive(Clone, Debug)] pub struct LocalParticipant { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) identity: ParticipantIdentity, - pub(super) room: Room, + pub(crate) identity: ParticipantIdentity, + pub(crate) room: Room, } #[derive(Clone, Debug)] pub struct RemoteParticipant { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) identity: ParticipantIdentity, - pub(super) room: WeakRoom, + pub(crate) identity: ParticipantIdentity, + pub(crate) room: WeakRoom, } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl Participant { pub fn identity(&self) -> ParticipantIdentity { match self { @@ -30,41 +28,53 @@ impl Participant { } } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl LocalParticipant { - pub async fn unpublish_track(&self, track: &TrackSid) -> Result<()> { + pub async fn unpublish_track(&self, track: TrackSid, _cx: &AsyncApp) -> Result<()> { self.room .test_server() - .unpublish_track(self.room.token(), track) + .unpublish_track(self.room.token(), &track) .await } - pub async fn publish_track( + pub(crate) async fn publish_microphone_track( &self, - track: LocalTrack, - _options: TrackPublishOptions, - ) -> Result { + _cx: &AsyncApp, + ) -> Result<(LocalTrackPublication, AudioStream)> { let this = self.clone(); - let track = track.clone(); let server = this.room.test_server(); - let sid = match track { - LocalTrack::Video(track) => { - server.publish_video_track(this.room.token(), track).await? - } - LocalTrack::Audio(track) => { - server - .publish_audio_track(this.room.token(), &track) - .await? - } - }; - Ok(LocalTrackPublication { - room: self.room.downgrade(), - sid, - }) + let sid = server + .publish_audio_track(this.room.token(), &LocalAudioTrack {}) + .await?; + + Ok(( + LocalTrackPublication { + room: self.room.downgrade(), + sid, + }, + AudioStream {}, + )) + } + + pub async fn publish_screenshare_track( + &self, + _source: &dyn ScreenCaptureSource, + _cx: &mut AsyncApp, + ) -> Result<(LocalTrackPublication, Box)> { + let this = self.clone(); + let server = this.room.test_server(); + let sid = server + .publish_video_track(this.room.token(), LocalVideoTrack {}) + .await?; + Ok(( + LocalTrackPublication { + room: self.room.downgrade(), + sid, + }, + Box::new(TestScreenCaptureStream {}), + )) } } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl RemoteParticipant { pub fn track_publications(&self) -> HashMap { if let Some(room) = self.room.upgrade() { @@ -109,3 +119,7 @@ impl RemoteParticipant { self.identity.clone() } } + +struct TestScreenCaptureStream; + +impl gpui::ScreenCaptureStream for TestScreenCaptureStream {} diff --git a/crates/livekit_client/src/test/publication.rs b/crates/livekit_client/src/mock_client/publication.rs similarity index 66% rename from crates/livekit_client/src/test/publication.rs rename to crates/livekit_client/src/mock_client/publication.rs index b6fd6c0431..0176e63a9e 100644 --- a/crates/livekit_client/src/test/publication.rs +++ b/crates/livekit_client/src/mock_client/publication.rs @@ -1,54 +1,30 @@ -use super::*; +use gpui::App; -#[derive(Clone, Debug)] -pub enum TrackPublication { - Local(LocalTrackPublication), - Remote(RemoteTrackPublication), -} +use crate::{test::WeakRoom, RemoteTrack, TrackSid}; #[derive(Clone, Debug)] pub struct LocalTrackPublication { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] pub(crate) sid: TrackSid, pub(crate) room: WeakRoom, } #[derive(Clone, Debug)] pub struct RemoteTrackPublication { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] pub(crate) sid: TrackSid, pub(crate) room: WeakRoom, pub(crate) track: RemoteTrack, } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl TrackPublication { - pub fn sid(&self) -> TrackSid { - match self { - TrackPublication::Local(track) => track.sid(), - TrackPublication::Remote(track) => track.sid(), - } - } - - pub fn is_muted(&self) -> bool { - match self { - TrackPublication::Local(track) => track.is_muted(), - TrackPublication::Remote(track) => track.is_muted(), - } - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl LocalTrackPublication { pub fn sid(&self) -> TrackSid { self.sid.clone() } - pub fn mute(&self) { + pub fn mute(&self, _cx: &App) { self.set_mute(true) } - pub fn unmute(&self) { + pub fn unmute(&self, _cx: &App) { self.set_mute(false) } @@ -71,7 +47,6 @@ impl LocalTrackPublication { } } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl RemoteTrackPublication { pub fn sid(&self) -> TrackSid { self.sid.clone() @@ -81,8 +56,8 @@ impl RemoteTrackPublication { Some(self.track.clone()) } - pub fn kind(&self) -> TrackKind { - self.track.kind() + pub fn is_audio(&self) -> bool { + matches!(self.track, RemoteTrack::Audio(_)) } pub fn is_muted(&self) -> bool { @@ -103,7 +78,7 @@ impl RemoteTrackPublication { } } - pub fn set_enabled(&self, enabled: bool) { + pub fn set_enabled(&self, enabled: bool, _cx: &App) { if let Some(room) = self.room.upgrade() { let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; if enabled { @@ -114,3 +89,12 @@ impl RemoteTrackPublication { } } } + +impl RemoteTrack { + pub fn set_enabled(&self, enabled: bool, _cx: &App) { + match self { + RemoteTrack::Audio(remote_audio_track) => remote_audio_track.set_enabled(enabled), + RemoteTrack::Video(remote_video_track) => remote_video_track.set_enabled(enabled), + } + } +} diff --git a/crates/livekit_client/src/mock_client/track.rs b/crates/livekit_client/src/mock_client/track.rs new file mode 100644 index 0000000000..aaa1e52783 --- /dev/null +++ b/crates/livekit_client/src/mock_client/track.rs @@ -0,0 +1,75 @@ +use std::sync::Arc; + +use crate::{ + test::{TestServerAudioTrack, TestServerVideoTrack, WeakRoom}, + ParticipantIdentity, TrackSid, +}; + +#[derive(Clone, Debug)] +pub struct LocalVideoTrack {} + +#[derive(Clone, Debug)] +pub struct LocalAudioTrack {} + +#[derive(Clone, Debug)] +pub struct RemoteVideoTrack { + pub(crate) server_track: Arc, + pub(crate) _room: WeakRoom, +} + +#[derive(Clone, Debug)] +pub struct RemoteAudioTrack { + pub(crate) server_track: Arc, + pub(crate) room: WeakRoom, +} + +impl RemoteAudioTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub fn enabled(&self) -> bool { + if let Some(room) = self.room.upgrade() { + !room + .0 + .lock() + .paused_audio_tracks + .contains(&self.server_track.sid) + } else { + false + } + } + + pub fn set_enabled(&self, enabled: bool) { + let Some(room) = self.room.upgrade() else { + return; + }; + if enabled { + room.0 + .lock() + .paused_audio_tracks + .remove(&self.server_track.sid); + } else { + room.0 + .lock() + .paused_audio_tracks + .insert(self.server_track.sid.clone()); + } + } +} + +impl RemoteVideoTrack { + pub fn sid(&self) -> TrackSid { + self.server_track.sid.clone() + } + + pub fn publisher_id(&self) -> ParticipantIdentity { + self.server_track.publisher_id.clone() + } + + pub(crate) fn set_enabled(&self, _enabled: bool) {} +} diff --git a/crates/livekit_client/src/remote_video_track_view.rs b/crates/livekit_client/src/remote_video_track_view.rs index 5602c829c6..9073b8729a 100644 --- a/crates/livekit_client/src/remote_video_track_view.rs +++ b/crates/livekit_client/src/remote_video_track_view.rs @@ -1,5 +1,4 @@ -use crate::track::RemoteVideoTrack; -use anyhow::Result; +use super::RemoteVideoTrack; use futures::StreamExt as _; use gpui::{ AppContext as _, Context, Empty, Entity, EventEmitter, IntoElement, Render, Task, Window, @@ -12,7 +11,7 @@ pub struct RemoteVideoTrackView { current_rendered_frame: Option, #[cfg(not(target_os = "macos"))] previous_rendered_frame: Option, - _maintain_frame: Task>, + _maintain_frame: Task<()>, } #[derive(Debug)] @@ -23,8 +22,27 @@ pub enum RemoteVideoTrackViewEvent { impl RemoteVideoTrackView { pub fn new(track: RemoteVideoTrack, window: &mut Window, cx: &mut Context) -> Self { cx.focus_handle(); - let frames = super::play_remote_video_track(&track); - let _window_handle = window.window_handle(); + let frames = crate::play_remote_video_track(&track); + + #[cfg(not(target_os = "macos"))] + { + use util::ResultExt; + + let window_handle = window.window_handle(); + cx.on_release(move |this, cx| { + if let Some(frame) = this.previous_rendered_frame.take() { + window_handle + .update(cx, |_, window, _cx| window.drop_image(frame).log_err()) + .ok(); + } + if let Some(frame) = this.current_rendered_frame.take() { + window_handle + .update(cx, |_, window, _cx| window.drop_image(frame).log_err()) + .ok(); + } + }) + .detach(); + } Self { track, @@ -35,28 +53,11 @@ impl RemoteVideoTrackView { this.update(cx, |this, cx| { this.latest_frame = Some(frame); cx.notify(); - })?; + }) + .ok(); } - this.update(cx, |_this, cx| { - #[cfg(not(target_os = "macos"))] - { - use util::ResultExt as _; - if let Some(frame) = _this.previous_rendered_frame.take() { - _window_handle - .update(cx, |_, window, _cx| window.drop_image(frame).log_err()) - .ok(); - } - // TODO(mgsloan): This might leak the last image of the screenshare if - // render is called after the screenshare ends. - if let Some(frame) = _this.current_rendered_frame.take() { - _window_handle - .update(cx, |_, window, _cx| window.drop_image(frame).log_err()) - .ok(); - } - } - cx.emit(RemoteVideoTrackViewEvent::Close) - })?; - Ok(()) + this.update(cx, |_this, cx| cx.emit(RemoteVideoTrackViewEvent::Close)) + .ok(); }), #[cfg(not(target_os = "macos"))] current_rendered_frame: None, diff --git a/crates/livekit_client/src/test.rs b/crates/livekit_client/src/test.rs index 363b51c84b..b27479d7b1 100644 --- a/crates/livekit_client/src/test.rs +++ b/crates/livekit_client/src/test.rs @@ -1,19 +1,10 @@ -pub mod participant; -pub mod publication; -pub mod track; +use crate::{AudioStream, Participant, RemoteTrack, RoomEvent, TrackPublication}; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub mod webrtc; - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -use self::id::*; -use self::{participant::*, publication::*, track::*}; +use crate::mock_client::{participant::*, publication::*, track::*}; use anyhow::{anyhow, Context as _, Result}; use async_trait::async_trait; use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet}; -use gpui::BackgroundExecutor; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -use livekit::options::TrackPublishOptions; +use gpui::{App, AsyncApp, BackgroundExecutor}; use livekit_api::{proto, token}; use parking_lot::Mutex; use postage::{mpsc, sink::Sink}; @@ -22,8 +13,32 @@ use std::sync::{ Arc, Weak, }; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub use livekit::{id, options, ConnectionState, DisconnectReason, RoomOptions}; +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct ParticipantIdentity(pub String); + +#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct TrackSid(pub(crate) String); + +impl std::fmt::Display for TrackSid { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl TryFrom for TrackSid { + type Error = anyhow::Error; + + fn try_from(value: String) -> Result { + Ok(TrackSid(value)) + } +} + +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[non_exhaustive] +pub enum ConnectionState { + Connected, + Disconnected, +} static SERVERS: Mutex>> = Mutex::new(BTreeMap::new()); @@ -31,12 +46,10 @@ pub struct TestServer { pub url: String, pub api_key: String, pub secret_key: String, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] rooms: Mutex>, executor: BackgroundExecutor, } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl TestServer { pub fn create( url: String, @@ -83,7 +96,7 @@ impl TestServer { } pub async fn create_room(&self, room: String) -> Result<()> { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); if let Entry::Vacant(e) = server_rooms.entry(room.clone()) { @@ -95,7 +108,7 @@ impl TestServer { } async fn delete_room(&self, room: String) -> Result<()> { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); server_rooms @@ -105,7 +118,7 @@ impl TestServer { } async fn join_room(&self, token: String, client_room: Room) -> Result { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let claims = livekit_api::token::validate(&token, &self.secret_key)?; let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -172,7 +185,7 @@ impl TestServer { } async fn leave_room(&self, token: String) -> Result<()> { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let claims = livekit_api::token::validate(&token, &self.secret_key)?; let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -229,7 +242,7 @@ impl TestServer { room_name: String, identity: ParticipantIdentity, ) -> Result<()> { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); let room = server_rooms @@ -251,7 +264,7 @@ impl TestServer { identity: String, permission: proto::ParticipantPermission, ) -> Result<()> { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); let room = server_rooms @@ -265,7 +278,7 @@ impl TestServer { pub async fn disconnect_client(&self, client_identity: String) { let client_identity = ParticipantIdentity(client_identity); - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let mut server_rooms = self.rooms.lock(); for room in server_rooms.values_mut() { @@ -274,19 +287,19 @@ impl TestServer { room.connection_state = ConnectionState::Disconnected; room.updates_tx .blocking_send(RoomEvent::Disconnected { - reason: DisconnectReason::SignalClose, + reason: "SIGNAL_CLOSED", }) .ok(); } } } - async fn publish_video_track( + pub(crate) async fn publish_video_track( &self, token: String, _local_track: LocalVideoTrack, ) -> Result { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let claims = livekit_api::token::validate(&token, &self.secret_key)?; let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -347,12 +360,12 @@ impl TestServer { Ok(sid) } - async fn publish_audio_track( + pub(crate) async fn publish_audio_track( &self, token: String, _local_track: &LocalAudioTrack, ) -> Result { - self.executor.simulate_random_delay().await; + self.simulate_random_delay().await; let claims = livekit_api::token::validate(&token, &self.secret_key)?; let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -414,11 +427,16 @@ impl TestServer { Ok(sid) } - async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> { + pub(crate) async fn unpublish_track(&self, _token: String, _track: &TrackSid) -> Result<()> { Ok(()) } - fn set_track_muted(&self, token: &str, track_sid: &TrackSid, muted: bool) -> Result<()> { + pub(crate) fn set_track_muted( + &self, + token: &str, + track_sid: &TrackSid, + muted: bool, + ) -> Result<()> { let claims = livekit_api::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -472,7 +490,7 @@ impl TestServer { Ok(()) } - fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option { + pub(crate) fn is_track_muted(&self, token: &str, track_sid: &TrackSid) -> Option { let claims = livekit_api::token::validate(&token, &self.secret_key).ok()?; let room_name = claims.video.room.unwrap(); @@ -487,7 +505,7 @@ impl TestServer { }) } - fn video_tracks(&self, token: String) -> Result> { + pub(crate) fn video_tracks(&self, token: String) -> Result> { let claims = livekit_api::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -510,7 +528,7 @@ impl TestServer { .collect()) } - fn audio_tracks(&self, token: String) -> Result> { + pub(crate) fn audio_tracks(&self, token: String) -> Result> { let claims = livekit_api::token::validate(&token, &self.secret_key)?; let room_name = claims.video.room.unwrap(); let identity = ParticipantIdentity(claims.sub.unwrap().to_string()); @@ -532,9 +550,13 @@ impl TestServer { }) .collect()) } + + async fn simulate_random_delay(&self) { + #[cfg(any(test, feature = "test-support"))] + self.executor.simulate_random_delay().await; + } } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] #[derive(Default, Debug)] struct TestServerRoom { client_rooms: HashMap, @@ -543,103 +565,24 @@ struct TestServerRoom { participant_permissions: HashMap, } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] #[derive(Debug)] -struct TestServerVideoTrack { - sid: TrackSid, - publisher_id: ParticipantIdentity, +pub(crate) struct TestServerVideoTrack { + pub(crate) sid: TrackSid, + pub(crate) publisher_id: ParticipantIdentity, // frames_rx: async_broadcast::Receiver, } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] #[derive(Debug)] -struct TestServerAudioTrack { - sid: TrackSid, - publisher_id: ParticipantIdentity, - muted: AtomicBool, +pub(crate) struct TestServerAudioTrack { + pub(crate) sid: TrackSid, + pub(crate) publisher_id: ParticipantIdentity, + pub(crate) muted: AtomicBool, } pub struct TestApiClient { url: String, } -#[derive(Clone, Debug)] -#[non_exhaustive] -pub enum RoomEvent { - ParticipantConnected(RemoteParticipant), - ParticipantDisconnected(RemoteParticipant), - LocalTrackPublished { - publication: LocalTrackPublication, - track: LocalTrack, - participant: LocalParticipant, - }, - LocalTrackUnpublished { - publication: LocalTrackPublication, - participant: LocalParticipant, - }, - TrackSubscribed { - track: RemoteTrack, - publication: RemoteTrackPublication, - participant: RemoteParticipant, - }, - TrackUnsubscribed { - track: RemoteTrack, - publication: RemoteTrackPublication, - participant: RemoteParticipant, - }, - TrackSubscriptionFailed { - participant: RemoteParticipant, - error: String, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - track_sid: TrackSid, - }, - TrackPublished { - publication: RemoteTrackPublication, - participant: RemoteParticipant, - }, - TrackUnpublished { - publication: RemoteTrackPublication, - participant: RemoteParticipant, - }, - TrackMuted { - participant: Participant, - publication: TrackPublication, - }, - TrackUnmuted { - participant: Participant, - publication: TrackPublication, - }, - RoomMetadataChanged { - old_metadata: String, - metadata: String, - }, - ParticipantMetadataChanged { - participant: Participant, - old_metadata: String, - metadata: String, - }, - ParticipantNameChanged { - participant: Participant, - old_name: String, - name: String, - }, - ActiveSpeakersChanged { - speakers: Vec, - }, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - ConnectionStateChanged(ConnectionState), - Connected { - participants_with_tracks: Vec<(RemoteParticipant, Vec)>, - }, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - Disconnected { - reason: DisconnectReason, - }, - Reconnecting, - Reconnected, -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] #[async_trait] impl livekit_api::Client for TestApiClient { fn url(&self) -> &str { @@ -700,25 +643,21 @@ impl livekit_api::Client for TestApiClient { } } -struct RoomState { - url: String, - token: String, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - local_identity: ParticipantIdentity, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - connection_state: ConnectionState, - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - paused_audio_tracks: HashSet, - updates_tx: mpsc::Sender, +pub(crate) struct RoomState { + pub(crate) url: String, + pub(crate) token: String, + pub(crate) local_identity: ParticipantIdentity, + pub(crate) connection_state: ConnectionState, + pub(crate) paused_audio_tracks: HashSet, + pub(crate) updates_tx: mpsc::Sender, } #[derive(Clone, Debug)] -pub struct Room(Arc>); +pub struct Room(pub(crate) Arc>); #[derive(Clone, Debug)] pub(crate) struct WeakRoom(Weak>); -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl std::fmt::Debug for RoomState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Room") @@ -731,19 +670,8 @@ impl std::fmt::Debug for RoomState { } } -#[cfg(all(target_os = "windows", target_env = "gnu"))] -impl std::fmt::Debug for RoomState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Room") - .field("url", &self.url) - .field("token", &self.token) - .finish() - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl Room { - fn downgrade(&self) -> WeakRoom { + pub(crate) fn downgrade(&self) -> WeakRoom { WeakRoom(Arc::downgrade(&self.0)) } @@ -760,9 +688,9 @@ impl Room { } pub async fn connect( - url: &str, - token: &str, - _options: RoomOptions, + url: String, + token: String, + _cx: &mut AsyncApp, ) -> Result<(Self, mpsc::Receiver)> { let server = TestServer::get(&url)?; let (updates_tx, updates_rx) = mpsc::channel(1024); @@ -794,16 +722,34 @@ impl Room { .unwrap() } - fn test_server(&self) -> Arc { + pub(crate) fn test_server(&self) -> Arc { TestServer::get(&self.0.lock().url).unwrap() } - fn token(&self) -> String { + pub(crate) fn token(&self) -> String { self.0.lock().token.clone() } + + pub fn play_remote_audio_track( + &self, + _track: &RemoteAudioTrack, + _cx: &App, + ) -> anyhow::Result { + Ok(AudioStream {}) + } + + pub async fn unpublish_local_track(&self, sid: TrackSid, cx: &mut AsyncApp) -> Result<()> { + self.local_participant().unpublish_track(sid, cx).await + } + + pub async fn publish_local_microphone_track( + &self, + cx: &mut AsyncApp, + ) -> Result<(LocalTrackPublication, AudioStream)> { + self.local_participant().publish_microphone_track(cx).await + } } -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] impl Drop for RoomState { fn drop(&mut self) { if self.connection_state == ConnectionState::Connected { @@ -819,7 +765,7 @@ impl Drop for RoomState { } impl WeakRoom { - fn upgrade(&self) -> Option { + pub(crate) fn upgrade(&self) -> Option { self.0.upgrade().map(Room) } } diff --git a/crates/livekit_client/src/test/track.rs b/crates/livekit_client/src/test/track.rs deleted file mode 100644 index 93984dea4a..0000000000 --- a/crates/livekit_client/src/test/track.rs +++ /dev/null @@ -1,201 +0,0 @@ -use super::*; -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -use webrtc::{audio_source::RtcAudioSource, video_source::RtcVideoSource}; - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -pub use livekit::track::{TrackKind, TrackSource}; - -#[derive(Clone, Debug)] -pub enum LocalTrack { - Audio(LocalAudioTrack), - Video(LocalVideoTrack), -} - -#[derive(Clone, Debug)] -pub enum RemoteTrack { - Audio(RemoteAudioTrack), - Video(RemoteVideoTrack), -} - -#[derive(Clone, Debug)] -pub struct LocalVideoTrack {} - -#[derive(Clone, Debug)] -pub struct LocalAudioTrack {} - -#[derive(Clone, Debug)] -pub struct RemoteVideoTrack { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) server_track: Arc, - pub(super) _room: WeakRoom, -} - -#[derive(Clone, Debug)] -pub struct RemoteAudioTrack { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) server_track: Arc, - pub(super) room: WeakRoom, -} - -pub enum RtcTrack { - Audio(RtcAudioTrack), - Video(RtcVideoTrack), -} - -pub struct RtcAudioTrack { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) server_track: Arc, - pub(super) room: WeakRoom, -} - -pub struct RtcVideoTrack { - #[cfg(not(all(target_os = "windows", target_env = "gnu")))] - pub(super) _server_track: Arc, -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl RemoteTrack { - pub fn sid(&self) -> TrackSid { - match self { - RemoteTrack::Audio(track) => track.sid(), - RemoteTrack::Video(track) => track.sid(), - } - } - - pub fn kind(&self) -> TrackKind { - match self { - RemoteTrack::Audio(_) => TrackKind::Audio, - RemoteTrack::Video(_) => TrackKind::Video, - } - } - - pub fn publisher_id(&self) -> ParticipantIdentity { - match self { - RemoteTrack::Audio(track) => track.publisher_id(), - RemoteTrack::Video(track) => track.publisher_id(), - } - } - - pub fn rtc_track(&self) -> RtcTrack { - match self { - RemoteTrack::Audio(track) => RtcTrack::Audio(track.rtc_track()), - RemoteTrack::Video(track) => RtcTrack::Video(track.rtc_track()), - } - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl LocalVideoTrack { - pub fn create_video_track(_name: &str, _source: RtcVideoSource) -> Self { - Self {} - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl LocalAudioTrack { - pub fn create_audio_track(_name: &str, _source: RtcAudioSource) -> Self { - Self {} - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl RemoteAudioTrack { - pub fn sid(&self) -> TrackSid { - self.server_track.sid.clone() - } - - pub fn publisher_id(&self) -> ParticipantIdentity { - self.server_track.publisher_id.clone() - } - - pub fn start(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .remove(&self.server_track.sid); - } - } - - pub fn stop(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .insert(self.server_track.sid.clone()); - } - } - - pub fn rtc_track(&self) -> RtcAudioTrack { - RtcAudioTrack { - server_track: self.server_track.clone(), - room: self.room.clone(), - } - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl RemoteVideoTrack { - pub fn sid(&self) -> TrackSid { - self.server_track.sid.clone() - } - - pub fn publisher_id(&self) -> ParticipantIdentity { - self.server_track.publisher_id.clone() - } - - pub fn rtc_track(&self) -> RtcVideoTrack { - RtcVideoTrack { - _server_track: self.server_track.clone(), - } - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl RtcTrack { - pub fn enabled(&self) -> bool { - match self { - RtcTrack::Audio(track) => track.enabled(), - RtcTrack::Video(track) => track.enabled(), - } - } - - pub fn set_enabled(&self, enabled: bool) { - match self { - RtcTrack::Audio(track) => track.set_enabled(enabled), - RtcTrack::Video(_) => {} - } - } -} - -#[cfg(not(all(target_os = "windows", target_env = "gnu")))] -impl RtcAudioTrack { - pub fn set_enabled(&self, enabled: bool) { - if let Some(room) = self.room.upgrade() { - let paused_audio_tracks = &mut room.0.lock().paused_audio_tracks; - if enabled { - paused_audio_tracks.remove(&self.server_track.sid); - } else { - paused_audio_tracks.insert(self.server_track.sid.clone()); - } - } - } - - pub fn enabled(&self) -> bool { - if let Some(room) = self.room.upgrade() { - !room - .0 - .lock() - .paused_audio_tracks - .contains(&self.server_track.sid) - } else { - false - } - } -} - -impl RtcVideoTrack { - pub fn enabled(&self) -> bool { - true - } -} diff --git a/crates/livekit_client/src/test/webrtc.rs b/crates/livekit_client/src/test/webrtc.rs deleted file mode 100644 index 6ac06e0484..0000000000 --- a/crates/livekit_client/src/test/webrtc.rs +++ /dev/null @@ -1,136 +0,0 @@ -use super::track::{RtcAudioTrack, RtcVideoTrack}; -use futures::Stream; -use livekit::webrtc as real; -use std::{ - pin::Pin, - task::{Context, Poll}, -}; - -pub mod video_stream { - use super::*; - - pub mod native { - use super::*; - use real::video_frame::BoxVideoFrame; - - pub struct NativeVideoStream { - pub track: RtcVideoTrack, - } - - impl NativeVideoStream { - pub fn new(track: RtcVideoTrack) -> Self { - Self { track } - } - } - - impl Stream for NativeVideoStream { - type Item = BoxVideoFrame; - - fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { - Poll::Pending - } - } - } -} - -pub mod audio_stream { - use super::*; - - pub mod native { - use super::*; - use real::audio_frame::AudioFrame; - - pub struct NativeAudioStream { - pub track: RtcAudioTrack, - } - - impl NativeAudioStream { - pub fn new(track: RtcAudioTrack, _sample_rate: i32, _num_channels: i32) -> Self { - Self { track } - } - } - - impl Stream for NativeAudioStream { - type Item = AudioFrame<'static>; - - fn poll_next(self: Pin<&mut Self>, _cx: &mut Context) -> Poll> { - Poll::Pending - } - } - } -} - -pub mod audio_source { - use super::*; - - pub use real::audio_source::AudioSourceOptions; - - pub mod native { - use std::sync::Arc; - - use super::*; - use real::{audio_frame::AudioFrame, RtcError}; - - #[derive(Clone)] - pub struct NativeAudioSource { - pub options: Arc, - pub sample_rate: u32, - pub num_channels: u32, - } - - impl NativeAudioSource { - pub fn new( - options: AudioSourceOptions, - sample_rate: u32, - num_channels: u32, - _queue_size_ms: u32, - ) -> Self { - Self { - options: Arc::new(options), - sample_rate, - num_channels, - } - } - - pub async fn capture_frame(&self, _frame: &AudioFrame<'_>) -> Result<(), RtcError> { - Ok(()) - } - } - } - - pub enum RtcAudioSource { - Native(native::NativeAudioSource), - } -} - -pub use livekit::webrtc::audio_frame; -pub use livekit::webrtc::video_frame; - -pub mod video_source { - use super::*; - pub use real::video_source::VideoResolution; - - pub struct RTCVideoSource; - - pub mod native { - use super::*; - use real::video_frame::{VideoBuffer, VideoFrame}; - - #[derive(Clone)] - pub struct NativeVideoSource { - pub resolution: VideoResolution, - } - - impl NativeVideoSource { - pub fn new(resolution: super::VideoResolution) -> Self { - Self { resolution } - } - - pub fn capture_frame>(&self, _frame: &VideoFrame) {} - } - } - - pub enum RtcVideoSource { - Native(native::NativeVideoSource), - } -} diff --git a/crates/livekit_client_macos/.cargo/config.toml b/crates/livekit_client_macos/.cargo/config.toml deleted file mode 100644 index 77f7c9dd6c..0000000000 --- a/crates/livekit_client_macos/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[livekit_client_test] -rustflags = ["-C", "link-args=-ObjC"] diff --git a/crates/livekit_client_macos/Cargo.toml b/crates/livekit_client_macos/Cargo.toml deleted file mode 100644 index 78d3ad1068..0000000000 --- a/crates/livekit_client_macos/Cargo.toml +++ /dev/null @@ -1,67 +0,0 @@ -[package] -name = "livekit_client_macos" -version = "0.1.0" -edition.workspace = true -description = "Bindings to LiveKit Swift client SDK" -publish.workspace = true -license = "GPL-3.0-or-later" - -[lints] -workspace = true - -[lib] -path = "src/livekit_client.rs" -doctest = false - -[[example]] -name = "test_app_macos" - -[features] -no-webrtc = [] -test-support = [ - "async-trait", - "collections/test-support", - "gpui/test-support", - "livekit_api", - "nanoid", -] - -[dependencies] -anyhow.workspace = true -async-broadcast = "0.7" -async-trait = { workspace = true, optional = true } -collections = { workspace = true, optional = true } -futures.workspace = true -gpui = { workspace = true, optional = true } -livekit_api = { workspace = true, optional = true } -log.workspace = true -media.workspace = true -nanoid = { workspace = true, optional = true} -parking_lot.workspace = true -postage.workspace = true - -[target.'cfg(target_os = "macos")'.dependencies] -core-foundation.workspace = true - -[target.'cfg(all(not(target_os = "macos")))'.dependencies] -async-trait.workspace = true -collections.workspace = true -gpui.workspace = true -livekit_api.workspace = true -nanoid.workspace = true - -[dev-dependencies] -async-trait.workspace = true -collections = { workspace = true, features = ["test-support"] } -gpui = { workspace = true, features = ["test-support"] } -livekit_api.workspace = true -nanoid.workspace = true -sha2.workspace = true -simplelog.workspace = true - -[build-dependencies] -serde.workspace = true -serde_json.workspace = true - -[package.metadata.cargo-machete] -ignored = ["serde_json"] diff --git a/crates/livekit_client_macos/LICENSE-GPL b/crates/livekit_client_macos/LICENSE-GPL deleted file mode 120000 index 89e542f750..0000000000 --- a/crates/livekit_client_macos/LICENSE-GPL +++ /dev/null @@ -1 +0,0 @@ -../../LICENSE-GPL \ No newline at end of file diff --git a/crates/livekit_client_macos/LiveKitBridge/Package.resolved b/crates/livekit_client_macos/LiveKitBridge/Package.resolved deleted file mode 100644 index c84933e5c1..0000000000 --- a/crates/livekit_client_macos/LiveKitBridge/Package.resolved +++ /dev/null @@ -1,52 +0,0 @@ -{ - "object": { - "pins": [ - { - "package": "LiveKit", - "repositoryURL": "https://github.com/livekit/client-sdk-swift.git", - "state": { - "branch": null, - "revision": "8cde9e66ce9b470c3a743f5c72784f57c5a6d0c3", - "version": "1.1.6" - } - }, - { - "package": "Promises", - "repositoryURL": "https://github.com/google/promises.git", - "state": { - "branch": null, - "revision": "ec957ccddbcc710ccc64c9dcbd4c7006fcf8b73a", - "version": "2.2.0" - } - }, - { - "package": "WebRTC", - "repositoryURL": "https://github.com/webrtc-sdk/Specs.git", - "state": { - "branch": null, - "revision": "4fa8d6d647fc759cdd0265fd413d2f28ea2e0e08", - "version": "114.5735.8" - } - }, - { - "package": "swift-log", - "repositoryURL": "https://github.com/apple/swift-log.git", - "state": { - "branch": null, - "revision": "32e8d724467f8fe623624570367e3d50c5638e46", - "version": "1.5.2" - } - }, - { - "package": "SwiftProtobuf", - "repositoryURL": "https://github.com/apple/swift-protobuf.git", - "state": { - "branch": null, - "revision": "ce20dc083ee485524b802669890291c0d8090170", - "version": "1.22.1" - } - } - ] - }, - "version": 1 -} diff --git a/crates/livekit_client_macos/LiveKitBridge/Package.swift b/crates/livekit_client_macos/LiveKitBridge/Package.swift deleted file mode 100644 index a2a5b3eb75..0000000000 --- a/crates/livekit_client_macos/LiveKitBridge/Package.swift +++ /dev/null @@ -1,27 +0,0 @@ -// swift-tools-version: 5.5 - -import PackageDescription - -let package = Package( - name: "LiveKitBridge", - platforms: [ - .macOS(.v10_15) - ], - products: [ - // Products define the executables and libraries a package produces, and make them visible to other packages. - .library( - name: "LiveKitBridge", - type: .static, - targets: ["LiveKitBridge"]) - ], - dependencies: [ - .package(url: "https://github.com/livekit/client-sdk-swift.git", .exact("1.1.6")) - ], - targets: [ - // Targets are the basic building blocks of a package. A target can define a module or a test suite. - // Targets can depend on other targets in this package, and on products in packages this package depends on. - .target( - name: "LiveKitBridge", - dependencies: [.product(name: "LiveKit", package: "client-sdk-swift")]) - ] -) diff --git a/crates/livekit_client_macos/LiveKitBridge/README.md b/crates/livekit_client_macos/LiveKitBridge/README.md deleted file mode 100644 index b982c67286..0000000000 --- a/crates/livekit_client_macos/LiveKitBridge/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# LiveKitBridge - -A description of this package. diff --git a/crates/livekit_client_macos/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift b/crates/livekit_client_macos/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift deleted file mode 100644 index 7468c08791..0000000000 --- a/crates/livekit_client_macos/LiveKitBridge/Sources/LiveKitBridge/LiveKitBridge.swift +++ /dev/null @@ -1,383 +0,0 @@ -import Foundation -import LiveKit -import WebRTC -import ScreenCaptureKit - -class LKRoomDelegate: RoomDelegate { - var data: UnsafeRawPointer - var onDidDisconnect: @convention(c) (UnsafeRawPointer) -> Void - var onDidSubscribeToRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onMuteChangedFromRemoteAudioTrack: @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void - var onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void - var onDidSubscribeToRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void - var onDidUnsubscribeFromRemoteVideoTrack: @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void - var onDidPublishOrUnpublishLocalAudioTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - var onDidPublishOrUnpublishLocalVideoTrack: @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - - init( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakersChanged: @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void - ) - { - self.data = data - self.onDidDisconnect = onDidDisconnect - self.onDidSubscribeToRemoteAudioTrack = onDidSubscribeToRemoteAudioTrack - self.onDidUnsubscribeFromRemoteAudioTrack = onDidUnsubscribeFromRemoteAudioTrack - self.onDidSubscribeToRemoteVideoTrack = onDidSubscribeToRemoteVideoTrack - self.onDidUnsubscribeFromRemoteVideoTrack = onDidUnsubscribeFromRemoteVideoTrack - self.onMuteChangedFromRemoteAudioTrack = onMuteChangedFromRemoteAudioTrack - self.onActiveSpeakersChanged = onActiveSpeakersChanged - self.onDidPublishOrUnpublishLocalAudioTrack = onDidPublishOrUnpublishLocalAudioTrack - self.onDidPublishOrUnpublishLocalVideoTrack = onDidPublishOrUnpublishLocalVideoTrack - } - - func room(_ room: Room, didUpdate connectionState: ConnectionState, oldValue: ConnectionState) { - if connectionState.isDisconnected { - self.onDidDisconnect(self.data) - } - } - - func room(_ room: Room, participant: RemoteParticipant, didSubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidSubscribeToRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque()) - } else if track.kind == .audio { - self.onDidSubscribeToRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString, Unmanaged.passUnretained(track).toOpaque(), Unmanaged.passUnretained(publication).toOpaque()) - } - } - - func room(_ room: Room, participant: Participant, didUpdate publication: TrackPublication, muted: Bool) { - if publication.kind == .audio { - self.onMuteChangedFromRemoteAudioTrack(self.data, publication.sid as CFString, muted) - } - } - - func room(_ room: Room, didUpdate speakers: [Participant]) { - guard let speaker_ids = speakers.compactMap({ $0.identity as CFString }) as CFArray? else { return } - self.onActiveSpeakersChanged(self.data, speaker_ids) - } - - func room(_ room: Room, participant: RemoteParticipant, didUnsubscribe publication: RemoteTrackPublication, track: Track) { - if track.kind == .video { - self.onDidUnsubscribeFromRemoteVideoTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } else if track.kind == .audio { - self.onDidUnsubscribeFromRemoteAudioTrack(self.data, participant.identity as CFString, track.sid! as CFString) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didPublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), true) - } - } - - func room(_ room: Room, localParticipant: LocalParticipant, didUnpublish publication: LocalTrackPublication) { - if publication.kind == .video { - self.onDidPublishOrUnpublishLocalVideoTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } else if publication.kind == .audio { - self.onDidPublishOrUnpublishLocalAudioTrack(self.data, Unmanaged.passUnretained(publication).toOpaque(), false) - } - } -} - -class LKVideoRenderer: NSObject, VideoRenderer { - var data: UnsafeRawPointer - var onFrame: @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool - var onDrop: @convention(c) (UnsafeRawPointer) -> Void - var adaptiveStreamIsEnabled: Bool = false - var adaptiveStreamSize: CGSize = .zero - weak var track: VideoTrack? - - init(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) { - self.data = data - self.onFrame = onFrame - self.onDrop = onDrop - } - - deinit { - self.onDrop(self.data) - } - - func setSize(_ size: CGSize) { - } - - func renderFrame(_ frame: RTCVideoFrame?) { - let buffer = frame?.buffer as? RTCCVPixelBuffer - if let pixelBuffer = buffer?.pixelBuffer { - if !self.onFrame(self.data, pixelBuffer) { - DispatchQueue.main.async { - self.track?.remove(videoRenderer: self) - } - } - } - } -} - -@_cdecl("LKRoomDelegateCreate") -public func LKRoomDelegateCreate( - data: UnsafeRawPointer, - onDidDisconnect: @escaping @convention(c) (UnsafeRawPointer) -> Void, - onDidSubscribeToRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onMuteChangedFromRemoteAudioTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, Bool) -> Void, - onActiveSpeakerChanged: @escaping @convention(c) (UnsafeRawPointer, CFArray) -> Void, - onDidSubscribeToRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString, UnsafeRawPointer) -> Void, - onDidUnsubscribeFromRemoteVideoTrack: @escaping @convention(c) (UnsafeRawPointer, CFString, CFString) -> Void, - onDidPublishOrUnpublishLocalAudioTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void, - onDidPublishOrUnpublishLocalVideoTrack: @escaping @convention(c) (UnsafeRawPointer, UnsafeRawPointer, Bool) -> Void -) -> UnsafeMutableRawPointer { - let delegate = LKRoomDelegate( - data: data, - onDidDisconnect: onDidDisconnect, - onDidSubscribeToRemoteAudioTrack: onDidSubscribeToRemoteAudioTrack, - onDidUnsubscribeFromRemoteAudioTrack: onDidUnsubscribeFromRemoteAudioTrack, - onMuteChangedFromRemoteAudioTrack: onMuteChangedFromRemoteAudioTrack, - onActiveSpeakersChanged: onActiveSpeakerChanged, - onDidSubscribeToRemoteVideoTrack: onDidSubscribeToRemoteVideoTrack, - onDidUnsubscribeFromRemoteVideoTrack: onDidUnsubscribeFromRemoteVideoTrack, - onDidPublishOrUnpublishLocalAudioTrack: onDidPublishOrUnpublishLocalAudioTrack, - onDidPublishOrUnpublishLocalVideoTrack: onDidPublishOrUnpublishLocalVideoTrack - ) - return Unmanaged.passRetained(delegate).toOpaque() -} - -@_cdecl("LKRoomCreate") -public func LKRoomCreate(delegate: UnsafeRawPointer) -> UnsafeMutableRawPointer { - let delegate = Unmanaged.fromOpaque(delegate).takeUnretainedValue() - return Unmanaged.passRetained(Room(delegate: delegate)).toOpaque() -} - -@_cdecl("LKRoomConnect") -public func LKRoomConnect(room: UnsafeRawPointer, url: CFString, token: CFString, callback: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - room.connect(url as String, token as String).then { _ in - callback(callback_data, UnsafeRawPointer(nil) as! CFString?) - }.catch { error in - callback(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomDisconnect") -public func LKRoomDisconnect(room: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - room.disconnect() -} - -@_cdecl("LKRoomPublishVideoTrack") -public func LKRoomPublishVideoTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishVideoTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRoomPublishAudioTrack") -public func LKRoomPublishAudioTrack(room: UnsafeRawPointer, track: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, UnsafeMutableRawPointer?, CFString?) -> Void, callback_data: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - room.localParticipant?.publishAudioTrack(track: track).then { publication in - callback(callback_data, Unmanaged.passRetained(publication).toOpaque(), nil) - }.catch { error in - callback(callback_data, nil, error.localizedDescription as CFString) - } -} - - -@_cdecl("LKRoomUnpublishTrack") -public func LKRoomUnpublishTrack(room: UnsafeRawPointer, publication: UnsafeRawPointer) { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - let _ = room.localParticipant?.unpublish(publication: publication) -} - -@_cdecl("LKRoomAudioTracksForRemoteParticipant") -public func LKRoomAudioTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0.track as? RemoteAudioTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomAudioTrackPublicationsForRemoteParticipant") -public func LKRoomAudioTrackPublicationsForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.audioTracks.compactMap { $0 as? RemoteTrackPublication } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKRoomVideoTracksForRemoteParticipant") -public func LKRoomVideoTracksForRemoteParticipant(room: UnsafeRawPointer, participantId: CFString) -> CFArray? { - let room = Unmanaged.fromOpaque(room).takeUnretainedValue() - - for (_, participant) in room.remoteParticipants { - if participant.identity == participantId as String { - return participant.videoTracks.compactMap { $0.track as? RemoteVideoTrack } as CFArray? - } - } - - return nil; -} - -@_cdecl("LKLocalAudioTrackCreateTrack") -public func LKLocalAudioTrackCreateTrack() -> UnsafeMutableRawPointer { - let track = LocalAudioTrack.createTrack(options: AudioCaptureOptions( - echoCancellation: true, - noiseSuppression: true - )) - - return Unmanaged.passRetained(track).toOpaque() -} - - -@_cdecl("LKCreateScreenShareTrackForDisplay") -public func LKCreateScreenShareTrackForDisplay(display: UnsafeMutableRawPointer) -> UnsafeMutableRawPointer { - let display = Unmanaged.fromOpaque(display).takeUnretainedValue() - let track = LocalVideoTrack.createMacOSScreenShareTrack(source: display, preferredMethod: .legacy) - return Unmanaged.passRetained(track).toOpaque() -} - -@_cdecl("LKVideoRendererCreate") -public func LKVideoRendererCreate(data: UnsafeRawPointer, onFrame: @escaping @convention(c) (UnsafeRawPointer, CVPixelBuffer) -> Bool, onDrop: @escaping @convention(c) (UnsafeRawPointer) -> Void) -> UnsafeMutableRawPointer { - Unmanaged.passRetained(LKVideoRenderer(data: data, onFrame: onFrame, onDrop: onDrop)).toOpaque() -} - -@_cdecl("LKVideoTrackAddRenderer") -public func LKVideoTrackAddRenderer(track: UnsafeRawPointer, renderer: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() as! VideoTrack - let renderer = Unmanaged.fromOpaque(renderer).takeRetainedValue() - renderer.track = track - track.add(videoRenderer: renderer) -} - -@_cdecl("LKRemoteVideoTrackGetSid") -public func LKRemoteVideoTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackGetSid") -public func LKRemoteAudioTrackGetSid(track: UnsafeRawPointer) -> CFString { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - return track.sid! as CFString -} - -@_cdecl("LKRemoteAudioTrackStart") -public func LKRemoteAudioTrackStart(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.start() -} - -@_cdecl("LKRemoteAudioTrackStop") -public func LKRemoteAudioTrackStop(track: UnsafeRawPointer) { - let track = Unmanaged.fromOpaque(track).takeUnretainedValue() - track.stop() -} - -@_cdecl("LKDisplaySources") -public func LKDisplaySources(data: UnsafeRawPointer, callback: @escaping @convention(c) (UnsafeRawPointer, CFArray?, CFString?) -> Void) { - MacOSScreenCapturer.sources(for: .display, includeCurrentApplication: false, preferredMethod: .legacy).then { displaySources in - callback(data, displaySources as CFArray, nil) - }.catch { error in - callback(data, nil, error.localizedDescription as CFString) - } -} - -@_cdecl("LKLocalTrackPublicationSetMute") -public func LKLocalTrackPublicationSetMute( - publication: UnsafeRawPointer, - muted: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - if muted { - publication.mute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } else { - publication.unmute().then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } - } -} - -@_cdecl("LKLocalTrackPublicationIsMuted") -public func LKLocalTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationSetEnabled") -public func LKRemoteTrackPublicationSetEnabled( - publication: UnsafeRawPointer, - enabled: Bool, - on_complete: @escaping @convention(c) (UnsafeRawPointer, CFString?) -> Void, - callback_data: UnsafeRawPointer -) { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - publication.set(enabled: enabled).then { - on_complete(callback_data, nil) - }.catch { error in - on_complete(callback_data, error.localizedDescription as CFString) - } -} - -@_cdecl("LKRemoteTrackPublicationIsMuted") -public func LKRemoteTrackPublicationIsMuted( - publication: UnsafeRawPointer -) -> Bool { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.muted -} - -@_cdecl("LKRemoteTrackPublicationGetSid") -public func LKRemoteTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} - -@_cdecl("LKLocalTrackPublicationGetSid") -public func LKLocalTrackPublicationGetSid( - publication: UnsafeRawPointer -) -> CFString { - let publication = Unmanaged.fromOpaque(publication).takeUnretainedValue() - - return publication.sid as CFString -} diff --git a/crates/livekit_client_macos/build.rs b/crates/livekit_client_macos/build.rs deleted file mode 100644 index 2fdfd982bf..0000000000 --- a/crates/livekit_client_macos/build.rs +++ /dev/null @@ -1,185 +0,0 @@ -use serde::Deserialize; -use std::{ - env, - path::{Path, PathBuf}, - process::Command, -}; - -const SWIFT_PACKAGE_NAME: &str = "LiveKitBridge"; - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftTargetInfo { - pub triple: String, - pub unversioned_triple: String, - pub module_triple: String, - pub swift_runtime_compatibility_version: String, - #[serde(rename = "librariesRequireRPath")] - pub libraries_require_rpath: bool, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SwiftPaths { - pub runtime_library_paths: Vec, - pub runtime_library_import_paths: Vec, - pub runtime_resource_path: String, -} - -#[derive(Debug, Deserialize)] -pub struct SwiftTarget { - pub target: SwiftTargetInfo, - pub paths: SwiftPaths, -} - -const MACOS_TARGET_VERSION: &str = "10.15.7"; - -fn main() { - if cfg!(all( - target_os = "macos", - not(any(test, feature = "test-support", feature = "no-webrtc")), - )) { - let swift_target = get_swift_target(); - - build_bridge(&swift_target); - link_swift_stdlib(&swift_target); - link_webrtc_framework(&swift_target); - - // Register exported Objective-C selectors, protocols, etc when building example binaries. - println!("cargo:rustc-link-arg=-Wl,-ObjC"); - } -} - -fn build_bridge(swift_target: &SwiftTarget) { - println!("cargo:rerun-if-env-changed=MACOSX_DEPLOYMENT_TARGET"); - println!("cargo:rerun-if-changed={}/Sources", SWIFT_PACKAGE_NAME); - println!( - "cargo:rerun-if-changed={}/Package.swift", - SWIFT_PACKAGE_NAME - ); - println!( - "cargo:rerun-if-changed={}/Package.resolved", - SWIFT_PACKAGE_NAME - ); - - let swift_package_root = swift_package_root(); - let swift_target_folder = swift_target_folder(); - let swift_cache_folder = swift_cache_folder(); - if !Command::new("swift") - .arg("build") - .arg("--disable-automatic-resolution") - .args(["--configuration", &env::var("PROFILE").unwrap()]) - .args(["--triple", &swift_target.target.triple]) - .args(["--build-path".into(), swift_target_folder]) - .args(["--cache-path".into(), swift_cache_folder]) - .current_dir(&swift_package_root) - .status() - .unwrap() - .success() - { - panic!( - "Failed to compile swift package in {}", - swift_package_root.display() - ); - } - - println!( - "cargo:rustc-link-search=native={}", - swift_target.out_dir_path().display() - ); - println!("cargo:rustc-link-lib=static={}", SWIFT_PACKAGE_NAME); -} - -fn link_swift_stdlib(swift_target: &SwiftTarget) { - for path in &swift_target.paths.runtime_library_paths { - println!("cargo:rustc-link-search=native={}", path); - } -} - -fn link_webrtc_framework(swift_target: &SwiftTarget) { - let swift_out_dir_path = swift_target.out_dir_path(); - println!("cargo:rustc-link-lib=framework=WebRTC"); - println!( - "cargo:rustc-link-search=framework={}", - swift_out_dir_path.display() - ); - // Find WebRTC.framework as a sibling of the executable when running tests. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - // Find WebRTC.framework in parent directory of the executable when running examples. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/.."); - - let source_path = swift_out_dir_path.join("WebRTC.framework"); - let deps_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../deps/WebRTC.framework"); - let target_dir_path = - PathBuf::from(env::var("OUT_DIR").unwrap()).join("../../../WebRTC.framework"); - copy_dir(&source_path, &deps_dir_path); - copy_dir(&source_path, &target_dir_path); -} - -fn get_swift_target() -> SwiftTarget { - let mut arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); - if arch == "aarch64" { - arch = "arm64".into(); - } - let target = format!("{}-apple-macosx{}", arch, MACOS_TARGET_VERSION); - - let swift_target_info_str = Command::new("swift") - .args(["-target", &target, "-print-target-info"]) - .output() - .unwrap() - .stdout; - - serde_json::from_slice(&swift_target_info_str).unwrap() -} - -fn swift_package_root() -> PathBuf { - env::current_dir().unwrap().join(SWIFT_PACKAGE_NAME) -} - -fn swift_target_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_target")) -} - -fn swift_cache_folder() -> PathBuf { - let target = env::var("TARGET").unwrap(); - env::current_dir() - .unwrap() - .join(format!("../../target/{target}/{SWIFT_PACKAGE_NAME}_cache")) -} - -fn copy_dir(source: &Path, destination: &Path) { - assert!( - Command::new("rm") - .arg("-rf") - .arg(destination) - .status() - .unwrap() - .success(), - "could not remove {:?} before copying", - destination - ); - - assert!( - Command::new("cp") - .arg("-R") - .args([source, destination]) - .status() - .unwrap() - .success(), - "could not copy {:?} to {:?}", - source, - destination - ); -} - -impl SwiftTarget { - fn out_dir_path(&self) -> PathBuf { - swift_target_folder() - .join(&self.target.unversioned_triple) - .join(env::var("PROFILE").unwrap()) - } -} diff --git a/crates/livekit_client_macos/examples/test_app_macos.rs b/crates/livekit_client_macos/examples/test_app_macos.rs deleted file mode 100644 index df0e0a3769..0000000000 --- a/crates/livekit_client_macos/examples/test_app_macos.rs +++ /dev/null @@ -1,172 +0,0 @@ -use std::time::Duration; - -use futures::StreamExt; -use gpui::{actions, KeyBinding, Menu, MenuItem}; -use livekit_api::token::{self, VideoGrant}; -use livekit_client_macos::{LocalAudioTrack, LocalVideoTrack, Room, RoomUpdate}; -use log::LevelFilter; -use simplelog::SimpleLogger; - -actions!(livekit_client_macos, [Quit]); - -fn main() { - SimpleLogger::init(LevelFilter::Info, Default::default()).expect("could not initialize logger"); - - gpui::Application::new().run(|cx| { - #[cfg(any(test, feature = "test-support"))] - println!("USING TEST LIVEKIT"); - - #[cfg(not(any(test, feature = "test-support")))] - println!("USING REAL LIVEKIT"); - - cx.activate(true); - - cx.on_action(quit); - cx.bind_keys([KeyBinding::new("cmd-q", Quit, None)]); - - cx.set_menus(vec![Menu { - name: "Zed".into(), - items: vec![MenuItem::Action { - name: "Quit".into(), - action: Box::new(Quit), - os_action: None, - }], - }]); - - let live_kit_url = std::env::var("LIVE_KIT_URL").unwrap_or("http://localhost:7880".into()); - let live_kit_key = std::env::var("LIVE_KIT_KEY").unwrap_or("devkey".into()); - let live_kit_secret = std::env::var("LIVE_KIT_SECRET").unwrap_or("secret".into()); - - cx.spawn(async move |cx| { - let user_a_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-1"), - VideoGrant::to_join("test-room"), - ) - .unwrap(); - let room_a = Room::new(); - room_a.connect(&live_kit_url, &user_a_token).await.unwrap(); - - let user2_token = token::create( - &live_kit_key, - &live_kit_secret, - Some("test-participant-2"), - VideoGrant::to_join("test-room"), - ) - .unwrap(); - let room_b = Room::new(); - room_b.connect(&live_kit_url, &user2_token).await.unwrap(); - - let mut room_updates = room_b.updates(); - let audio_track = LocalAudioTrack::create(); - let audio_track_publication = room_a.publish_audio_track(audio_track).await.unwrap(); - - if let RoomUpdate::SubscribedToRemoteAudioTrack(track, _) = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks.len(), 1); - assert_eq!(remote_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); - } - - audio_track_publication.set_mute(true).await.unwrap(); - - println!("waiting for mute changed!"); - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(muted); - } else { - panic!("unexpected message"); - } - - audio_track_publication.set_mute(false).await.unwrap(); - - if let RoomUpdate::RemoteAudioTrackMuteChanged { track_id, muted } = - room_updates.next().await.unwrap() - { - let remote_tracks = room_b.remote_audio_tracks("test-participant-1"); - assert_eq!(remote_tracks[0].sid(), track_id); - assert!(!muted); - } else { - panic!("unexpected message"); - } - - println!("Pausing for 5 seconds to test audio, make some noise!"); - let timer = cx.background_executor().timer(Duration::from_secs(5)); - timer.await; - let remote_audio_track = room_b - .remote_audio_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(audio_track_publication); - - // Clear out any active speakers changed messages - let mut next = room_updates.next().await.unwrap(); - while let RoomUpdate::ActiveSpeakersChanged { speakers } = next { - println!("Speakers changed: {:?}", speakers); - next = room_updates.next().await.unwrap(); - } - - if let RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id, - track_id, - } = next - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_audio_track.sid(), track_id); - assert_eq!(room_b.remote_audio_tracks("test-participant-1").len(), 0); - } else { - panic!("unexpected message"); - } - - let displays = room_a.display_sources().await.unwrap(); - let display = displays.into_iter().next().unwrap(); - - let local_video_track = LocalVideoTrack::screen_share_for_display(&display); - let local_video_track_publication = - room_a.publish_video_track(local_video_track).await.unwrap(); - - if let RoomUpdate::SubscribedToRemoteVideoTrack(track) = - room_updates.next().await.unwrap() - { - let remote_video_tracks = room_b.remote_video_tracks("test-participant-1"); - assert_eq!(remote_video_tracks.len(), 1); - assert_eq!(remote_video_tracks[0].publisher_id(), "test-participant-1"); - assert_eq!(track.publisher_id(), "test-participant-1"); - } else { - panic!("unexpected message"); - } - - let remote_video_track = room_b - .remote_video_tracks("test-participant-1") - .pop() - .unwrap(); - room_a.unpublish_track(local_video_track_publication); - if let RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id, - track_id, - } = room_updates.next().await.unwrap() - { - assert_eq!(publisher_id, "test-participant-1"); - assert_eq!(remote_video_track.sid(), track_id); - assert_eq!(room_b.remote_video_tracks("test-participant-1").len(), 0); - } else { - panic!("unexpected message"); - } - - cx.update(|cx| cx.shutdown()).ok(); - }) - .detach(); - }); -} - -fn quit(_: &Quit, cx: &mut gpui::App) { - cx.quit(); -} diff --git a/crates/livekit_client_macos/src/livekit_client.rs b/crates/livekit_client_macos/src/livekit_client.rs deleted file mode 100644 index 4820a4eedb..0000000000 --- a/crates/livekit_client_macos/src/livekit_client.rs +++ /dev/null @@ -1,37 +0,0 @@ -#![allow(clippy::arc_with_non_send_sync)] - -use std::sync::Arc; - -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub mod prod; - -#[cfg(all(target_os = "macos", not(any(test, feature = "test-support"))))] -pub use prod::*; - -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub mod test; - -#[cfg(any(test, feature = "test-support", not(target_os = "macos")))] -pub use test::*; - -pub type Sid = String; - -#[derive(Clone, Eq, PartialEq)] -pub enum ConnectionState { - Disconnected, - Connected { url: String, token: String }, -} - -#[derive(Clone)] -pub enum RoomUpdate { - ActiveSpeakersChanged { speakers: Vec }, - RemoteAudioTrackMuteChanged { track_id: Sid, muted: bool }, - SubscribedToRemoteVideoTrack(Arc), - SubscribedToRemoteAudioTrack(Arc, Arc), - UnsubscribedFromRemoteVideoTrack { publisher_id: Sid, track_id: Sid }, - UnsubscribedFromRemoteAudioTrack { publisher_id: Sid, track_id: Sid }, - LocalAudioTrackPublished { publication: LocalTrackPublication }, - LocalAudioTrackUnpublished { publication: LocalTrackPublication }, - LocalVideoTrackPublished { publication: LocalTrackPublication }, - LocalVideoTrackUnpublished { publication: LocalTrackPublication }, -} diff --git a/crates/livekit_client_macos/src/prod.rs b/crates/livekit_client_macos/src/prod.rs deleted file mode 100644 index cc577b9574..0000000000 --- a/crates/livekit_client_macos/src/prod.rs +++ /dev/null @@ -1,981 +0,0 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; -use anyhow::{anyhow, Context as _, Result}; -use core_foundation::{ - array::{CFArray, CFArrayRef}, - base::{CFRelease, CFRetain, TCFType}, - string::{CFString, CFStringRef}, -}; -use futures::{ - channel::{mpsc, oneshot}, - Future, -}; -pub use media::core_video::CVImageBuffer; -use media::core_video::CVImageBufferRef; -use parking_lot::Mutex; -use postage::watch; -use std::{ - ffi::c_void, - sync::{Arc, Weak}, -}; - -macro_rules! pointer_type { - ($pointer_name:ident) => { - #[repr(transparent)] - #[derive(Copy, Clone, Debug)] - pub struct $pointer_name(pub *const std::ffi::c_void); - unsafe impl Send for $pointer_name {} - }; -} - -mod swift { - pointer_type!(Room); - pointer_type!(LocalAudioTrack); - pointer_type!(RemoteAudioTrack); - pointer_type!(LocalVideoTrack); - pointer_type!(RemoteVideoTrack); - pointer_type!(LocalTrackPublication); - pointer_type!(RemoteTrackPublication); - pointer_type!(MacOSDisplay); - pointer_type!(RoomDelegate); -} - -extern "C" { - fn LKRoomDelegateCreate( - callback_data: *mut c_void, - on_did_disconnect: extern "C" fn(callback_data: *mut c_void), - on_did_subscribe_to_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteAudioTrack, - remote_publication: swift::RemoteTrackPublication, - ), - on_did_unsubscribe_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_mute_changed_from_remote_audio_track: extern "C" fn( - callback_data: *mut c_void, - track_id: CFStringRef, - muted: bool, - ), - on_active_speakers_changed: extern "C" fn( - callback_data: *mut c_void, - participants: CFArrayRef, - ), - on_did_subscribe_to_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - remote_track: swift::RemoteVideoTrack, - ), - on_did_unsubscribe_from_remote_video_track: extern "C" fn( - callback_data: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ), - on_did_publish_or_unpublish_local_audio_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - on_did_publish_or_unpublish_local_video_track: extern "C" fn( - callback_data: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ), - ) -> swift::RoomDelegate; - - fn LKRoomCreate(delegate: swift::RoomDelegate) -> swift::Room; - fn LKRoomConnect( - room: swift::Room, - url: CFStringRef, - token: CFStringRef, - callback: extern "C" fn(*mut c_void, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomDisconnect(room: swift::Room); - fn LKRoomPublishVideoTrack( - room: swift::Room, - track: swift::LocalVideoTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomPublishAudioTrack( - room: swift::Room, - track: swift::LocalAudioTrack, - callback: extern "C" fn(*mut c_void, swift::LocalTrackPublication, CFStringRef), - callback_data: *mut c_void, - ); - fn LKRoomUnpublishTrack(room: swift::Room, publication: swift::LocalTrackPublication); - - fn LKRoomAudioTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomAudioTrackPublicationsForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKRoomVideoTracksForRemoteParticipant( - room: swift::Room, - participant_id: CFStringRef, - ) -> CFArrayRef; - - fn LKVideoRendererCreate( - callback_data: *mut c_void, - on_frame: extern "C" fn(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool, - on_drop: extern "C" fn(callback_data: *mut c_void), - ) -> *const c_void; - - fn LKRemoteAudioTrackGetSid(track: swift::RemoteAudioTrack) -> CFStringRef; - fn LKRemoteVideoTrackGetSid(track: swift::RemoteVideoTrack) -> CFStringRef; - fn LKRemoteAudioTrackStart(track: swift::RemoteAudioTrack); - fn LKRemoteAudioTrackStop(track: swift::RemoteAudioTrack); - fn LKVideoTrackAddRenderer(track: swift::RemoteVideoTrack, renderer: *const c_void); - - fn LKDisplaySources( - callback_data: *mut c_void, - callback: extern "C" fn( - callback_data: *mut c_void, - sources: CFArrayRef, - error: CFStringRef, - ), - ); - fn LKCreateScreenShareTrackForDisplay(display: swift::MacOSDisplay) -> swift::LocalVideoTrack; - fn LKLocalAudioTrackCreateTrack() -> swift::LocalAudioTrack; - - fn LKLocalTrackPublicationSetMute( - publication: swift::LocalTrackPublication, - muted: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKRemoteTrackPublicationSetEnabled( - publication: swift::RemoteTrackPublication, - enabled: bool, - on_complete: extern "C" fn(callback_data: *mut c_void, error: CFStringRef), - callback_data: *mut c_void, - ); - - fn LKLocalTrackPublicationIsMuted(publication: swift::LocalTrackPublication) -> bool; - fn LKRemoteTrackPublicationIsMuted(publication: swift::RemoteTrackPublication) -> bool; - fn LKLocalTrackPublicationGetSid(publication: swift::LocalTrackPublication) -> CFStringRef; - fn LKRemoteTrackPublicationGetSid(publication: swift::RemoteTrackPublication) -> CFStringRef; -} - -pub struct Room { - native_room: swift::Room, - connection: Mutex<( - watch::Sender, - watch::Receiver, - )>, - update_subscribers: Mutex>>, - _delegate: RoomDelegate, -} - -impl Room { - pub fn new() -> Arc { - Arc::new_cyclic(|weak_room| { - let delegate = RoomDelegate::new(weak_room.clone()); - Self { - native_room: unsafe { LKRoomCreate(delegate.native_delegate) }, - connection: Mutex::new(watch::channel_with(ConnectionState::Disconnected)), - update_subscribers: Default::default(), - _delegate: delegate, - } - }) - } - - pub fn status(&self) -> watch::Receiver { - self.connection.lock().1.clone() - } - - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let url = CFString::new(url); - let token = CFString::new(token); - let (did_connect, tx, rx) = Self::build_done_callback(); - unsafe { - LKRoomConnect( - self.native_room, - url.as_concrete_TypeRef(), - token.as_concrete_TypeRef(), - did_connect, - tx, - ) - } - - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - rx.await.unwrap().context("error connecting to room")?; - *this.connection.lock().0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } - } - - fn did_disconnect(&self) { - *self.connection.lock().0.borrow_mut() = ConnectionState::Disconnected; - } - - pub fn display_sources(self: &Arc) -> impl Future>> { - extern "C" fn callback(tx: *mut c_void, sources: CFArrayRef, error: CFStringRef) { - unsafe { - let tx = Box::from_raw(tx as *mut oneshot::Sender>>); - - if sources.is_null() { - let _ = tx.send(Err(anyhow!("{}", CFString::wrap_under_get_rule(error)))); - } else { - let sources = CFArray::wrap_under_get_rule(sources) - .into_iter() - .map(|source| MacOSDisplay::new(swift::MacOSDisplay(*source))) - .collect(); - - let _ = tx.send(Ok(sources)); - } - } - } - - let (tx, rx) = oneshot::channel(); - - unsafe { - LKDisplaySources(Box::into_raw(Box::new(tx)) as *mut _, callback); - } - - async move { rx.await.unwrap() } - } - - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishVideoTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing video track") } - } - - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let (tx, rx) = oneshot::channel::>(); - extern "C" fn callback( - tx: *mut c_void, - publication: swift::LocalTrackPublication, - error: CFStringRef, - ) { - let tx = - unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(LocalTrackPublication::new(publication))); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - unsafe { - LKRoomPublishAudioTrack( - self.native_room, - track.0, - callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ); - } - async { rx.await.unwrap().context("error publishing audio track") } - } - - pub fn unpublish_track(&self, publication: LocalTrackPublication) { - unsafe { - LKRoomUnpublishTrack(self.native_room, publication.0); - } - } - - pub fn remote_video_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomVideoTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteVideoTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteVideoTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteVideoTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_tracks(&self, participant_id: &str) -> Vec> { - unsafe { - let tracks = LKRoomAudioTracksForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track| { - let native_track = swift::RemoteAudioTrack(*native_track); - let id = - CFString::wrap_under_get_rule(LKRemoteAudioTrackGetSid(native_track)) - .to_string(); - Arc::new(RemoteAudioTrack::new( - native_track, - id, - participant_id.into(), - )) - }) - .collect() - } - } - } - - pub fn remote_audio_track_publications( - &self, - participant_id: &str, - ) -> Vec> { - unsafe { - let tracks = LKRoomAudioTrackPublicationsForRemoteParticipant( - self.native_room, - CFString::new(participant_id).as_concrete_TypeRef(), - ); - - if tracks.is_null() { - Vec::new() - } else { - let tracks = CFArray::wrap_under_get_rule(tracks); - tracks - .into_iter() - .map(|native_track_publication| { - let native_track_publication = - swift::RemoteTrackPublication(*native_track_publication); - Arc::new(RemoteTrackPublication::new(native_track_publication)) - }) - .collect() - } - } - } - - pub fn updates(&self) -> mpsc::UnboundedReceiver { - let (tx, rx) = mpsc::unbounded(); - self.update_subscribers.lock().push(tx); - rx - } - - fn did_subscribe_to_remote_audio_track( - &self, - track: RemoteAudioTrack, - publication: RemoteTrackPublication, - ) { - let track = Arc::new(track); - let publication = Arc::new(publication); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteAudioTrack( - track.clone(), - publication.clone(), - )) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_audio_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteAudioTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn mute_changed_from_remote_audio_track(&self, track_id: String, muted: bool) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_id.clone(), - muted, - }) - .is_ok() - }); - } - - fn active_speakers_changed(&self, speakers: Vec) { - self.update_subscribers.lock().retain(move |tx| { - tx.unbounded_send(RoomUpdate::ActiveSpeakersChanged { - speakers: speakers.clone(), - }) - .is_ok() - }); - } - - fn did_subscribe_to_remote_video_track(&self, track: RemoteVideoTrack) { - let track = Arc::new(track); - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::SubscribedToRemoteVideoTrack(track.clone())) - .is_ok() - }); - } - - fn did_unsubscribe_from_remote_video_track(&self, publisher_id: String, track_id: String) { - self.update_subscribers.lock().retain(|tx| { - tx.unbounded_send(RoomUpdate::UnsubscribedFromRemoteVideoTrack { - publisher_id: publisher_id.clone(), - track_id: track_id.clone(), - }) - .is_ok() - }); - } - - fn build_done_callback() -> ( - extern "C" fn(*mut c_void, CFStringRef), - *mut c_void, - oneshot::Receiver>, - ) { - let (tx, rx) = oneshot::channel(); - extern "C" fn done_callback(tx: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(tx as *mut oneshot::Sender>) }; - if error.is_null() { - let _ = tx.send(Ok(())); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - let _ = tx.send(Err(anyhow!(error))); - } - } - ( - done_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - rx, - ) - } - - pub fn set_display_sources(&self, _: Vec) { - unreachable!("This is a test-only function") - } -} - -impl Drop for Room { - fn drop(&mut self) { - unsafe { - LKRoomDisconnect(self.native_room); - CFRelease(self.native_room.0); - } - } -} - -struct RoomDelegate { - native_delegate: swift::RoomDelegate, - weak_room: *mut c_void, -} - -impl RoomDelegate { - fn new(weak_room: Weak) -> Self { - let weak_room = weak_room.into_raw() as *mut c_void; - let native_delegate = unsafe { - LKRoomDelegateCreate( - weak_room, - Self::on_did_disconnect, - Self::on_did_subscribe_to_remote_audio_track, - Self::on_did_unsubscribe_from_remote_audio_track, - Self::on_mute_change_from_remote_audio_track, - Self::on_active_speakers_changed, - Self::on_did_subscribe_to_remote_video_track, - Self::on_did_unsubscribe_from_remote_video_track, - Self::on_did_publish_or_unpublish_local_audio_track, - Self::on_did_publish_or_unpublish_local_video_track, - ) - }; - Self { - native_delegate, - weak_room, - } - } - - extern "C" fn on_did_disconnect(room: *mut c_void) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - room.did_disconnect(); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteAudioTrack, - publication: swift::RemoteTrackPublication, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteAudioTrack::new(track, track_id, publisher_id); - let publication = RemoteTrackPublication::new(publication); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_audio_track(track, publication); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_audio_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_audio_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_mute_change_from_remote_audio_track( - room: *mut c_void, - track_id: CFStringRef, - muted: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.mute_changed_from_remote_audio_track(track_id, muted); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_active_speakers_changed(room: *mut c_void, participants: CFArrayRef) { - if participants.is_null() { - return; - } - - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let speakers = unsafe { - CFArray::wrap_under_get_rule(participants) - .into_iter() - .map( - |speaker: core_foundation::base::ItemRef<'_, *const c_void>| { - CFString::wrap_under_get_rule(*speaker as CFStringRef).to_string() - }, - ) - .collect() - }; - - if let Some(room) = room.upgrade() { - room.active_speakers_changed(speakers); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_subscribe_to_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - track: swift::RemoteVideoTrack, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - let track = RemoteVideoTrack::new(track, track_id, publisher_id); - if let Some(room) = room.upgrade() { - room.did_subscribe_to_remote_video_track(track); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_unsubscribe_from_remote_video_track( - room: *mut c_void, - publisher_id: CFStringRef, - track_id: CFStringRef, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - let publisher_id = unsafe { CFString::wrap_under_get_rule(publisher_id).to_string() }; - let track_id = unsafe { CFString::wrap_under_get_rule(track_id).to_string() }; - if let Some(room) = room.upgrade() { - room.did_unsubscribe_from_remote_video_track(publisher_id, track_id); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_audio_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalAudioTrackPublished { publication } - } else { - RoomUpdate::LocalAudioTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } - - extern "C" fn on_did_publish_or_unpublish_local_video_track( - room: *mut c_void, - publication: swift::LocalTrackPublication, - is_published: bool, - ) { - let room = unsafe { Weak::from_raw(room as *mut Room) }; - if let Some(room) = room.upgrade() { - let publication = LocalTrackPublication::new(publication); - let update = if is_published { - RoomUpdate::LocalVideoTrackPublished { publication } - } else { - RoomUpdate::LocalVideoTrackUnpublished { publication } - }; - room.update_subscribers - .lock() - .retain(|tx| tx.unbounded_send(update.clone()).is_ok()); - } - let _ = Weak::into_raw(room); - } -} - -impl Drop for RoomDelegate { - fn drop(&mut self) { - unsafe { - CFRelease(self.native_delegate.0); - let _ = Weak::from_raw(self.weak_room as *mut Room); - } - } -} - -pub struct LocalAudioTrack(swift::LocalAudioTrack); - -impl LocalAudioTrack { - pub fn create() -> Self { - Self(unsafe { LKLocalAudioTrackCreateTrack() }) - } -} - -impl Drop for LocalAudioTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalVideoTrack(swift::LocalVideoTrack); - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self(unsafe { LKCreateScreenShareTrackForDisplay(display.0) }) - } -} - -impl Drop for LocalVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct LocalTrackPublication(swift::LocalTrackPublication); - -impl LocalTrackPublication { - pub fn new(native_track_publication: swift::LocalTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKLocalTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn set_mute(&self, muted: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKLocalTrackPublicationSetMute( - self.0, - muted, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKLocalTrackPublicationIsMuted(self.0) } - } -} - -impl Clone for LocalTrackPublication { - fn clone(&self) -> Self { - unsafe { - CFRetain(self.0 .0); - } - Self(self.0) - } -} - -impl Drop for LocalTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -pub struct RemoteTrackPublication(swift::RemoteTrackPublication); - -impl RemoteTrackPublication { - pub fn new(native_track_publication: swift::RemoteTrackPublication) -> Self { - unsafe { - CFRetain(native_track_publication.0); - } - Self(native_track_publication) - } - - pub fn sid(&self) -> String { - unsafe { CFString::wrap_under_get_rule(LKRemoteTrackPublicationGetSid(self.0)).to_string() } - } - - pub fn is_muted(&self) -> bool { - unsafe { LKRemoteTrackPublicationIsMuted(self.0) } - } - - pub fn set_enabled(&self, enabled: bool) -> impl Future> { - let (tx, rx) = futures::channel::oneshot::channel(); - - extern "C" fn complete_callback(callback_data: *mut c_void, error: CFStringRef) { - let tx = unsafe { Box::from_raw(callback_data as *mut oneshot::Sender>) }; - if error.is_null() { - tx.send(Ok(())).ok(); - } else { - let error = unsafe { CFString::wrap_under_get_rule(error).to_string() }; - tx.send(Err(anyhow!(error))).ok(); - } - } - - unsafe { - LKRemoteTrackPublicationSetEnabled( - self.0, - enabled, - complete_callback, - Box::into_raw(Box::new(tx)) as *mut c_void, - ) - } - - async move { rx.await.unwrap() } - } -} - -impl Drop for RemoteTrackPublication { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - native_track: swift::RemoteAudioTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteAudioTrack { - fn new(native_track: swift::RemoteAudioTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn start(&self) { - unsafe { LKRemoteAudioTrackStart(self.native_track) } - } - - pub fn stop(&self) { - unsafe { LKRemoteAudioTrackStop(self.native_track) } - } -} - -impl Drop for RemoteAudioTrack { - fn drop(&mut self) { - // todo: uncomment this `CFRelease`, unless we find that it was causing - // the crash in the `livekit.multicast` thread. - // - // unsafe { CFRelease(self.native_track.0) } - let _ = self.native_track; - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - native_track: swift::RemoteVideoTrack, - sid: Sid, - publisher_id: String, -} - -impl RemoteVideoTrack { - fn new(native_track: swift::RemoteVideoTrack, sid: Sid, publisher_id: String) -> Self { - unsafe { - CFRetain(native_track.0); - } - Self { - native_track, - sid, - publisher_id, - } - } - - pub fn sid(&self) -> &str { - &self.sid - } - - pub fn publisher_id(&self) -> &str { - &self.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - extern "C" fn on_frame(callback_data: *mut c_void, frame: CVImageBufferRef) -> bool { - unsafe { - let tx = Box::from_raw(callback_data as *mut async_broadcast::Sender); - let buffer = CVImageBuffer::wrap_under_get_rule(frame); - let result = tx.try_broadcast(Frame(buffer)); - let _ = Box::into_raw(tx); - match result { - Ok(_) => true, - Err(async_broadcast::TrySendError::Closed(_)) - | Err(async_broadcast::TrySendError::Inactive(_)) => { - log::warn!("no active receiver for frame"); - false - } - Err(async_broadcast::TrySendError::Full(_)) => { - log::warn!("skipping frame as receiver is not keeping up"); - true - } - } - } - } - - extern "C" fn on_drop(callback_data: *mut c_void) { - unsafe { - let _ = Box::from_raw(callback_data as *mut async_broadcast::Sender); - } - } - - let (tx, rx) = async_broadcast::broadcast(64); - unsafe { - let renderer = LKVideoRendererCreate( - Box::into_raw(Box::new(tx)) as *mut c_void, - on_frame, - on_drop, - ); - LKVideoTrackAddRenderer(self.native_track, renderer); - rx - } - } -} - -impl Drop for RemoteVideoTrack { - fn drop(&mut self) { - unsafe { CFRelease(self.native_track.0) } - } -} - -pub struct MacOSDisplay(swift::MacOSDisplay); - -impl MacOSDisplay { - fn new(ptr: swift::MacOSDisplay) -> Self { - unsafe { - CFRetain(ptr.0); - } - Self(ptr) - } -} - -impl Drop for MacOSDisplay { - fn drop(&mut self) { - unsafe { CFRelease(self.0 .0) } - } -} - -#[derive(Clone)] -pub struct Frame(CVImageBuffer); - -impl Frame { - pub fn width(&self) -> usize { - self.0.width() - } - - pub fn height(&self) -> usize { - self.0.height() - } - - pub fn image(&self) -> CVImageBuffer { - self.0.clone() - } -} diff --git a/crates/livekit_client_macos/src/test.rs b/crates/livekit_client_macos/src/test.rs deleted file mode 100644 index bccc713c87..0000000000 --- a/crates/livekit_client_macos/src/test.rs +++ /dev/null @@ -1,882 +0,0 @@ -use crate::{ConnectionState, RoomUpdate, Sid}; -use anyhow::{anyhow, Context as _, Result}; -use async_trait::async_trait; -use collections::{btree_map::Entry as BTreeEntry, hash_map::Entry, BTreeMap, HashMap, HashSet}; -use futures::Stream; -use gpui::{BackgroundExecutor, SurfaceSource}; -use livekit_api::{proto, token}; - -use parking_lot::Mutex; -use postage::watch; -use std::{ - future::Future, - mem, - sync::{ - atomic::{AtomicBool, Ordering::SeqCst}, - Arc, Weak, - }, -}; - -static SERVERS: Mutex>> = Mutex::new(BTreeMap::new()); - -pub struct TestServer { - pub url: String, - pub api_key: String, - pub secret_key: String, - rooms: Mutex>, - executor: BackgroundExecutor, -} - -impl TestServer { - pub fn create( - url: String, - api_key: String, - secret_key: String, - executor: BackgroundExecutor, - ) -> Result> { - let mut servers = SERVERS.lock(); - if let BTreeEntry::Vacant(e) = servers.entry(url.clone()) { - let server = Arc::new(TestServer { - url, - api_key, - secret_key, - rooms: Default::default(), - executor, - }); - e.insert(server.clone()); - Ok(server) - } else { - Err(anyhow!("a server with url {:?} already exists", url)) - } - } - - fn get(url: &str) -> Result> { - Ok(SERVERS - .lock() - .get(url) - .ok_or_else(|| anyhow!("no server found for url"))? - .clone()) - } - - pub fn teardown(&self) -> Result<()> { - SERVERS - .lock() - .remove(&self.url) - .ok_or_else(|| anyhow!("server with url {:?} does not exist", self.url))?; - Ok(()) - } - - pub fn create_api_client(&self) -> TestApiClient { - TestApiClient { - url: self.url.clone(), - } - } - - pub async fn create_room(&self, room: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - if let Entry::Vacant(e) = server_rooms.entry(room.clone()) { - e.insert(Default::default()); - Ok(()) - } else { - Err(anyhow!("room {:?} already exists", room)) - } - } - - async fn delete_room(&self, room: String) -> Result<()> { - // TODO: clear state associated with all `Room`s. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - server_rooms - .remove(&room) - .ok_or_else(|| anyhow!("room {:?} does not exist", room))?; - Ok(()) - } - - async fn join_room(&self, token: String, client_room: Arc) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = (*server_rooms).entry(room_name.to_string()).or_default(); - - if let Entry::Vacant(e) = room.client_rooms.entry(identity.clone()) { - for track in &room.video_tracks { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), - }, - ))) - .unwrap(); - } - for track in &room.audio_tracks { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(&client_room), - }), - Arc::new(RemoteTrackPublication), - )) - .unwrap(); - } - e.insert(client_room); - Ok(()) - } else { - Err(anyhow!( - "{:?} attempted to join room {:?} twice", - identity, - room_name - )) - } - } - - async fn leave_room(&self, token: String) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms.remove(&identity).ok_or_else(|| { - anyhow!( - "{:?} attempted to leave room {:?} before joining it", - identity, - room_name - ) - })?; - Ok(()) - } - - async fn remove_participant(&self, room_name: String, identity: String) -> Result<()> { - // TODO: clear state associated with the `Room`. - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms.remove(&identity).ok_or_else(|| { - anyhow!( - "participant {:?} did not join room {:?}", - identity, - room_name - ) - })?; - Ok(()) - } - - async fn update_participant( - &self, - room_name: String, - identity: String, - permission: proto::ParticipantPermission, - ) -> Result<()> { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.participant_permissions.insert(identity, permission); - Ok(()) - } - - pub async fn disconnect_client(&self, client_identity: String) { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let mut server_rooms = self.rooms.lock(); - for room in server_rooms.values_mut() { - if let Some(room) = room.client_rooms.remove(&client_identity) { - *room.0.lock().connection.0.borrow_mut() = ConnectionState::Disconnected; - } - } - } - - async fn publish_video_track( - &self, - token: String, - local_track: LocalVideoTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - - let can_publish = room - .participant_permissions - .get(&identity) - .map(|permission| permission.can_publish) - .or(claims.video.can_publish) - .unwrap_or(true); - - if !can_publish { - return Err(anyhow!("user is not allowed to publish")); - } - - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerVideoTrack { - sid: sid.clone(), - publisher_id: identity.clone(), - frames_rx: local_track.frames_rx.clone(), - }); - - room.video_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteVideoTrack(Arc::new( - RemoteVideoTrack { - server_track: track.clone(), - }, - ))) - .unwrap(); - } - } - - Ok(sid) - } - - async fn publish_audio_track( - &self, - token: String, - _local_track: &LocalAudioTrack, - ) -> Result { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - self.executor.simulate_random_delay().await; - - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let identity = claims.sub.unwrap().to_string(); - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - - let can_publish = room - .participant_permissions - .get(&identity) - .map(|permission| permission.can_publish) - .or(claims.video.can_publish) - .unwrap_or(true); - - if !can_publish { - return Err(anyhow!("user is not allowed to publish")); - } - - let sid = nanoid::nanoid!(17); - let track = Arc::new(TestServerAudioTrack { - sid: sid.clone(), - publisher_id: identity.clone(), - muted: AtomicBool::new(false), - }); - - let publication = Arc::new(RemoteTrackPublication); - - room.audio_tracks.push(track.clone()); - - for (id, client_room) in &room.client_rooms { - if *id != identity { - let _ = client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::SubscribedToRemoteAudioTrack( - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }), - publication.clone(), - )) - .unwrap(); - } - } - - Ok(sid) - } - - fn set_track_muted(&self, token: &str, track_sid: &str, muted: bool) -> Result<()> { - let claims = livekit_api::token::validate(token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - if let Some(track) = room - .audio_tracks - .iter_mut() - .find(|track| track.sid == track_sid) - { - track.muted.store(muted, SeqCst); - for (id, client_room) in room.client_rooms.iter() { - if *id != identity { - client_room - .0 - .lock() - .updates_tx - .try_broadcast(RoomUpdate::RemoteAudioTrackMuteChanged { - track_id: track_sid.to_string(), - muted, - }) - .unwrap(); - } - } - } - Ok(()) - } - - fn is_track_muted(&self, token: &str, track_sid: &str) -> Option { - let claims = livekit_api::token::validate(token, &self.secret_key).ok()?; - let room_name = claims.video.room.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms.get_mut(&*room_name)?; - room.audio_tracks.iter().find_map(|track| { - if track.sid == track_sid { - Some(track.muted.load(SeqCst)) - } else { - None - } - }) - } - - fn video_tracks(&self, token: String) -> Result>> { - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - room.client_rooms - .get(identity.as_ref()) - .ok_or_else(|| anyhow!("not a participant in room"))?; - Ok(room - .video_tracks - .iter() - .map(|track| { - Arc::new(RemoteVideoTrack { - server_track: track.clone(), - }) - }) - .collect()) - } - - fn audio_tracks(&self, token: String) -> Result>> { - let claims = livekit_api::token::validate(&token, &self.secret_key)?; - let room_name = claims.video.room.unwrap(); - let identity = claims.sub.unwrap(); - - let mut server_rooms = self.rooms.lock(); - let room = server_rooms - .get_mut(&*room_name) - .ok_or_else(|| anyhow!("room {} does not exist", room_name))?; - let client_room = room - .client_rooms - .get(identity.as_ref()) - .ok_or_else(|| anyhow!("not a participant in room"))?; - Ok(room - .audio_tracks - .iter() - .map(|track| { - Arc::new(RemoteAudioTrack { - server_track: track.clone(), - room: Arc::downgrade(client_room), - }) - }) - .collect()) - } -} - -#[derive(Default)] -struct TestServerRoom { - client_rooms: HashMap>, - video_tracks: Vec>, - audio_tracks: Vec>, - participant_permissions: HashMap, -} - -#[derive(Debug)] -struct TestServerVideoTrack { - sid: Sid, - publisher_id: Sid, - frames_rx: async_broadcast::Receiver, -} - -#[derive(Debug)] -struct TestServerAudioTrack { - sid: Sid, - publisher_id: Sid, - muted: AtomicBool, -} - -impl TestServerRoom {} - -pub struct TestApiClient { - url: String, -} - -#[async_trait] -impl livekit_api::Client for TestApiClient { - fn url(&self) -> &str { - &self.url - } - - async fn create_room(&self, name: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.create_room(name).await?; - Ok(()) - } - - async fn delete_room(&self, name: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.delete_room(name).await?; - Ok(()) - } - - async fn remove_participant(&self, room: String, identity: String) -> Result<()> { - let server = TestServer::get(&self.url)?; - server.remove_participant(room, identity).await?; - Ok(()) - } - - async fn update_participant( - &self, - room: String, - identity: String, - permission: livekit_api::proto::ParticipantPermission, - ) -> Result<()> { - let server = TestServer::get(&self.url)?; - server - .update_participant(room, identity, permission) - .await?; - Ok(()) - } - - fn room_token(&self, room: &str, identity: &str) -> Result { - let server = TestServer::get(&self.url)?; - token::create( - &server.api_key, - &server.secret_key, - Some(identity), - token::VideoGrant::to_join(room), - ) - } - - fn guest_token(&self, room: &str, identity: &str) -> Result { - let server = TestServer::get(&self.url)?; - token::create( - &server.api_key, - &server.secret_key, - Some(identity), - token::VideoGrant::for_guest(room), - ) - } -} - -struct RoomState { - connection: ( - watch::Sender, - watch::Receiver, - ), - display_sources: Vec, - paused_audio_tracks: HashSet, - updates_tx: async_broadcast::Sender, - updates_rx: async_broadcast::Receiver, -} - -pub struct Room(Mutex); - -impl Room { - pub fn new() -> Arc { - let (updates_tx, updates_rx) = async_broadcast::broadcast(128); - Arc::new(Self(Mutex::new(RoomState { - connection: watch::channel_with(ConnectionState::Disconnected), - display_sources: Default::default(), - paused_audio_tracks: Default::default(), - updates_tx, - updates_rx, - }))) - } - - pub fn status(&self) -> watch::Receiver { - self.0.lock().connection.1.clone() - } - - pub fn connect(self: &Arc, url: &str, token: &str) -> impl Future> { - let this = self.clone(); - let url = url.to_string(); - let token = token.to_string(); - async move { - let server = TestServer::get(&url)?; - server - .join_room(token.clone(), this.clone()) - .await - .context("room join")?; - *this.0.lock().connection.0.borrow_mut() = ConnectionState::Connected { url, token }; - Ok(()) - } - } - - pub fn display_sources(self: &Arc) -> impl Future>> { - let this = self.clone(); - async move { - // todo(linux): Remove this once the cross-platform LiveKit implementation is merged - #[cfg(any(test, feature = "test-support"))] - { - let server = this.test_server(); - server.executor.simulate_random_delay().await; - } - - Ok(this.0.lock().display_sources.clone()) - } - } - - pub fn publish_video_track( - self: &Arc, - track: LocalVideoTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_video_track(this.token(), track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) - } - } - - pub fn publish_audio_track( - self: &Arc, - track: LocalAudioTrack, - ) -> impl Future> { - let this = self.clone(); - let track = track.clone(); - async move { - let sid = this - .test_server() - .publish_audio_track(this.token(), &track) - .await?; - Ok(LocalTrackPublication { - room: Arc::downgrade(&this), - sid, - }) - } - } - - pub fn unpublish_track(&self, _publication: LocalTrackPublication) {} - - pub fn remote_audio_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } - - pub fn remote_audio_track_publications( - &self, - publisher_id: &str, - ) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .audio_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .map(|_track| Arc::new(RemoteTrackPublication {})) - .collect() - } - - pub fn remote_video_tracks(&self, publisher_id: &str) -> Vec> { - if !self.is_connected() { - return Vec::new(); - } - - self.test_server() - .video_tracks(self.token()) - .unwrap() - .into_iter() - .filter(|track| track.publisher_id() == publisher_id) - .collect() - } - - pub fn updates(&self) -> impl Stream { - self.0.lock().updates_rx.clone() - } - - pub fn set_display_sources(&self, sources: Vec) { - self.0.lock().display_sources = sources; - } - - fn test_server(&self) -> Arc { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { url, .. } => TestServer::get(&url).unwrap(), - } - } - - fn token(&self) -> String { - match self.0.lock().connection.1.borrow().clone() { - ConnectionState::Disconnected => panic!("must be connected to call this method"), - ConnectionState::Connected { token, .. } => token, - } - } - - fn is_connected(&self) -> bool { - match *self.0.lock().connection.1.borrow() { - ConnectionState::Disconnected => false, - ConnectionState::Connected { .. } => true, - } - } -} - -impl Drop for Room { - fn drop(&mut self) { - if let ConnectionState::Connected { token, .. } = mem::replace( - &mut *self.0.lock().connection.0.borrow_mut(), - ConnectionState::Disconnected, - ) { - if let Ok(server) = TestServer::get(&token) { - let executor = server.executor.clone(); - executor - .spawn(async move { server.leave_room(token).await.unwrap() }) - .detach(); - } - } - } -} - -#[derive(Clone)] -pub struct LocalTrackPublication { - sid: String, - room: Weak, -} - -impl LocalTrackPublication { - pub fn set_mute(&self, mute: bool) -> impl Future> { - let sid = self.sid.clone(); - let room = self.room.clone(); - async move { - if let Some(room) = room.upgrade() { - room.test_server() - .set_track_muted(&room.token(), &sid, mute) - } else { - Err(anyhow!("no such room")) - } - } - } - - pub fn is_muted(&self) -> bool { - if let Some(room) = self.room.upgrade() { - room.test_server() - .is_track_muted(&room.token(), &self.sid) - .unwrap_or(false) - } else { - false - } - } - - pub fn sid(&self) -> String { - self.sid.clone() - } -} - -pub struct RemoteTrackPublication; - -impl RemoteTrackPublication { - pub fn set_enabled(&self, _enabled: bool) -> impl Future> { - async { Ok(()) } - } - - pub fn is_muted(&self) -> bool { - false - } - - pub fn sid(&self) -> String { - "".to_string() - } -} - -#[derive(Clone)] -pub struct LocalVideoTrack { - frames_rx: async_broadcast::Receiver, -} - -impl LocalVideoTrack { - pub fn screen_share_for_display(display: &MacOSDisplay) -> Self { - Self { - frames_rx: display.frames.1.clone(), - } - } -} - -#[derive(Clone)] -pub struct LocalAudioTrack; - -impl LocalAudioTrack { - pub fn create() -> Self { - Self - } -} - -#[derive(Debug)] -pub struct RemoteVideoTrack { - server_track: Arc, -} - -impl RemoteVideoTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn frames(&self) -> async_broadcast::Receiver { - self.server_track.frames_rx.clone() - } -} - -#[derive(Debug)] -pub struct RemoteAudioTrack { - server_track: Arc, - room: Weak, -} - -impl RemoteAudioTrack { - pub fn sid(&self) -> &str { - &self.server_track.sid - } - - pub fn publisher_id(&self) -> &str { - &self.server_track.publisher_id - } - - pub fn start(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .remove(&self.server_track.sid); - } - } - - pub fn stop(&self) { - if let Some(room) = self.room.upgrade() { - room.0 - .lock() - .paused_audio_tracks - .insert(self.server_track.sid.clone()); - } - } - - pub fn is_playing(&self) -> bool { - !self - .room - .upgrade() - .unwrap() - .0 - .lock() - .paused_audio_tracks - .contains(&self.server_track.sid) - } -} - -#[derive(Clone)] -pub struct MacOSDisplay { - frames: ( - async_broadcast::Sender, - async_broadcast::Receiver, - ), -} - -impl Default for MacOSDisplay { - fn default() -> Self { - Self::new() - } -} - -impl MacOSDisplay { - pub fn new() -> Self { - Self { - frames: async_broadcast::broadcast(128), - } - } - - pub fn send_frame(&self, frame: Frame) { - self.frames.0.try_broadcast(frame).unwrap(); - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Frame { - pub label: String, - pub width: usize, - pub height: usize, -} - -impl Frame { - pub fn width(&self) -> usize { - self.width - } - - pub fn height(&self) -> usize { - self.height - } - - pub fn image(&self) -> SurfaceSource { - unimplemented!("you can't call this in test mode") - } -} diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index d078e9afd6..934f6f9d0d 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -20,6 +20,7 @@ core-foundation.workspace = true ctor.workspace = true foreign-types = "0.5" metal.workspace = true +core-video.workspace = true objc = "0.2" [build-dependencies] diff --git a/crates/media/src/media.rs b/crates/media/src/media.rs index 3f55475589..29563de85f 100644 --- a/crates/media/src/media.rs +++ b/crates/media/src/media.rs @@ -3,213 +3,6 @@ mod bindings; -#[cfg(target_os = "macos")] -use core_foundation::{ - base::{CFTypeID, TCFType}, - declare_TCFType, impl_CFTypeDescription, impl_TCFType, -}; -#[cfg(target_os = "macos")] -use std::ffi::c_void; - -#[cfg(target_os = "macos")] -pub mod io_surface { - use super::*; - - #[repr(C)] - pub struct __IOSurface(c_void); - // The ref type must be a pointer to the underlying struct. - pub type IOSurfaceRef = *const __IOSurface; - - declare_TCFType!(IOSurface, IOSurfaceRef); - impl_TCFType!(IOSurface, IOSurfaceRef, IOSurfaceGetTypeID); - impl_CFTypeDescription!(IOSurface); - - #[link(name = "IOSurface", kind = "framework")] - extern "C" { - fn IOSurfaceGetTypeID() -> CFTypeID; - } -} - -#[cfg(target_os = "macos")] -pub mod core_video { - #![allow(non_snake_case)] - - use super::*; - pub use crate::bindings::{ - kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, - kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, - }; - use crate::bindings::{kCVReturnSuccess, CVReturn, OSType}; - use anyhow::{anyhow, Result}; - use core_foundation::{ - base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, - }; - use foreign_types::ForeignTypeRef; - use io_surface::{IOSurface, IOSurfaceRef}; - use metal::{MTLDevice, MTLPixelFormat}; - use std::ptr; - - #[repr(C)] - pub struct __CVImageBuffer(c_void); - // The ref type must be a pointer to the underlying struct. - pub type CVImageBufferRef = *const __CVImageBuffer; - - declare_TCFType!(CVImageBuffer, CVImageBufferRef); - impl_TCFType!(CVImageBuffer, CVImageBufferRef, CVImageBufferGetTypeID); - impl_CFTypeDescription!(CVImageBuffer); - - impl CVImageBuffer { - pub fn io_surface(&self) -> IOSurface { - unsafe { - IOSurface::wrap_under_get_rule(CVPixelBufferGetIOSurface( - self.as_concrete_TypeRef(), - )) - } - } - - pub fn width(&self) -> usize { - unsafe { CVPixelBufferGetWidth(self.as_concrete_TypeRef()) } - } - - pub fn height(&self) -> usize { - unsafe { CVPixelBufferGetHeight(self.as_concrete_TypeRef()) } - } - - pub fn plane_width(&self, plane: usize) -> usize { - unsafe { CVPixelBufferGetWidthOfPlane(self.as_concrete_TypeRef(), plane) } - } - - pub fn plane_height(&self, plane: usize) -> usize { - unsafe { CVPixelBufferGetHeightOfPlane(self.as_concrete_TypeRef(), plane) } - } - - pub fn pixel_format_type(&self) -> OSType { - unsafe { CVPixelBufferGetPixelFormatType(self.as_concrete_TypeRef()) } - } - } - - #[link(name = "CoreVideo", kind = "framework")] - extern "C" { - fn CVImageBufferGetTypeID() -> CFTypeID; - fn CVPixelBufferGetIOSurface(buffer: CVImageBufferRef) -> IOSurfaceRef; - fn CVPixelBufferGetWidth(buffer: CVImageBufferRef) -> usize; - fn CVPixelBufferGetHeight(buffer: CVImageBufferRef) -> usize; - fn CVPixelBufferGetWidthOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetHeightOfPlane(buffer: CVImageBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetPixelFormatType(buffer: CVImageBufferRef) -> OSType; - } - - #[repr(C)] - pub struct __CVMetalTextureCache(c_void); - pub type CVMetalTextureCacheRef = *const __CVMetalTextureCache; - - declare_TCFType!(CVMetalTextureCache, CVMetalTextureCacheRef); - impl_TCFType!( - CVMetalTextureCache, - CVMetalTextureCacheRef, - CVMetalTextureCacheGetTypeID - ); - impl_CFTypeDescription!(CVMetalTextureCache); - - impl CVMetalTextureCache { - /// # Safety - /// - /// metal_device must be valid according to CVMetalTextureCacheCreate - pub unsafe fn new(metal_device: *mut MTLDevice) -> Result { - let mut this = ptr::null(); - let result = CVMetalTextureCacheCreate( - kCFAllocatorDefault, - ptr::null(), - metal_device, - ptr::null(), - &mut this, - ); - if result == kCVReturnSuccess { - Ok(CVMetalTextureCache::wrap_under_create_rule(this)) - } else { - Err(anyhow!("could not create texture cache, code: {}", result)) - } - } - - /// # Safety - /// - /// The arguments to this function must be valid according to CVMetalTextureCacheCreateTextureFromImage - pub unsafe fn create_texture_from_image( - &self, - source: CVImageBufferRef, - texture_attributes: CFDictionaryRef, - pixel_format: MTLPixelFormat, - width: usize, - height: usize, - plane_index: usize, - ) -> Result { - let mut this = ptr::null(); - let result = CVMetalTextureCacheCreateTextureFromImage( - kCFAllocatorDefault, - self.as_concrete_TypeRef(), - source, - texture_attributes, - pixel_format, - width, - height, - plane_index, - &mut this, - ); - if result == kCVReturnSuccess { - Ok(CVMetalTexture::wrap_under_create_rule(this)) - } else { - Err(anyhow!("could not create texture, code: {}", result)) - } - } - } - - #[link(name = "CoreVideo", kind = "framework")] - extern "C" { - fn CVMetalTextureCacheGetTypeID() -> CFTypeID; - fn CVMetalTextureCacheCreate( - allocator: CFAllocatorRef, - cache_attributes: CFDictionaryRef, - metal_device: *const MTLDevice, - texture_attributes: CFDictionaryRef, - cache_out: *mut CVMetalTextureCacheRef, - ) -> CVReturn; - fn CVMetalTextureCacheCreateTextureFromImage( - allocator: CFAllocatorRef, - texture_cache: CVMetalTextureCacheRef, - source_image: CVImageBufferRef, - texture_attributes: CFDictionaryRef, - pixel_format: MTLPixelFormat, - width: usize, - height: usize, - plane_index: usize, - texture_out: *mut CVMetalTextureRef, - ) -> CVReturn; - } - - #[repr(C)] - pub struct __CVMetalTexture(c_void); - pub type CVMetalTextureRef = *const __CVMetalTexture; - - declare_TCFType!(CVMetalTexture, CVMetalTextureRef); - impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); - impl_CFTypeDescription!(CVMetalTexture); - - impl CVMetalTexture { - pub fn as_texture_ref(&self) -> &metal::TextureRef { - unsafe { - let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef()); - metal::TextureRef::from_ptr(texture as *mut _) - } - } - } - - #[link(name = "CoreVideo", kind = "framework")] - extern "C" { - fn CVMetalTextureGetTypeID() -> CFTypeID; - fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; - } -} - #[cfg(target_os = "macos")] pub mod core_media { #![allow(non_snake_case)] @@ -218,7 +11,6 @@ pub mod core_media { kCMSampleAttachmentKey_NotSync, kCMTimeInvalid, kCMVideoCodecType_H264, CMItemIndex, CMSampleTimingInfo, CMTime, CMTimeMake, CMVideoCodecType, }; - use crate::core_video::{CVImageBuffer, CVImageBufferRef}; use anyhow::{anyhow, Result}; use core_foundation::{ array::{CFArray, CFArrayRef}, @@ -228,6 +20,7 @@ pub mod core_media { impl_CFTypeDescription, impl_TCFType, string::CFString, }; + use core_video::image_buffer::{CVImageBuffer, CVImageBufferRef}; use std::{ffi::c_void, ptr}; #[repr(C)] @@ -422,129 +215,138 @@ pub mod core_media { } #[cfg(target_os = "macos")] -pub mod video_toolbox { +pub mod core_video { #![allow(non_snake_case)] - use super::*; - use crate::{ - core_media::{CMSampleBufferRef, CMTime, CMVideoCodecType}, - core_video::CVImageBufferRef, + #[cfg(target_os = "macos")] + use core_foundation::{ + base::{CFTypeID, TCFType}, + declare_TCFType, impl_CFTypeDescription, impl_TCFType, }; + #[cfg(target_os = "macos")] + use std::ffi::c_void; + + pub use crate::bindings::{ + kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr8Planar, + }; + use crate::bindings::{kCVReturnSuccess, CVReturn}; use anyhow::{anyhow, Result}; - pub use bindings::VTEncodeInfoFlags; - use core_foundation::{base::OSStatus, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef}; + use core_foundation::{ + base::kCFAllocatorDefault, dictionary::CFDictionaryRef, mach_port::CFAllocatorRef, + }; + use foreign_types::ForeignTypeRef; + + use metal::{MTLDevice, MTLPixelFormat}; use std::ptr; #[repr(C)] - pub struct __VTCompressionSession(c_void); - // The ref type must be a pointer to the underlying struct. - pub type VTCompressionSessionRef = *const __VTCompressionSession; + pub struct __CVMetalTextureCache(c_void); + pub type CVMetalTextureCacheRef = *const __CVMetalTextureCache; - declare_TCFType!(VTCompressionSession, VTCompressionSessionRef); + declare_TCFType!(CVMetalTextureCache, CVMetalTextureCacheRef); impl_TCFType!( - VTCompressionSession, - VTCompressionSessionRef, - VTCompressionSessionGetTypeID + CVMetalTextureCache, + CVMetalTextureCacheRef, + CVMetalTextureCacheGetTypeID ); - impl_CFTypeDescription!(VTCompressionSession); + impl_CFTypeDescription!(CVMetalTextureCache); - impl VTCompressionSession { - /// Create a new compression session. - /// + impl CVMetalTextureCache { /// # Safety /// - /// The callback must be a valid function pointer. and the callback_data must be valid - /// in whatever terms that callback expects. - pub unsafe fn new( - width: usize, - height: usize, - codec: CMVideoCodecType, - callback: VTCompressionOutputCallback, - callback_data: *const c_void, - ) -> Result { + /// metal_device must be valid according to CVMetalTextureCacheCreate + pub unsafe fn new(metal_device: *mut MTLDevice) -> Result { let mut this = ptr::null(); - let result = VTCompressionSessionCreate( + let result = CVMetalTextureCacheCreate( + kCFAllocatorDefault, ptr::null(), - width as i32, - height as i32, - codec, + metal_device, ptr::null(), - ptr::null(), - ptr::null(), - callback, - callback_data, &mut this, ); - - if result == 0 { - Ok(Self::wrap_under_create_rule(this)) + if result == kCVReturnSuccess { + Ok(CVMetalTextureCache::wrap_under_create_rule(this)) } else { - Err(anyhow!( - "error creating compression session, code {}", - result - )) + Err(anyhow!("could not create texture cache, code: {}", result)) } } /// # Safety /// - /// The arguments to this function must be valid according to VTCompressionSessionEncodeFrame - pub unsafe fn encode_frame( + /// The arguments to this function must be valid according to CVMetalTextureCacheCreateTextureFromImage + pub unsafe fn create_texture_from_image( &self, - buffer: CVImageBufferRef, - presentation_timestamp: CMTime, - duration: CMTime, - ) -> Result<()> { - let result = VTCompressionSessionEncodeFrame( + source: ::core_video::image_buffer::CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + ) -> Result { + let mut this = ptr::null(); + let result = CVMetalTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, self.as_concrete_TypeRef(), - buffer, - presentation_timestamp, - duration, - ptr::null(), - ptr::null(), - ptr::null_mut(), + source, + texture_attributes, + pixel_format, + width, + height, + plane_index, + &mut this, ); - if result == 0 { - Ok(()) + if result == kCVReturnSuccess { + Ok(CVMetalTexture::wrap_under_create_rule(this)) } else { - Err(anyhow!("error encoding frame, code {}", result)) + Err(anyhow!("could not create texture, code: {}", result)) } } } - type VTCompressionOutputCallback = Option< - unsafe extern "C" fn( - outputCallbackRefCon: *mut c_void, - sourceFrameRefCon: *mut c_void, - status: OSStatus, - infoFlags: VTEncodeInfoFlags, - sampleBuffer: CMSampleBufferRef, - ), - >; - - #[link(name = "VideoToolbox", kind = "framework")] + #[link(name = "CoreVideo", kind = "framework")] extern "C" { - fn VTCompressionSessionGetTypeID() -> CFTypeID; - fn VTCompressionSessionCreate( + fn CVMetalTextureCacheGetTypeID() -> CFTypeID; + fn CVMetalTextureCacheCreate( allocator: CFAllocatorRef, - width: i32, - height: i32, - codec_type: CMVideoCodecType, - encoder_specification: CFDictionaryRef, - source_image_buffer_attributes: CFDictionaryRef, - compressed_data_allocator: CFAllocatorRef, - output_callback: VTCompressionOutputCallback, - output_callback_ref_con: *const c_void, - compression_session_out: *mut VTCompressionSessionRef, - ) -> OSStatus; - fn VTCompressionSessionEncodeFrame( - session: VTCompressionSessionRef, - image_buffer: CVImageBufferRef, - presentation_timestamp: CMTime, - duration: CMTime, - frame_properties: CFDictionaryRef, - source_frame_ref_con: *const c_void, - output_flags: *mut VTEncodeInfoFlags, - ) -> OSStatus; + cache_attributes: CFDictionaryRef, + metal_device: *const MTLDevice, + texture_attributes: CFDictionaryRef, + cache_out: *mut CVMetalTextureCacheRef, + ) -> CVReturn; + fn CVMetalTextureCacheCreateTextureFromImage( + allocator: CFAllocatorRef, + texture_cache: CVMetalTextureCacheRef, + source_image: ::core_video::image_buffer::CVImageBufferRef, + texture_attributes: CFDictionaryRef, + pixel_format: MTLPixelFormat, + width: usize, + height: usize, + plane_index: usize, + texture_out: *mut CVMetalTextureRef, + ) -> CVReturn; + } + + #[repr(C)] + pub struct __CVMetalTexture(c_void); + pub type CVMetalTextureRef = *const __CVMetalTexture; + + declare_TCFType!(CVMetalTexture, CVMetalTextureRef); + impl_TCFType!(CVMetalTexture, CVMetalTextureRef, CVMetalTextureGetTypeID); + impl_CFTypeDescription!(CVMetalTexture); + + impl CVMetalTexture { + pub fn as_texture_ref(&self) -> &metal::TextureRef { + unsafe { + let texture = CVMetalTextureGetTexture(self.as_concrete_TypeRef()); + metal::TextureRef::from_ptr(texture as *mut _) + } + } + } + + #[link(name = "CoreVideo", kind = "framework")] + extern "C" { + fn CVMetalTextureGetTypeID() -> CFTypeID; + fn CVMetalTextureGetTexture(texture: CVMetalTextureRef) -> *mut c_void; } } diff --git a/crates/storybook/build.rs b/crates/storybook/build.rs index ec0e8944f5..66791cae42 100644 --- a/crates/storybook/build.rs +++ b/crates/storybook/build.rs @@ -1,11 +1,4 @@ fn main() { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - // TODO: We shouldn't depend on WebRTC in editor - #[cfg(target_os = "macos")] - { - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } - #[cfg(target_os = "windows")] { #[cfg(target_env = "msvc")] diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs index 1d17cfa145..7d109074b7 100644 --- a/crates/workspace/src/shared_screen.rs +++ b/crates/workspace/src/shared_screen.rs @@ -1,11 +1,133 @@ -#[cfg(target_os = "macos")] -mod macos; +use crate::{ + item::{Item, ItemEvent}, + ItemNavHistory, WorkspaceId, +}; +use call::{RemoteVideoTrack, RemoteVideoTrackView, Room}; +use client::{proto::PeerId, User}; +use gpui::{ + div, AppContext as _, Entity, EventEmitter, FocusHandle, Focusable, InteractiveElement, + ParentElement, Render, SharedString, Styled, +}; +use std::sync::Arc; +use ui::{prelude::*, Icon, IconName}; -#[cfg(target_os = "macos")] -pub use macos::*; +pub enum Event { + Close, +} -#[cfg(not(target_os = "macos"))] -mod cross_platform; +pub struct SharedScreen { + pub peer_id: PeerId, + user: Arc, + nav_history: Option, + view: Entity, + focus: FocusHandle, +} -#[cfg(not(target_os = "macos"))] -pub use cross_platform::*; +impl SharedScreen { + pub fn new( + track: RemoteVideoTrack, + peer_id: PeerId, + user: Arc, + room: Entity, + window: &mut Window, + cx: &mut Context, + ) -> Self { + let my_sid = track.sid(); + cx.subscribe(&room, move |_, _, ev, cx| match ev { + call::room::Event::RemoteVideoTrackUnsubscribed { sid } => { + if sid == &my_sid { + cx.emit(Event::Close) + } + } + _ => {} + }) + .detach(); + + let view = cx.new(|cx| RemoteVideoTrackView::new(track.clone(), window, cx)); + cx.subscribe(&view, |_, _, ev, cx| match ev { + call::RemoteVideoTrackViewEvent::Close => cx.emit(Event::Close), + }) + .detach(); + Self { + view, + peer_id, + user, + nav_history: Default::default(), + focus: cx.focus_handle(), + } + } +} + +impl EventEmitter for SharedScreen {} + +impl Focusable for SharedScreen { + fn focus_handle(&self, _: &App) -> FocusHandle { + self.focus.clone() + } +} +impl Render for SharedScreen { + fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { + div() + .bg(cx.theme().colors().editor_background) + .track_focus(&self.focus) + .key_context("SharedScreen") + .size_full() + .child(self.view.clone()) + } +} + +impl Item for SharedScreen { + type Event = Event; + + fn tab_tooltip_text(&self, _: &App) -> Option { + Some(format!("{}'s screen", self.user.github_login).into()) + } + + fn deactivated(&mut self, _window: &mut Window, cx: &mut Context) { + if let Some(nav_history) = self.nav_history.as_mut() { + nav_history.push::<()>(None, cx); + } + } + + fn tab_icon(&self, _window: &Window, _cx: &App) -> Option { + Some(Icon::new(IconName::Screen)) + } + + fn tab_content_text(&self, _window: &Window, _cx: &App) -> Option { + Some(format!("{}'s screen", self.user.github_login).into()) + } + + fn telemetry_event_text(&self) -> Option<&'static str> { + None + } + + fn set_nav_history( + &mut self, + history: ItemNavHistory, + _window: &mut Window, + _cx: &mut Context, + ) { + self.nav_history = Some(history); + } + + fn clone_on_split( + &self, + _workspace_id: Option, + window: &mut Window, + cx: &mut Context, + ) -> Option> { + Some(cx.new(|cx| Self { + view: self.view.update(cx, |view, cx| view.clone(window, cx)), + peer_id: self.peer_id, + user: self.user.clone(), + nav_history: Default::default(), + focus: cx.focus_handle(), + })) + } + + fn to_item_events(event: &Self::Event, mut f: impl FnMut(ItemEvent)) { + match event { + Event::Close => f(ItemEvent::CloseItem), + } + } +} diff --git a/crates/workspace/src/shared_screen/cross_platform.rs b/crates/workspace/src/shared_screen/cross_platform.rs deleted file mode 100644 index 376140622a..0000000000 --- a/crates/workspace/src/shared_screen/cross_platform.rs +++ /dev/null @@ -1,121 +0,0 @@ -use crate::{ - item::{Item, ItemEvent}, - ItemNavHistory, WorkspaceId, -}; -use call::{RemoteVideoTrack, RemoteVideoTrackView}; -use client::{proto::PeerId, User}; -use gpui::{ - div, AppContext as _, Entity, EventEmitter, FocusHandle, Focusable, InteractiveElement, - ParentElement, Render, SharedString, Styled, -}; -use std::sync::Arc; -use ui::{prelude::*, Icon, IconName}; - -pub enum Event { - Close, -} - -pub struct SharedScreen { - pub peer_id: PeerId, - user: Arc, - nav_history: Option, - view: Entity, - focus: FocusHandle, -} - -impl SharedScreen { - pub fn new( - track: RemoteVideoTrack, - peer_id: PeerId, - user: Arc, - window: &mut Window, - cx: &mut Context, - ) -> Self { - let view = cx.new(|cx| RemoteVideoTrackView::new(track.clone(), window, cx)); - cx.subscribe(&view, |_, _, ev, cx| match ev { - call::RemoteVideoTrackViewEvent::Close => cx.emit(Event::Close), - }) - .detach(); - Self { - view, - peer_id, - user, - nav_history: Default::default(), - focus: cx.focus_handle(), - } - } -} - -impl EventEmitter for SharedScreen {} - -impl Focusable for SharedScreen { - fn focus_handle(&self, _: &App) -> FocusHandle { - self.focus.clone() - } -} -impl Render for SharedScreen { - fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { - div() - .bg(cx.theme().colors().editor_background) - .track_focus(&self.focus) - .key_context("SharedScreen") - .size_full() - .child(self.view.clone()) - } -} - -impl Item for SharedScreen { - type Event = Event; - - fn tab_tooltip_text(&self, _: &App) -> Option { - Some(format!("{}'s screen", self.user.github_login).into()) - } - - fn deactivated(&mut self, _window: &mut Window, cx: &mut Context) { - if let Some(nav_history) = self.nav_history.as_mut() { - nav_history.push::<()>(None, cx); - } - } - - fn tab_icon(&self, _window: &Window, _cx: &App) -> Option { - Some(Icon::new(IconName::Screen)) - } - - fn tab_content_text(&self, _window: &Window, _cx: &App) -> Option { - Some(format!("{}'s screen", self.user.github_login).into()) - } - - fn telemetry_event_text(&self) -> Option<&'static str> { - None - } - - fn set_nav_history( - &mut self, - history: ItemNavHistory, - _window: &mut Window, - _cx: &mut Context, - ) { - self.nav_history = Some(history); - } - - fn clone_on_split( - &self, - _workspace_id: Option, - window: &mut Window, - cx: &mut Context, - ) -> Option> { - Some(cx.new(|cx| Self { - view: self.view.update(cx, |view, cx| view.clone(window, cx)), - peer_id: self.peer_id, - user: self.user.clone(), - nav_history: Default::default(), - focus: cx.focus_handle(), - })) - } - - fn to_item_events(event: &Self::Event, mut f: impl FnMut(ItemEvent)) { - match event { - Event::Close => f(ItemEvent::CloseItem), - } - } -} diff --git a/crates/workspace/src/shared_screen/macos.rs b/crates/workspace/src/shared_screen/macos.rs deleted file mode 100644 index e943b18692..0000000000 --- a/crates/workspace/src/shared_screen/macos.rs +++ /dev/null @@ -1,132 +0,0 @@ -use crate::{ - item::{Item, ItemEvent}, - ItemNavHistory, WorkspaceId, -}; -use anyhow::Result; -use call::participant::{Frame, RemoteVideoTrack}; -use client::{proto::PeerId, User}; -use futures::StreamExt; -use gpui::{ - div, surface, App, Context, Entity, EventEmitter, FocusHandle, Focusable, InteractiveElement, - ParentElement, Render, SharedString, Styled, Task, Window, -}; -use std::sync::{Arc, Weak}; -use ui::{prelude::*, Icon, IconName}; - -pub enum Event { - Close, -} - -pub struct SharedScreen { - track: Weak, - frame: Option, - pub peer_id: PeerId, - user: Arc, - nav_history: Option, - _maintain_frame: Task>, - focus: FocusHandle, -} - -impl SharedScreen { - pub fn new( - track: Arc, - peer_id: PeerId, - user: Arc, - window: &mut Window, - cx: &mut Context, - ) -> Self { - cx.focus_handle(); - let mut frames = track.frames(); - Self { - track: Arc::downgrade(&track), - frame: None, - peer_id, - user, - nav_history: Default::default(), - _maintain_frame: cx.spawn_in(window, async move |this, cx| { - while let Some(frame) = frames.next().await { - this.update(cx, |this, cx| { - this.frame = Some(frame); - cx.notify(); - })?; - } - this.update(cx, |_, cx| cx.emit(Event::Close))?; - Ok(()) - }), - focus: cx.focus_handle(), - } - } -} - -impl EventEmitter for SharedScreen {} - -impl Focusable for SharedScreen { - fn focus_handle(&self, _: &App) -> FocusHandle { - self.focus.clone() - } -} -impl Render for SharedScreen { - fn render(&mut self, _window: &mut Window, cx: &mut Context) -> impl IntoElement { - div() - .bg(cx.theme().colors().editor_background) - .track_focus(&self.focus) - .key_context("SharedScreen") - .size_full() - .children( - self.frame - .as_ref() - .map(|frame| surface(frame.image()).size_full()), - ) - } -} - -impl Item for SharedScreen { - type Event = Event; - - fn tab_tooltip_text(&self, _: &App) -> Option { - Some(format!("{}'s screen", self.user.github_login).into()) - } - - fn deactivated(&mut self, _: &mut Window, cx: &mut Context) { - if let Some(nav_history) = self.nav_history.as_mut() { - nav_history.push::<()>(None, cx); - } - } - - fn tab_icon(&self, _window: &Window, _cx: &App) -> Option { - Some(Icon::new(IconName::Screen)) - } - - fn tab_content_text(&self, _window: &Window, _cx: &App) -> Option { - Some(format!("{}'s screen", self.user.github_login).into()) - } - - fn telemetry_event_text(&self) -> Option<&'static str> { - None - } - - fn set_nav_history( - &mut self, - history: ItemNavHistory, - _window: &mut Window, - _: &mut Context, - ) { - self.nav_history = Some(history); - } - - fn clone_on_split( - &self, - _workspace_id: Option, - window: &mut Window, - cx: &mut Context, - ) -> Option> { - let track = self.track.upgrade()?; - Some(cx.new(|cx| Self::new(track, self.peer_id, self.user.clone(), window, cx))) - } - - fn to_item_events(event: &Self::Event, mut f: impl FnMut(ItemEvent)) { - match event { - Event::Close => f(ItemEvent::CloseItem), - } - } -} diff --git a/crates/workspace/src/workspace.rs b/crates/workspace/src/workspace.rs index 99160b1701..d289d52e83 100644 --- a/crates/workspace/src/workspace.rs +++ b/crates/workspace/src/workspace.rs @@ -4435,8 +4435,8 @@ impl Workspace { cx: &mut App, ) -> Option> { let call = self.active_call()?; - let room = call.read(cx).room()?.read(cx); - let participant = room.remote_participant_for_peer_id(peer_id)?; + let room = call.read(cx).room()?.clone(); + let participant = room.read(cx).remote_participant_for_peer_id(peer_id)?; let track = participant.video_tracks.values().next()?.clone(); let user = participant.user.clone(); @@ -4446,7 +4446,7 @@ impl Workspace { } } - Some(cx.new(|cx| SharedScreen::new(track, peer_id, user.clone(), window, cx))) + Some(cx.new(|cx| SharedScreen::new(track, peer_id, user.clone(), room.clone(), window, cx))) } pub fn on_window_activation_changed(&mut self, window: &mut Window, cx: &mut Context) { diff --git a/crates/zed/Cargo.toml b/crates/zed/Cargo.toml index ae9759c167..059e5dbc7f 100644 --- a/crates/zed/Cargo.toml +++ b/crates/zed/Cargo.toml @@ -162,7 +162,7 @@ tree-sitter-md.workspace = true tree-sitter-rust.workspace = true workspace = { workspace = true, features = ["test-support"] } -[package.metadata.bundle-dev] +[package.metadata.bundle] icon = ["resources/app-icon-dev@2x.png", "resources/app-icon-dev.png"] identifier = "dev.zed.Zed-Dev" name = "Zed Dev" diff --git a/crates/zed/build.rs b/crates/zed/build.rs index b97bda1681..7c097ba068 100644 --- a/crates/zed/build.rs +++ b/crates/zed/build.rs @@ -4,15 +4,6 @@ fn main() { if cfg!(target_os = "macos") { println!("cargo:rustc-env=MACOSX_DEPLOYMENT_TARGET=10.15.7"); - println!("cargo:rerun-if-env-changed=ZED_BUNDLE"); - if std::env::var("ZED_BUNDLE").ok().as_deref() == Some("true") { - // Find WebRTC.framework in the Frameworks folder when running as part of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path/../Frameworks"); - } else { - // Find WebRTC.framework as a sibling of the executable when running outside of an application bundle. - println!("cargo:rustc-link-arg=-Wl,-rpath,@executable_path"); - } - // Weakly link ReplayKit to ensure Zed can be used on macOS 10.15+. println!("cargo:rustc-link-arg=-Wl,-weak_framework,ReplayKit"); diff --git a/nix/build.nix b/nix/build.nix index 6590c9b445..77e7804538 100644 --- a/nix/build.nix +++ b/nix/build.nix @@ -197,12 +197,6 @@ craneLib.buildPackage ( lib.recursiveUpdate commonArgs { inherit cargoArtifacts; - patches = lib.optionals stdenv.hostPlatform.isDarwin [ - # Livekit requires Swift 6 - # We need this until livekit-rust sdk is used - ../script/patches/use-cross-platform-livekit.patch - ]; - dontUseCmakeConfigure = true; # without the env var generate-licenses fails due to crane's fetchCargoVendor, see: diff --git a/script/bundle-mac b/script/bundle-mac index dcdf0f765d..9aa9763e9c 100755 --- a/script/bundle-mac +++ b/script/bundle-mac @@ -221,13 +221,9 @@ function sign_app_binaries() { local app_path=$1 local architecture=$2 local architecture_dir=$3 - echo "Copying WebRTC.framework into the frameworks folder" rm -rf "${app_path}/Contents/Frameworks" mkdir -p "${app_path}/Contents/Frameworks" - if [ "$local_arch" = false ]; then - cp -R target/${local_target_triple}/${target_dir}/WebRTC.framework "${app_path}/Contents/Frameworks/" - else - cp -R target/${target_dir}/WebRTC.framework "${app_path}/Contents/Frameworks/" + if [ "$local_arch" = true ]; then cp -R target/${target_dir}/cli "${app_path}/Contents/MacOS/" fi @@ -240,7 +236,6 @@ function sign_app_binaries() { if [[ $can_code_sign = true ]]; then echo "Code signing binaries" # sequence of codesign commands modeled after this example: https://developer.apple.com/forums/thread/701514 - /usr/bin/codesign --deep --force --timestamp --sign "$IDENTITY" "${app_path}/Contents/Frameworks/WebRTC.framework" -v /usr/bin/codesign --deep --force --timestamp --options runtime --sign "$IDENTITY" "${app_path}/Contents/MacOS/cli" -v /usr/bin/codesign --deep --force --timestamp --options runtime --sign "$IDENTITY" "${app_path}/Contents/MacOS/git" -v /usr/bin/codesign --deep --force --timestamp --options runtime --entitlements crates/zed/resources/zed.entitlements --sign "$IDENTITY" "${app_path}/Contents/MacOS/zed" -v diff --git a/script/check-rust-livekit-macos b/script/check-rust-livekit-macos deleted file mode 100755 index 1afdc81b22..0000000000 --- a/script/check-rust-livekit-macos +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - - -set -exuo pipefail - -git apply script/patches/use-cross-platform-livekit.patch - -# Re-enable error skipping for this check, so that we can unapply the patch -set +e - -cargo check -p workspace -exit_code=$? - -# Disable error skipping again -set -e - -git apply -R script/patches/use-cross-platform-livekit.patch - -exit "$exit_code" diff --git a/script/patches/use-cross-platform-livekit.patch b/script/patches/use-cross-platform-livekit.patch deleted file mode 100644 index 93010a0eec..0000000000 --- a/script/patches/use-cross-platform-livekit.patch +++ /dev/null @@ -1,59 +0,0 @@ -diff --git a/crates/call/Cargo.toml b/crates/call/Cargo.toml -index 9ba10e56ba..bb69440691 100644 ---- a/crates/call/Cargo.toml -+++ b/crates/call/Cargo.toml -@@ -41,10 +41,10 @@ serde_derive.workspace = true - telemetry.workspace = true - util.workspace = true - --[target.'cfg(target_os = "macos")'.dependencies] -+[target.'cfg(any())'.dependencies] - livekit_client_macos.workspace = true - --[target.'cfg(not(target_os = "macos"))'.dependencies] -+[target.'cfg(all())'.dependencies] - livekit_client.workspace = true - - [dev-dependencies] -diff --git a/crates/call/src/call.rs b/crates/call/src/call.rs -index 5e212d35b7..a8f9e8f43e 100644 ---- a/crates/call/src/call.rs -+++ b/crates/call/src/call.rs -@@ -1,13 +1,13 @@ - pub mod call_settings; - --#[cfg(target_os = "macos")] -+#[cfg(any())] - mod macos; - --#[cfg(target_os = "macos")] -+#[cfg(any())] - pub use macos::*; - --#[cfg(not(target_os = "macos"))] -+#[cfg(all())] - mod cross_platform; - --#[cfg(not(target_os = "macos"))] -+#[cfg(all())] - pub use cross_platform::*; -diff --git a/crates/workspace/src/shared_screen.rs b/crates/workspace/src/shared_screen.rs -index 1d17cfa145..f845234987 100644 ---- a/crates/workspace/src/shared_screen.rs -+++ b/crates/workspace/src/shared_screen.rs -@@ -1,11 +1,11 @@ --#[cfg(target_os = "macos")] -+#[cfg(any())] - mod macos; - --#[cfg(target_os = "macos")] -+#[cfg(any())] - pub use macos::*; - --#[cfg(not(target_os = "macos"))] -+#[cfg(all())] - mod cross_platform; - --#[cfg(not(target_os = "macos"))] -+#[cfg(all())] - pub use cross_platform::*; diff --git a/typos.toml b/typos.toml index 7c8bd02c74..9500d8f895 100644 --- a/typos.toml +++ b/typos.toml @@ -41,8 +41,6 @@ extend-exclude = [ "docs/theme/css/", # Spellcheck triggers on `|Fixe[sd]|` regex part. "script/danger/dangerfile.ts", - # Hashes are not typos - "script/patches/use-cross-platform-livekit.patch" ] [default]