WIP, frames are being sent to the other end

Cargo.lock

@@ -1193,6 +1193,7 @@ dependencies = [
  "fs2",
  "futures 0.3.28",
  "gpui2",
+ "image",
  "language2",
  "live_kit_client2",
  "log",
@@ -1204,6 +1205,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "settings2",
+ "smallvec",
  "util",
  "workspace2",
 ]

crates/call2/Cargo.toml

@@ -36,11 +36,13 @@ async-trait.workspace = true
 anyhow.workspace = true
 async-broadcast = "0.4"
 futures.workspace = true
+image = "0.23"
 postage.workspace = true
 schemars.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 serde_derive.workspace = true
+smallvec.workspace = true
 
 [dev-dependencies]
 client = { package = "client2", path = "../client2", features = ["test-support"] }

crates/call2/src/call2.rs

@@ -1,6 +1,7 @@
 pub mod call_settings;
 pub mod participant;
 pub mod room;
+mod shared_screen;
 
 use anyhow::{anyhow, bail, Result};
 use async_trait::async_trait;
@@ -14,7 +15,7 @@ use collections::HashSet;
 use futures::{channel::oneshot, future::Shared, Future, FutureExt};
 use gpui::{
     AppContext, AsyncAppContext, AsyncWindowContext, Context, EventEmitter, Model, ModelContext,
-    Subscription, Task, View, ViewContext, WeakModel, WeakView,
+    Subscription, Task, View, ViewContext, VisualContext, WeakModel, WeakView,
 };
 pub use participant::ParticipantLocation;
 use participant::RemoteParticipant;
@@ -23,6 +24,7 @@ use project::Project;
 use room::Event;
 pub use room::Room;
 use settings::Settings;
+use shared_screen::SharedScreen;
 use std::sync::Arc;
 use util::ResultExt;
 use workspace::{item::ItemHandle, CallHandler, Pane, Workspace};
@@ -587,24 +589,27 @@ impl CallHandler for Call {
     fn shared_screen_for_peer(
         &self,
         peer_id: PeerId,
-        _pane: &View<Pane>,
+        pane: &View<Pane>,
         cx: &mut ViewContext<Workspace>,
     ) -> Option<Box<dyn ItemHandle>> {
         let (call, _) = self.active_call.as_ref()?;
+        dbg!("A");
         let room = call.read(cx).room()?.read(cx);
+        dbg!("B");
         let participant = room.remote_participant_for_peer_id(peer_id)?;
-        let _track = participant.video_tracks.values().next()?.clone();
+        dbg!("C");
-        let _user = participant.user.clone();
+        let track = participant.video_tracks.values().next()?.clone();
-        todo!();
+        dbg!("D");
-        // for item in pane.read(cx).items_of_type::<SharedScreen>() {
+        let user = participant.user.clone();
-        //     if item.read(cx).peer_id == peer_id {
+        for item in pane.read(cx).items_of_type::<SharedScreen>() {
-        //         return Box::new(Some(item));
+            if item.read(cx).peer_id == peer_id {
-        //     }
+                return Some(Box::new(item));
-        // }
+            }
+        }
 
-        // Some(Box::new(cx.build_view(|cx| {
+        Some(Box::new(cx.build_view(|cx| {
-        //     SharedScreen::new(&track, peer_id, user.clone(), cx)
+            SharedScreen::new(&track, peer_id, user.clone(), cx)
-        // })))
+        })))
     }
     fn room_id(&self, cx: &AppContext) -> Option<u64> {
         Some(self.active_call.as_ref()?.0.read(cx).room()?.read(cx).id())
@@ -629,7 +634,7 @@ impl CallHandler for Call {
             this.invite(called_user_id, initial_project, cx)
         })
     }
-    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<Arc<User>>> {
+    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<(Arc<User>, PeerId)>> {
         self.active_call
             .as_ref()
             .map(|call| {
@@ -637,7 +642,9 @@ impl CallHandler for Call {
                     room.read(cx)
                         .remote_participants()
                         .iter()
-                        .map(|participant| participant.1.user.clone())
+                        .map(|participant| {
+                            (participant.1.user.clone(), participant.1.peer_id.clone())
+                        })
                         .collect()
                 })
             })
@@ -665,6 +672,27 @@ impl CallHandler for Call {
             })
         });
     }
+    fn toggle_screen_share(&self, cx: &mut AppContext) {
+        self.active_call.as_ref().map(|call| {
+            call.0.update(cx, |this, cx| {
+                this.room().map(|room| {
+                    room.update(cx, |this, cx| {
+                        if this.is_screen_sharing() {
+                            dbg!("Unsharing");
+                            this.unshare_screen(cx);
+                        } else {
+                            dbg!("Sharing");
+                            let t = this.share_screen(cx);
+                            cx.spawn(move |_, cx| async move {
+                                t.await.log_err();
+                            })
+                            .detach();
+                        }
+                    })
+                })
+            })
+        });
+    }
 }
 
 #[cfg(test)]

crates/call2/src/participant.rs

@@ -4,7 +4,7 @@ use client::{proto, User};
 use collections::HashMap;
 use gpui::WeakModel;
 pub use live_kit_client::Frame;
-use live_kit_client::{RemoteAudioTrack, RemoteVideoTrack};
+pub(crate) use live_kit_client::{RemoteAudioTrack, RemoteVideoTrack};
 use project::Project;
 use std::sync::Arc;
 

crates/call2/src/room.rs

@@ -1345,6 +1345,8 @@ impl Room {
                 let display = displays
                     .first()
                     .ok_or_else(|| anyhow!("no display found"))?;
+                dbg!("Been there");
+                dbg!(displays.len());
                 let track = LocalVideoTrack::screen_share_for_display(&display);
                 this.upgrade()
                     .ok_or_else(|| anyhow!("room was dropped"))?

crates/call2/src/shared_screen.rs

@@ -0,0 +1,159 @@
+use crate::participant::{Frame, RemoteVideoTrack};
+use anyhow::Result;
+use client::{proto::PeerId, User};
+use futures::StreamExt;
+use gpui::{
+    div, img, AppContext, Div, Element, Entity, EventEmitter, FocusHandle, FocusableView,
+    ImageData, Img, MouseButton, ParentElement, Render, SharedString, Task, View, ViewContext,
+    VisualContext, WindowContext,
+};
+use smallvec::SmallVec;
+use std::{
+    borrow::Cow,
+    sync::{Arc, Weak},
+};
+use workspace::{
+    item::{Item, ItemEvent},
+    ItemNavHistory, WorkspaceId,
+};
+
+pub enum Event {
+    Close,
+}
+
+pub struct SharedScreen {
+    track: Weak<RemoteVideoTrack>,
+    frame: Option<Frame>,
+    pub peer_id: PeerId,
+    user: Arc<User>,
+    nav_history: Option<ItemNavHistory>,
+    _maintain_frame: Task<Result<()>>,
+    focus: FocusHandle,
+}
+
+impl SharedScreen {
+    pub fn new(
+        track: &Arc<RemoteVideoTrack>,
+        peer_id: PeerId,
+        user: Arc<User>,
+        cx: &mut ViewContext<Self>,
+    ) -> Self {
+        cx.focus_handle();
+        let mut frames = track.frames();
+        Self {
+            track: Arc::downgrade(track),
+            frame: None,
+            peer_id,
+            user,
+            nav_history: Default::default(),
+            _maintain_frame: cx.spawn(|this, mut cx| async move {
+                while let Some(frame) = frames.next().await {
+                    this.update(&mut cx, |this, cx| {
+                        this.frame = Some(frame);
+                        cx.notify();
+                    })?;
+                }
+                this.update(&mut cx, |_, cx| cx.emit(Event::Close))?;
+                Ok(())
+            }),
+            focus: cx.focus_handle(),
+        }
+    }
+}
+
+impl EventEmitter<Event> for SharedScreen {}
+impl EventEmitter<workspace::item::ItemEvent> for SharedScreen {}
+
+impl FocusableView for SharedScreen {
+    fn focus_handle(&self, cx: &AppContext) -> FocusHandle {
+        self.focus.clone()
+    }
+}
+impl Render for SharedScreen {
+    type Element = Div;
+    fn render(&mut self, cx: &mut ViewContext<Self>) -> Self::Element {
+        let frame = self.frame.clone();
+        div().children(frame.map(|frame| {
+            img().data(Arc::new(ImageData::new(image::ImageBuffer::new(
+                frame.width() as u32,
+                frame.height() as u32,
+            ))))
+        }))
+    }
+}
+// impl View for SharedScreen {
+//     fn ui_name() -> &'static str {
+//         "SharedScreen"
+//     }
+
+//     fn render(&mut self, cx: &mut ViewContext<Self>) -> AnyElement<Self> {
+//         enum Focus {}
+
+//         let frame = self.frame.clone();
+//         MouseEventHandler::new::<Focus, _>(0, cx, |_, cx| {
+//             Canvas::new(move |bounds, _, _, cx| {
+//                 if let Some(frame) = frame.clone() {
+//                     let size = constrain_size_preserving_aspect_ratio(
+//                         bounds.size(),
+//                         vec2f(frame.width() as f32, frame.height() as f32),
+//                     );
+//                     let origin = bounds.origin() + (bounds.size() / 2.) - size / 2.;
+//                     cx.scene().push_surface(gpui::platform::mac::Surface {
+//                         bounds: RectF::new(origin, size),
+//                         image_buffer: frame.image(),
+//                     });
+//                 }
+//             })
+//             .contained()
+//             .with_style(theme::current(cx).shared_screen)
+//         })
+//         .on_down(MouseButton::Left, |_, _, cx| cx.focus_parent())
+//         .into_any()
+//     }
+// }
+
+impl Item for SharedScreen {
+    fn tab_tooltip_text(&self, _: &AppContext) -> Option<SharedString> {
+        Some(format!("{}'s screen", self.user.github_login).into())
+    }
+    fn deactivated(&mut self, cx: &mut ViewContext<Self>) {
+        if let Some(nav_history) = self.nav_history.as_mut() {
+            nav_history.push::<()>(None, cx);
+        }
+    }
+
+    fn tab_content(&self, _: Option<usize>, _: &WindowContext<'_>) -> gpui::AnyElement {
+        div().child("Shared screen").into_any()
+        // Flex::row()
+        //     .with_child(
+        //         Svg::new("icons/desktop.svg")
+        //             .with_color(style.label.text.color)
+        //             .constrained()
+        //             .with_width(style.type_icon_width)
+        //             .aligned()
+        //             .contained()
+        //             .with_margin_right(style.spacing),
+        //     )
+        //     .with_child(
+        //         Label::new(
+        //             format!("{}'s screen", self.user.github_login),
+        //             style.label.clone(),
+        //         )
+        //         .aligned(),
+        //     )
+        //     .into_any()
+    }
+
+    fn set_nav_history(&mut self, history: ItemNavHistory, _: &mut ViewContext<Self>) {
+        self.nav_history = Some(history);
+    }
+
+    fn clone_on_split(
+        &self,
+        _workspace_id: WorkspaceId,
+        cx: &mut ViewContext<Self>,
+    ) -> Option<View<Self>> {
+        let track = self.track.upgrade()?;
+        Some(cx.build_view(|cx| Self::new(&track, self.peer_id, self.user.clone(), cx)))
+    }
+}

crates/collab_ui2/src/collab_titlebar_item.rs

@@ -31,9 +31,9 @@ use std::sync::Arc;
 use call::ActiveCall;
 use client::{Client, UserStore};
 use gpui::{
-    div, px, rems, AppContext, Div, InteractiveElement, IntoElement, Model, ParentElement, Render,
+    div, px, rems, AppContext, Div, InteractiveElement, IntoElement, Model, MouseButton,
-    Stateful, StatefulInteractiveElement, Styled, Subscription, ViewContext, VisualContext,
+    ParentElement, Render, Stateful, StatefulInteractiveElement, Styled, Subscription, ViewContext,
-    WeakView, WindowBounds,
+    VisualContext, WeakView, WindowBounds,
 };
 use project::Project;
 use theme::ActiveTheme;
@@ -182,12 +182,22 @@ impl Render for CollabTitlebarItem {
                         current_user
                             .avatar
                             .clone()
-                            .map(|avatar| Avatar::new(avatar.clone()))
+                            .map(|avatar| div().child(Avatar::new(avatar.clone())))
                             .into_iter()
-                            .chain(remote_participants.into_iter().flat_map(|user| {
+                            .chain(remote_participants.into_iter().flat_map(|(user, peer_id)| {
-                                user.avatar
+                                user.avatar.as_ref().map(|avatar| {
-                                    .as_ref()
+                                    div()
-                                    .map(|avatar| Avatar::new(avatar.clone()))
+                                        .child(Avatar::new(avatar.clone()).into_element())
+                                        .on_mouse_down(MouseButton::Left, {
+                                            let workspace = workspace.clone();
+                                            let id = peer_id.clone();
+                                            move |_, cx| {
+                                                workspace.update(cx, |this, cx| {
+                                                    this.open_shared_screen(peer_id, cx);
+                                                });
+                                            }
+                                        })
+                                })
                             })),
                     )
                 },
@@ -202,15 +212,22 @@ impl Render for CollabTitlebarItem {
                         )
                         .child(
                             h_stack()
-                                .child(IconButton::new("mute-microphone", mic_icon).on_click(
+                                .child(IconButton::new("mute-microphone", mic_icon).on_click({
+                                    let workspace = workspace.clone();
                                     move |_, cx| {
                                         workspace.update(cx, |this, cx| {
                                             this.call_state().toggle_mute(cx);
                                         });
+                                    }
+                                }))
+                                .child(IconButton::new("mute-sound", ui::Icon::AudioOn))
+                                .child(IconButton::new("screen-share", ui::Icon::Screen).on_click(
+                                    move |_, cx| {
+                                        workspace.update(cx, |this, cx| {
+                                            this.call_state().toggle_screen_share(cx);
+                                        });
                                     },
                                 ))
-                                .child(IconButton::new("mute-sound", ui::Icon::AudioOn))
-                                .child(IconButton::new("screen-share", ui::Icon::Screen))
                                 .pl_2(),
                         ),
                 )

crates/workspace2/src/workspace2.rs

@@ -361,7 +361,7 @@ impl CallHandler for TestCallHandler {
         unimplemented!()
     }
 
-    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<User>> {
+    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<(Arc<User>, PeerId)>> {
         None
     }
 
@@ -370,6 +370,8 @@ impl CallHandler for TestCallHandler {
     }
 
     fn toggle_mute(&self, cx: &mut AppContext) {}
+
+    fn toggle_screen_share(&self, cx: &mut AppContext) {}
 }
 
 impl AppState {
@@ -480,9 +482,10 @@ pub trait CallHandler {
         initial_project: Option<Model<Project>>,
         cx: &mut AppContext,
     ) -> Task<Result<()>>;
-    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<Arc<User>>>;
+    fn remote_participants(&self, cx: &AppContext) -> Option<Vec<(Arc<User>, PeerId)>>;
     fn is_muted(&self, cx: &AppContext) -> Option<bool>;
     fn toggle_mute(&self, cx: &mut AppContext);
+    fn toggle_screen_share(&self, cx: &mut AppContext);
 }
 
 pub struct Workspace {
@@ -1996,13 +1999,15 @@ impl Workspace {
         item
     }
 
-    //     pub fn open_shared_screen(&mut self, peer_id: PeerId, cx: &mut ViewContext<Self>) {
+    pub fn open_shared_screen(&mut self, peer_id: PeerId, cx: &mut ViewContext<Self>) {
-    //         if let Some(shared_screen) = self.shared_screen_for_peer(peer_id, &self.active_pane, cx) {
+        if let Some(shared_screen) = self.shared_screen_for_peer(peer_id, &self.active_pane, cx) {
-    //             self.active_pane.update(cx, |pane, cx| {
+            self.active_pane.update(cx, |pane, cx| {
-    //                 pane.add_item(Box::new(shared_screen), false, true, None, cx)
+                pane.add_item(shared_screen, false, true, None, cx)
-    //             });
+            });
-    //         }
+        } else {
-    //     }
+            dbg!("WTF");
+        }
+    }
 
     pub fn activate_item(&mut self, item: &dyn ItemHandle, cx: &mut ViewContext<Self>) -> bool {
         let result = self.panes.iter().find_map(|pane| {
@@ -2877,10 +2882,10 @@ impl Workspace {
                 }
                 continue;
             }
-            // todo!()
+
-            // if let Some(shared_screen) = self.shared_screen_for_peer(leader_id, pane, cx) {
+            if let Some(shared_screen) = self.shared_screen_for_peer(leader_id, pane, cx) {
-            //     items_to_activate.push((pane.clone(), Box::new(shared_screen)));
+                items_to_activate.push((pane.clone(), shared_screen));
-            // }
+            }
         }
 
         for (pane, item) in items_to_activate {
@@ -2901,27 +2906,27 @@ impl Workspace {
         None
     }
 
-    // todo!()
+    fn shared_screen_for_peer(
-    //     fn shared_screen_for_peer(
+        &self,
-    //         &self,
+        peer_id: PeerId,
-    //         peer_id: PeerId,
+        pane: &View<Pane>,
-    //         pane: &View<Pane>,
+        cx: &mut ViewContext<Self>,
-    //         cx: &mut ViewContext<Self>,
+    ) -> Option<Box<dyn ItemHandle>> {
-    //     ) -> Option<View<SharedScreen>> {
+        self.call_handler.shared_screen_for_peer(peer_id, pane, cx)
-    //         let call = self.active_call()?;
+        // let call = self.active_call()?;
-    //         let room = call.read(cx).room()?.read(cx);
+        // let room = call.read(cx).room()?.read(cx);
-    //         let participant = room.remote_participant_for_peer_id(peer_id)?;
+        // let participant = room.remote_participant_for_peer_id(peer_id)?;
-    //         let track = participant.video_tracks.values().next()?.clone();
+        // let track = participant.video_tracks.values().next()?.clone();
-    //         let user = participant.user.clone();
+        // let user = participant.user.clone();
 
-    //         for item in pane.read(cx).items_of_type::<SharedScreen>() {
+        // for item in pane.read(cx).items_of_type::<SharedScreen>() {
-    //             if item.read(cx).peer_id == peer_id {
+        //     if item.read(cx).peer_id == peer_id {
-    //                 return Some(item);
+        //         return Some(item);
-    //             }
+        //     }
-    //         }
+        // }
 
-    //         Some(cx.build_view(|cx| SharedScreen::new(&track, peer_id, user.clone(), cx)))
+        // Some(cx.build_view(|cx| SharedScreen::new(&track, peer_id, user.clone(), cx)))
-    //     }
+    }
 
     pub fn on_window_activation_changed(&mut self, cx: &mut ViewContext<Self>) {
         if cx.is_window_active() {