Improve window decorations: check for compositor support (#13822)

Adds the `compositor_support` to the `X11WindowState` struct so that
correct window decorations are selected

Release notes:

- N/A

---------

Co-authored-by: Thorsten Ball <mrnugget@gmail.com>
Co-authored-by: Conrad Irwin <conrad.irwin@gmail.com>

crates/gpui/src/platform/linux/x11/client.rs

@@ -105,6 +105,7 @@ pub struct X11ClientState {
     pub(crate) scale_factor: f32,
 
     pub(crate) xcb_connection: Rc<XCBConnection>,
+    client_side_decorations_supported: bool,
     pub(crate) x_root_index: usize,
     pub(crate) _resource_database: Database,
     pub(crate) atoms: XcbAtoms,
@@ -224,13 +225,25 @@ impl X11Client {
             .map(|class| *class)
             .collect::<Vec<_>>();
 
-        let atoms = XcbAtoms::new(&xcb_connection).unwrap();
+        let atoms = XcbAtoms::new(&xcb_connection).unwrap().reply().unwrap();
+
+        let root = xcb_connection.setup().roots[0].root;
+        let compositor_present = check_compositor_present(&xcb_connection, root);
+        let gtk_frame_extents_supported =
+            check_gtk_frame_extents_supported(&xcb_connection, &atoms, root);
+        let client_side_decorations_supported = compositor_present && gtk_frame_extents_supported;
+        log::info!(
+            "x11: compositor present: {}, gtk_frame_extents_supported: {}",
+            compositor_present,
+            gtk_frame_extents_supported
+        );
+
         let xkb = xcb_connection
             .xkb_use_extension(XKB_X11_MIN_MAJOR_XKB_VERSION, XKB_X11_MIN_MINOR_XKB_VERSION)
+            .unwrap()
+            .reply()
             .unwrap();
 
-        let atoms = atoms.reply().unwrap();
-        let xkb = xkb.reply().unwrap();
         let events = xkb::EventType::STATE_NOTIFY;
         xcb_connection
             .xkb_select_events(
@@ -328,6 +341,7 @@ impl X11Client {
             scale_factor,
 
             xcb_connection,
+            client_side_decorations_supported,
             x_root_index,
             _resource_database: resource_database,
             atoms,
@@ -1052,6 +1066,7 @@ impl LinuxClient for X11Client {
             state.common.foreground_executor.clone(),
             params,
             &state.xcb_connection,
+            state.client_side_decorations_supported,
             state.x_root_index,
             x_window,
             &state.atoms,
@@ -1279,3 +1294,104 @@ pub fn mode_refresh_rate(mode: &randr::ModeInfo) -> Duration {
 fn fp3232_to_f32(value: xinput::Fp3232) -> f32 {
     value.integral as f32 + value.frac as f32 / u32::MAX as f32
 }
+
+fn check_compositor_present(xcb_connection: &XCBConnection, root: u32) -> bool {
+    // Method 1: Check for _NET_WM_CM_S{root}
+    let atom_name = format!("_NET_WM_CM_S{}", root);
+    let atom = xcb_connection
+        .intern_atom(false, atom_name.as_bytes())
+        .unwrap()
+        .reply()
+        .map(|reply| reply.atom)
+        .unwrap_or(0);
+
+    let method1 = if atom != 0 {
+        xcb_connection
+            .get_selection_owner(atom)
+            .unwrap()
+            .reply()
+            .map(|reply| reply.owner != 0)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    // Method 2: Check for _NET_WM_CM_OWNER
+    let atom_name = "_NET_WM_CM_OWNER";
+    let atom = xcb_connection
+        .intern_atom(false, atom_name.as_bytes())
+        .unwrap()
+        .reply()
+        .map(|reply| reply.atom)
+        .unwrap_or(0);
+
+    let method2 = if atom != 0 {
+        xcb_connection
+            .get_property(false, root, atom, xproto::AtomEnum::WINDOW, 0, 1)
+            .unwrap()
+            .reply()
+            .map(|reply| reply.value_len > 0)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    // Method 3: Check for _NET_SUPPORTING_WM_CHECK
+    let atom_name = "_NET_SUPPORTING_WM_CHECK";
+    let atom = xcb_connection
+        .intern_atom(false, atom_name.as_bytes())
+        .unwrap()
+        .reply()
+        .map(|reply| reply.atom)
+        .unwrap_or(0);
+
+    let method3 = if atom != 0 {
+        xcb_connection
+            .get_property(false, root, atom, xproto::AtomEnum::WINDOW, 0, 1)
+            .unwrap()
+            .reply()
+            .map(|reply| reply.value_len > 0)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    // TODO: Remove this
+    log::info!(
+        "Compositor detection: _NET_WM_CM_S?={}, _NET_WM_CM_OWNER={}, _NET_SUPPORTING_WM_CHECK={}",
+        method1,
+        method2,
+        method3
+    );
+
+    method1 || method2 || method3
+}
+
+fn check_gtk_frame_extents_supported(
+    xcb_connection: &XCBConnection,
+    atoms: &XcbAtoms,
+    root: xproto::Window,
+) -> bool {
+    let supported_atoms = xcb_connection
+        .get_property(
+            false,
+            root,
+            atoms._NET_SUPPORTED,
+            xproto::AtomEnum::ATOM,
+            0,
+            1024,
+        )
+        .unwrap()
+        .reply()
+        .map(|reply| {
+            // Convert Vec<u8> to Vec<u32>
+            reply
+                .value
+                .chunks_exact(4)
+                .map(|chunk| u32::from_ne_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]))
+                .collect::<Vec<u32>>()
+        })
+        .unwrap_or_default();
+
+    supported_atoms.contains(&atoms._GTK_FRAME_EXTENTS)
+}

crates/gpui/src/platform/linux/x11/window.rs

@@ -50,6 +50,7 @@ x11rb::atom_manager! {
         _NET_WM_WINDOW_TYPE,
         _NET_WM_WINDOW_TYPE_NOTIFICATION,
         _NET_WM_SYNC,
+        _NET_SUPPORTED,
         _MOTIF_WM_HINTS,
         _GTK_SHOW_WINDOW_MENU,
         _GTK_FRAME_EXTENTS,
@@ -238,6 +239,7 @@ pub struct X11WindowState {
     hidden: bool,
     active: bool,
     fullscreen: bool,
+    client_side_decorations_supported: bool,
     decorations: WindowDecorations,
     edge_constraints: Option<EdgeConstraints>,
     pub handle: AnyWindowHandle,
@@ -293,6 +295,7 @@ impl X11WindowState {
         executor: ForegroundExecutor,
         params: WindowParams,
         xcb_connection: &Rc<XCBConnection>,
+        client_side_decorations_supported: bool,
         x_main_screen_index: usize,
         x_window: xproto::Window,
         atoms: &XcbAtoms,
@@ -513,6 +516,7 @@ impl X11WindowState {
             handle,
             background_appearance: WindowBackgroundAppearance::Opaque,
             destroyed: false,
+            client_side_decorations_supported,
             decorations: WindowDecorations::Server,
             last_insets: [0, 0, 0, 0],
             edge_constraints: None,
@@ -581,6 +585,7 @@ impl X11Window {
         executor: ForegroundExecutor,
         params: WindowParams,
         xcb_connection: &Rc<XCBConnection>,
+        client_side_decorations_supported: bool,
         x_main_screen_index: usize,
         x_window: xproto::Window,
         atoms: &XcbAtoms,
@@ -594,6 +599,7 @@ impl X11Window {
                 executor,
                 params,
                 xcb_connection,
+                client_side_decorations_supported,
                 x_main_screen_index,
                 x_window,
                 atoms,
@@ -1227,6 +1233,11 @@ impl PlatformWindow for X11Window {
     fn window_decorations(&self) -> crate::Decorations {
         let state = self.0.state.borrow();
 
+        // Client window decorations require compositor support
+        if !state.client_side_decorations_supported {
+            return Decorations::Server;
+        }
+
         match state.decorations {
             WindowDecorations::Server => Decorations::Server,
             WindowDecorations::Client => {
@@ -1293,15 +1304,24 @@ impl PlatformWindow for X11Window {
         }
     }
 
-    fn request_decorations(&self, decorations: crate::WindowDecorations) {
+    fn request_decorations(&self, mut decorations: crate::WindowDecorations) {
+        let mut state = self.0.state.borrow_mut();
+
+        if matches!(decorations, crate::WindowDecorations::Client)
+            && !state.client_side_decorations_supported
+        {
+            log::info!(
+                "x11: no compositor present, falling back to server-side window decorations"
+            );
+            decorations = crate::WindowDecorations::Server;
+        }
+
         // https://github.com/rust-windowing/winit/blob/master/src/platform_impl/linux/x11/util/hint.rs#L53-L87
         let hints_data: [u32; 5] = match decorations {
             WindowDecorations::Server => [1 << 1, 0, 1, 0, 0],
             WindowDecorations::Client => [1 << 1, 0, 0, 0, 0],
         };
 
-        let mut state = self.0.state.borrow_mut();
-
         self.0
             .xcb_connection
             .change_property(