Copy/paste images into editors (Mac only) (#15782)

For future reference: WIP branch of copy/pasting a mixture of images and
text: https://github.com/zed-industries/zed/tree/copy-paste-images -
we'll come back to that one after landing this one.

Release Notes:

- You can now paste images into the Assistant Panel to include them as
context. Currently works only on Mac, and with Anthropic models. Future
support is planned for more models, operating systems, and image
clipboard operations.

---------

Co-authored-by: Antonio <antonio@zed.dev>
Co-authored-by: Mikayla <mikayla@zed.dev>
Co-authored-by: Jason <jason@zed.dev>
Co-authored-by: Kyle <kylek@zed.dev>

crates/gpui/src/platform/mac/platform.rs

@@ -1,8 +1,13 @@
-use super::{events::key_to_native, BoolExt};
+use super::{
+    attributed_string::{NSAttributedString, NSMutableAttributedString},
+    events::key_to_native,
+    BoolExt,
+};
 use crate::{
-    Action, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor,
-    Keymap, MacDispatcher, MacDisplay, MacTextSystem, MacWindow, Menu, MenuItem, PathPromptOptions,
-    Platform, PlatformDisplay, PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task,
+    hash, Action, AnyWindowHandle, BackgroundExecutor, ClipboardEntry, ClipboardItem,
+    ClipboardString, CursorStyle, ForegroundExecutor, Image, ImageFormat, Keymap, MacDispatcher,
+    MacDisplay, MacTextSystem, MacWindow, Menu, MenuItem, PathPromptOptions, Platform,
+    PlatformDisplay, PlatformTextSystem, PlatformWindow, Result, SemanticVersion, Task,
     WindowAppearance, WindowParams,
 };
 use anyhow::anyhow;
@@ -11,16 +16,17 @@ use cocoa::{
     appkit::{
         NSApplication, NSApplicationActivationPolicy::NSApplicationActivationPolicyRegular,
         NSEventModifierFlags, NSMenu, NSMenuItem, NSModalResponse, NSOpenPanel, NSPasteboard,
-        NSPasteboardTypeString, NSSavePanel, NSWindow,
+        NSPasteboardTypePNG, NSPasteboardTypeRTF, NSPasteboardTypeRTFD, NSPasteboardTypeString,
+        NSPasteboardTypeTIFF, NSSavePanel, NSWindow,
     },
     base::{id, nil, selector, BOOL, YES},
     foundation::{
-        NSArray, NSAutoreleasePool, NSBundle, NSData, NSInteger, NSProcessInfo, NSString,
+        NSArray, NSAutoreleasePool, NSBundle, NSData, NSInteger, NSProcessInfo, NSRange, NSString,
         NSUInteger, NSURL,
     },
 };
 use core_foundation::{
-    base::{CFRelease, CFType, CFTypeRef, OSStatus, TCFType as _},
+    base::{CFRelease, CFType, CFTypeRef, OSStatus, TCFType},
     boolean::CFBoolean,
     data::CFData,
     dictionary::{CFDictionary, CFDictionaryRef, CFMutableDictionary},
@@ -50,6 +56,7 @@ use std::{
     slice, str,
     sync::Arc,
 };
+use strum::IntoEnumIterator;
 
 use super::renderer;
 
@@ -421,7 +428,7 @@ impl Platform for MacPlatform {
             pool.drain();
 
             (*app).set_ivar(MAC_PLATFORM_IVAR, null_mut::<c_void>());
-            (*app.delegate()).set_ivar(MAC_PLATFORM_IVAR, null_mut::<c_void>());
+            (*NSWindow::delegate(app)).set_ivar(MAC_PLATFORM_IVAR, null_mut::<c_void>());
         }
     }
 
@@ -749,7 +756,7 @@ impl Platform for MacPlatform {
             let app: id = msg_send![APP_CLASS, sharedApplication];
             let mut state = self.0.lock();
             let actions = &mut state.menu_actions;
-            app.setMainMenu_(self.create_menu_bar(menus, app.delegate(), actions, keymap));
+            app.setMainMenu_(self.create_menu_bar(menus, NSWindow::delegate(app), actions, keymap));
         }
     }
 
@@ -758,7 +765,7 @@ impl Platform for MacPlatform {
             let app: id = msg_send![APP_CLASS, sharedApplication];
             let mut state = self.0.lock();
             let actions = &mut state.menu_actions;
-            let new = self.create_dock_menu(menu, app.delegate(), actions, keymap);
+            let new = self.create_dock_menu(menu, NSWindow::delegate(app), actions, keymap);
             if let Some(old) = state.dock_menu.replace(new) {
                 CFRelease(old as _)
             }
@@ -851,79 +858,115 @@ impl Platform for MacPlatform {
     }
 
     fn write_to_clipboard(&self, item: ClipboardItem) {
-        let state = self.0.lock();
+        use crate::ClipboardEntry;
+
         unsafe {
-            state.pasteboard.clearContents();
+            // We only want to use NSAttributedString if there are multiple entries to write.
+            if item.entries.len() <= 1 {
+                match item.entries.first() {
+                    Some(entry) => match entry {
+                        ClipboardEntry::String(string) => {
+                            self.write_plaintext_to_clipboard(string);
+                        }
+                        ClipboardEntry::Image(image) => {
+                            self.write_image_to_clipboard(image);
+                        }
+                    },
+                    None => {
+                        // Writing an empty list of entries just clears the clipboard.
+                        let state = self.0.lock();
+                        state.pasteboard.clearContents();
+                    }
+                }
+            } else {
+                let mut any_images = false;
+                let attributed_string = {
+                    let mut buf = NSMutableAttributedString::alloc(nil)
+                        // TODO can we skip this? Or at least part of it?
+                        .init_attributed_string(NSString::alloc(nil).init_str(""));
 
-            let text_bytes = NSData::dataWithBytes_length_(
-                nil,
-                item.text.as_ptr() as *const c_void,
-                item.text.len() as u64,
-            );
-            state
-                .pasteboard
-                .setData_forType(text_bytes, NSPasteboardTypeString);
+                    for entry in item.entries {
+                        if let ClipboardEntry::String(ClipboardString { text, metadata: _ }) = entry
+                        {
+                            let to_append = NSAttributedString::alloc(nil)
+                                .init_attributed_string(NSString::alloc(nil).init_str(&text));
 
-            if let Some(metadata) = item.metadata.as_ref() {
-                let hash_bytes = ClipboardItem::text_hash(&item.text).to_be_bytes();
-                let hash_bytes = NSData::dataWithBytes_length_(
-                    nil,
-                    hash_bytes.as_ptr() as *const c_void,
-                    hash_bytes.len() as u64,
-                );
+                            buf.appendAttributedString_(to_append);
+                        }
+                    }
+
+                    buf
+                };
+
+                let state = self.0.lock();
+                state.pasteboard.clearContents();
+
+                // Only set rich text clipboard types if we actually have 1+ images to include.
+                if any_images {
+                    let rtfd_data = attributed_string.RTFDFromRange_documentAttributes_(
+                        NSRange::new(0, msg_send![attributed_string, length]),
+                        nil,
+                    );
+                    if rtfd_data != nil {
+                        state
+                            .pasteboard
+                            .setData_forType(rtfd_data, NSPasteboardTypeRTFD);
+                    }
+
+                    let rtf_data = attributed_string.RTFFromRange_documentAttributes_(
+                        NSRange::new(0, attributed_string.length()),
+                        nil,
+                    );
+                    if rtf_data != nil {
+                        state
+                            .pasteboard
+                            .setData_forType(rtf_data, NSPasteboardTypeRTF);
+                    }
+                }
+
+                let plain_text = attributed_string.string();
                 state
                     .pasteboard
-                    .setData_forType(hash_bytes, state.text_hash_pasteboard_type);
-
-                let metadata_bytes = NSData::dataWithBytes_length_(
-                    nil,
-                    metadata.as_ptr() as *const c_void,
-                    metadata.len() as u64,
-                );
-                state
-                    .pasteboard
-                    .setData_forType(metadata_bytes, state.metadata_pasteboard_type);
+                    .setString_forType(plain_text, NSPasteboardTypeString);
             }
         }
     }
 
     fn read_from_clipboard(&self) -> Option<ClipboardItem> {
         let state = self.0.lock();
-        unsafe {
-            if let Some(text_bytes) =
-                self.read_from_pasteboard(state.pasteboard, NSPasteboardTypeString)
-            {
-                let text = String::from_utf8_lossy(text_bytes).to_string();
-                let hash_bytes = self
-                    .read_from_pasteboard(state.pasteboard, state.text_hash_pasteboard_type)
-                    .and_then(|bytes| bytes.try_into().ok())
-                    .map(u64::from_be_bytes);
-                let metadata_bytes = self
-                    .read_from_pasteboard(state.pasteboard, state.metadata_pasteboard_type)
-                    .and_then(|bytes| String::from_utf8(bytes.to_vec()).ok());
+        let pasteboard = state.pasteboard;
 
-                if let Some((hash, metadata)) = hash_bytes.zip(metadata_bytes) {
-                    if hash == ClipboardItem::text_hash(&text) {
-                        Some(ClipboardItem {
-                            text,
-                            metadata: Some(metadata),
-                        })
-                    } else {
-                        Some(ClipboardItem {
-                            text,
-                            metadata: None,
-                        })
-                    }
+        // First, see if it's a string.
+        unsafe {
+            let types: id = pasteboard.types();
+            let string_type: id = ns_string("public.utf8-plain-text");
+
+            if msg_send![types, containsObject: string_type] {
+                let data = pasteboard.dataForType(string_type);
+                if data == nil {
+                    return None;
+                } else if data.bytes().is_null() {
+                    // https://developer.apple.com/documentation/foundation/nsdata/1410616-bytes?language=objc
+                    // "If the length of the NSData object is 0, this property returns nil."
+                    return Some(self.read_string_from_clipboard(&state, &[]));
                 } else {
-                    Some(ClipboardItem {
-                        text,
-                        metadata: None,
-                    })
+                    let bytes =
+                        slice::from_raw_parts(data.bytes() as *mut u8, data.length() as usize);
+
+                    return Some(self.read_string_from_clipboard(&state, bytes));
+                }
+            }
+
+            // If it wasn't a string, try the various supported image types.
+            for format in ImageFormat::iter() {
+                if let Some(item) = try_clipboard_image(pasteboard, format) {
+                    return Some(item);
                 }
-            } else {
-                None
             }
         }
+
+        // If it wasn't a string or a supported image type, give up.
+        None
     }
 
     fn write_credentials(&self, url: &str, username: &str, password: &[u8]) -> Task<Result<()>> {
@@ -1038,6 +1081,110 @@ impl Platform for MacPlatform {
     }
 }
 
+impl MacPlatform {
+    unsafe fn read_string_from_clipboard(
+        &self,
+        state: &MacPlatformState,
+        text_bytes: &[u8],
+    ) -> ClipboardItem {
+        let text = String::from_utf8_lossy(text_bytes).to_string();
+        let metadata = self
+            .read_from_pasteboard(state.pasteboard, state.text_hash_pasteboard_type)
+            .and_then(|hash_bytes| {
+                let hash_bytes = hash_bytes.try_into().ok()?;
+                let hash = u64::from_be_bytes(hash_bytes);
+                let metadata =
+                    self.read_from_pasteboard(state.pasteboard, state.metadata_pasteboard_type)?;
+
+                if hash == ClipboardString::text_hash(&text) {
+                    String::from_utf8(metadata.to_vec()).ok()
+                } else {
+                    None
+                }
+            });
+
+        ClipboardItem {
+            entries: vec![ClipboardEntry::String(ClipboardString { text, metadata })],
+        }
+    }
+
+    unsafe fn write_plaintext_to_clipboard(&self, string: &ClipboardString) {
+        let state = self.0.lock();
+        state.pasteboard.clearContents();
+
+        let text_bytes = NSData::dataWithBytes_length_(
+            nil,
+            string.text.as_ptr() as *const c_void,
+            string.text.len() as u64,
+        );
+        state
+            .pasteboard
+            .setData_forType(text_bytes, NSPasteboardTypeString);
+
+        if let Some(metadata) = string.metadata.as_ref() {
+            let hash_bytes = ClipboardString::text_hash(&string.text).to_be_bytes();
+            let hash_bytes = NSData::dataWithBytes_length_(
+                nil,
+                hash_bytes.as_ptr() as *const c_void,
+                hash_bytes.len() as u64,
+            );
+            state
+                .pasteboard
+                .setData_forType(hash_bytes, state.text_hash_pasteboard_type);
+
+            let metadata_bytes = NSData::dataWithBytes_length_(
+                nil,
+                metadata.as_ptr() as *const c_void,
+                metadata.len() as u64,
+            );
+            state
+                .pasteboard
+                .setData_forType(metadata_bytes, state.metadata_pasteboard_type);
+        }
+    }
+
+    unsafe fn write_image_to_clipboard(&self, image: &Image) {
+        let state = self.0.lock();
+        state.pasteboard.clearContents();
+
+        let bytes = NSData::dataWithBytes_length_(
+            nil,
+            image.bytes.as_ptr() as *const c_void,
+            image.bytes.len() as u64,
+        );
+
+        state
+            .pasteboard
+            .setData_forType(bytes, Into::<UTType>::into(image.format).inner_mut());
+    }
+}
+
+fn try_clipboard_image(pasteboard: id, format: ImageFormat) -> Option<ClipboardItem> {
+    let mut ut_type: UTType = format.into();
+
+    unsafe {
+        let types: id = pasteboard.types();
+        if msg_send![types, containsObject: ut_type.inner()] {
+            let data = pasteboard.dataForType(ut_type.inner_mut());
+            if data == nil {
+                None
+            } else {
+                let bytes = Vec::from(slice::from_raw_parts(
+                    data.bytes() as *mut u8,
+                    data.length() as usize,
+                ));
+                let id = hash(&bytes);
+
+                Some(ClipboardItem {
+                    entries: vec![ClipboardEntry::Image(Image { format, bytes, id })],
+                })
+            }
+        } else {
+            None
+        }
+    }
+}
+
 unsafe fn path_from_objc(path: id) -> PathBuf {
     let len = msg_send![path, lengthOfBytesUsingEncoding: NSUTF8StringEncoding];
     let bytes = path.UTF8String() as *const u8;
@@ -1216,6 +1363,68 @@ mod security {
     pub const errSecItemNotFound: OSStatus = -25300;
 }
 
+impl From<ImageFormat> for UTType {
+    fn from(value: ImageFormat) -> Self {
+        match value {
+            ImageFormat::Png => Self::png(),
+            ImageFormat::Jpeg => Self::jpeg(),
+            ImageFormat::Tiff => Self::tiff(),
+            ImageFormat::Webp => Self::webp(),
+            ImageFormat::Gif => Self::gif(),
+            ImageFormat::Bmp => Self::bmp(),
+            ImageFormat::Svg => Self::svg(),
+        }
+    }
+}
+
+// See https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/
+struct UTType(id);
+
+impl UTType {
+    pub fn png() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/png
+        Self(unsafe { NSPasteboardTypePNG }) // This is a rare case where there's a built-in NSPasteboardType
+    }
+
+    pub fn jpeg() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/jpeg
+        Self(unsafe { ns_string("public.jpeg") })
+    }
+
+    pub fn gif() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/gif
+        Self(unsafe { ns_string("com.compuserve.gif") })
+    }
+
+    pub fn webp() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/webp
+        Self(unsafe { ns_string("org.webmproject.webp") })
+    }
+
+    pub fn bmp() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/bmp
+        Self(unsafe { ns_string("com.microsoft.bmp") })
+    }
+
+    pub fn svg() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/svg
+        Self(unsafe { ns_string("public.svg-image") })
+    }
+
+    pub fn tiff() -> Self {
+        // https://developer.apple.com/documentation/uniformtypeidentifiers/uttype-swift.struct/tiff
+        Self(unsafe { NSPasteboardTypeTIFF }) // This is a rare case where there's a built-in NSPasteboardType
+    }
+
+    fn inner(&self) -> *const Object {
+        self.0
+    }
+
+    fn inner_mut(&mut self) -> *mut Object {
+        self.0 as *mut _
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use crate::ClipboardItem;
@@ -1227,11 +1436,15 @@ mod tests {
         let platform = build_platform();
         assert_eq!(platform.read_from_clipboard(), None);
 
-        let item = ClipboardItem::new("1".to_string());
+        let item = ClipboardItem::new_string("1".to_string());
         platform.write_to_clipboard(item.clone());
         assert_eq!(platform.read_from_clipboard(), Some(item));
 
-        let item = ClipboardItem::new("2".to_string()).with_metadata(vec![3, 4]);
+        let item = ClipboardItem {
+            entries: vec![ClipboardEntry::String(
+                ClipboardString::new("2".to_string()).with_json_metadata(vec![3, 4]),
+            )],
+        };
         platform.write_to_clipboard(item.clone());
         assert_eq!(platform.read_from_clipboard(), Some(item));
 
@@ -1250,7 +1463,7 @@ mod tests {
         }
         assert_eq!(
             platform.read_from_clipboard(),
-            Some(ClipboardItem::new(text_from_other_app.to_string()))
+            Some(ClipboardItem::new_string(text_from_other_app.to_string()))
         );
     }