Copy/paste images into editors (Mac only) (#15782)
For future reference: WIP branch of copy/pasting a mixture of images and text: https://github.com/zed-industries/zed/tree/copy-paste-images - we'll come back to that one after landing this one. Release Notes: - You can now paste images into the Assistant Panel to include them as context. Currently works only on Mac, and with Anthropic models. Future support is planned for more models, operating systems, and image clipboard operations. --------- Co-authored-by: Antonio <antonio@zed.dev> Co-authored-by: Mikayla <mikayla@zed.dev> Co-authored-by: Jason <jason@zed.dev> Co-authored-by: Kyle <kylek@zed.dev>
This commit is contained in:
Richard Feldman
GitHub
parent
e3b0de5dda
commit
b1a581e81b
58 changed files with 2983 additions and 1708 deletions
|
|
@ -20,21 +20,25 @@ mod test;
|
|||
mod windows;
|
||||
|
||||
use crate::{
|
||||
point, Action, AnyWindowHandle, AsyncWindowContext, BackgroundExecutor, Bounds, DevicePixels,
|
||||
DispatchEventResult, Font, FontId, FontMetrics, FontRun, ForegroundExecutor, GPUSpecs, GlyphId,
|
||||
Keymap, LineLayout, Pixels, PlatformInput, Point, RenderGlyphParams, RenderImageParams,
|
||||
RenderSvgParams, Scene, SharedString, Size, Task, TaskLabel, WindowContext,
|
||||
DEFAULT_WINDOW_SIZE,
|
||||
point, Action, AnyWindowHandle, AppContext, AsyncWindowContext, BackgroundExecutor, Bounds,
|
||||
DevicePixels, DispatchEventResult, Font, FontId, FontMetrics, FontRun, ForegroundExecutor,
|
||||
GPUSpecs, GlyphId, ImageSource, Keymap, LineLayout, Pixels, PlatformInput, Point,
|
||||
RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, Scene, SharedString, Size,
|
||||
SvgSize, Task, TaskLabel, WindowContext, DEFAULT_WINDOW_SIZE,
|
||||
};
|
||||
use anyhow::Result;
|
||||
use async_task::Runnable;
|
||||
use futures::channel::oneshot;
|
||||
use image::codecs::gif::GifDecoder;
|
||||
use image::{AnimationDecoder as _, Frame};
|
||||
use parking::Unparker;
|
||||
use raw_window_handle::{HasDisplayHandle, HasWindowHandle};
|
||||
use seahash::SeaHasher;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use smallvec::SmallVec;
|
||||
use std::borrow::Cow;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::io::Cursor;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::{
|
||||
fmt::{self, Debug},
|
||||
|
|
@ -43,6 +47,7 @@ use std::{
|
|||
rc::Rc,
|
||||
sync::Arc,
|
||||
};
|
||||
use strum::EnumIter;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub use app_menu::*;
|
||||
|
|
@ -969,12 +974,210 @@ impl Default for CursorStyle {
|
|||
/// A clipboard item that should be copied to the clipboard
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ClipboardItem {
|
||||
entries: Vec<ClipboardEntry>,
|
||||
}
|
||||
|
||||
/// Either a ClipboardString or a ClipboardImage
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub enum ClipboardEntry {
|
||||
/// A string entry
|
||||
String(ClipboardString),
|
||||
/// An image entry
|
||||
Image(Image),
|
||||
}
|
||||
|
||||
impl ClipboardItem {
|
||||
/// Create a new ClipboardItem::String with no associated metadata
|
||||
pub fn new_string(text: String) -> Self {
|
||||
Self {
|
||||
entries: vec![ClipboardEntry::String(ClipboardString::new(text))],
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ClipboardItem::String with the given text and associated metadata
|
||||
pub fn new_string_with_metadata(text: String, metadata: String) -> Self {
|
||||
Self {
|
||||
entries: vec![ClipboardEntry::String(ClipboardString {
|
||||
text,
|
||||
metadata: Some(metadata),
|
||||
})],
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ClipboardItem::String with the given text and associated metadata
|
||||
pub fn new_string_with_json_metadata<T: Serialize>(text: String, metadata: T) -> Self {
|
||||
Self {
|
||||
entries: vec![ClipboardEntry::String(
|
||||
ClipboardString::new(text).with_json_metadata(metadata),
|
||||
)],
|
||||
}
|
||||
}
|
||||
|
||||
/// Concatenates together all the ClipboardString entries in the item.
|
||||
/// Returns None if there were no ClipboardString entries.
|
||||
pub fn text(&self) -> Option<String> {
|
||||
let mut answer = String::new();
|
||||
let mut any_entries = false;
|
||||
|
||||
for entry in self.entries.iter() {
|
||||
if let ClipboardEntry::String(ClipboardString { text, metadata: _ }) = entry {
|
||||
answer.push_str(text);
|
||||
any_entries = true;
|
||||
}
|
||||
}
|
||||
|
||||
if any_entries {
|
||||
Some(answer)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// If this item is one ClipboardEntry::String, returns its metadata.
|
||||
#[cfg_attr(not(target_os = "windows"), allow(dead_code))]
|
||||
pub fn metadata(&self) -> Option<&String> {
|
||||
match self.entries().first() {
|
||||
Some(ClipboardEntry::String(clipboard_string)) if self.entries.len() == 1 => {
|
||||
clipboard_string.metadata.as_ref()
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the item's entries
|
||||
pub fn entries(&self) -> &[ClipboardEntry] {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
/// Get owned versions of the item's entries
|
||||
pub fn into_entries(self) -> impl Iterator<Item = ClipboardEntry> {
|
||||
self.entries.into_iter()
|
||||
}
|
||||
}
|
||||
|
||||
/// One of the editor's supported image formats (e.g. PNG, JPEG) - used when dealing with images in the clipboard
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, Hash)]
|
||||
pub enum ImageFormat {
|
||||
// Sorted from most to least likely to be pasted into an editor,
|
||||
// which matters when we iterate through them trying to see if
|
||||
// clipboard content matches them.
|
||||
/// .png
|
||||
Png,
|
||||
/// .jpeg or .jpg
|
||||
Jpeg,
|
||||
/// .webp
|
||||
Webp,
|
||||
/// .gif
|
||||
Gif,
|
||||
/// .svg
|
||||
Svg,
|
||||
/// .bmp
|
||||
Bmp,
|
||||
/// .tif or .tiff
|
||||
Tiff,
|
||||
}
|
||||
|
||||
/// An image, with a format and certain bytes
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Image {
|
||||
/// The image format the bytes represent (e.g. PNG)
|
||||
format: ImageFormat,
|
||||
/// The raw image bytes
|
||||
bytes: Vec<u8>,
|
||||
id: u64,
|
||||
}
|
||||
|
||||
impl Hash for Image {
|
||||
fn hash<H: Hasher>(&self, state: &mut H) {
|
||||
state.write_u64(self.id);
|
||||
}
|
||||
}
|
||||
|
||||
impl Image {
|
||||
/// Get this image's ID
|
||||
pub fn id(&self) -> u64 {
|
||||
self.id
|
||||
}
|
||||
|
||||
/// Use the GPUI `use_asset` API to make this image renderable
|
||||
pub fn use_render_image(self: Arc<Self>, cx: &mut WindowContext) -> Option<Arc<RenderImage>> {
|
||||
ImageSource::Image(self).use_data(cx)
|
||||
}
|
||||
|
||||
/// Convert the clipboard image to an `ImageData` object.
|
||||
pub fn to_image_data(&self, cx: &AppContext) -> Result<Arc<RenderImage>> {
|
||||
fn frames_for_image(
|
||||
bytes: &[u8],
|
||||
format: image::ImageFormat,
|
||||
) -> Result<SmallVec<[Frame; 1]>> {
|
||||
let mut data = image::load_from_memory_with_format(bytes, format)?.into_rgba8();
|
||||
|
||||
// Convert from RGBA to BGRA.
|
||||
for pixel in data.chunks_exact_mut(4) {
|
||||
pixel.swap(0, 2);
|
||||
}
|
||||
|
||||
Ok(SmallVec::from_elem(Frame::new(data), 1))
|
||||
}
|
||||
|
||||
let frames = match self.format {
|
||||
ImageFormat::Gif => {
|
||||
let decoder = GifDecoder::new(Cursor::new(&self.bytes))?;
|
||||
let mut frames = SmallVec::new();
|
||||
|
||||
for frame in decoder.into_frames() {
|
||||
let mut frame = frame?;
|
||||
// Convert from RGBA to BGRA.
|
||||
for pixel in frame.buffer_mut().chunks_exact_mut(4) {
|
||||
pixel.swap(0, 2);
|
||||
}
|
||||
frames.push(frame);
|
||||
}
|
||||
|
||||
frames
|
||||
}
|
||||
ImageFormat::Png => frames_for_image(&self.bytes, image::ImageFormat::Png)?,
|
||||
ImageFormat::Jpeg => frames_for_image(&self.bytes, image::ImageFormat::Jpeg)?,
|
||||
ImageFormat::Webp => frames_for_image(&self.bytes, image::ImageFormat::WebP)?,
|
||||
ImageFormat::Bmp => frames_for_image(&self.bytes, image::ImageFormat::Bmp)?,
|
||||
ImageFormat::Tiff => frames_for_image(&self.bytes, image::ImageFormat::Tiff)?,
|
||||
ImageFormat::Svg => {
|
||||
// TODO: Fix this
|
||||
let pixmap = cx
|
||||
.svg_renderer()
|
||||
.render_pixmap(&self.bytes, SvgSize::ScaleFactor(1.0))?;
|
||||
|
||||
let buffer =
|
||||
image::ImageBuffer::from_raw(pixmap.width(), pixmap.height(), pixmap.take())
|
||||
.unwrap();
|
||||
|
||||
SmallVec::from_elem(Frame::new(buffer), 1)
|
||||
}
|
||||
};
|
||||
|
||||
Ok(Arc::new(RenderImage::new(frames)))
|
||||
}
|
||||
|
||||
/// Get the format of the clipboard image
|
||||
pub fn format(&self) -> ImageFormat {
|
||||
self.format
|
||||
}
|
||||
|
||||
/// Get the raw bytes of the clipboard image
|
||||
pub fn bytes(&self) -> &[u8] {
|
||||
self.bytes.as_slice()
|
||||
}
|
||||
}
|
||||
|
||||
/// A clipboard item that should be copied to the clipboard
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct ClipboardString {
|
||||
pub(crate) text: String,
|
||||
pub(crate) metadata: Option<String>,
|
||||
}
|
||||
|
||||
impl ClipboardItem {
|
||||
/// Create a new clipboard item with the given text
|
||||
impl ClipboardString {
|
||||
/// Create a new clipboard string with the given text
|
||||
pub fn new(text: String) -> Self {
|
||||
Self {
|
||||
text,
|
||||
|
|
@ -982,19 +1185,25 @@ impl ClipboardItem {
|
|||
}
|
||||
}
|
||||
|
||||
/// Create a new clipboard item with the given text and metadata
|
||||
pub fn with_metadata<T: Serialize>(mut self, metadata: T) -> Self {
|
||||
/// Return a new clipboard item with the metadata replaced by the given metadata,
|
||||
/// after serializing it as JSON.
|
||||
pub fn with_json_metadata<T: Serialize>(mut self, metadata: T) -> Self {
|
||||
self.metadata = Some(serde_json::to_string(&metadata).unwrap());
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the text of the clipboard item
|
||||
/// Get the text of the clipboard string
|
||||
pub fn text(&self) -> &String {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Get the metadata of the clipboard item
|
||||
pub fn metadata<T>(&self) -> Option<T>
|
||||
/// Get the owned text of the clipboard string
|
||||
pub fn into_text(self) -> String {
|
||||
self.text
|
||||
}
|
||||
|
||||
/// Get the metadata of the clipboard string, formatted as JSON
|
||||
pub fn metadata_json<T>(&self) -> Option<T>
|
||||
where
|
||||
T: for<'a> Deserialize<'a>,
|
||||
{
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue