agent: Support pasting images as context (#29177)

https://github.com/user-attachments/assets/d6a27b05-3590-4f40-a820-f6f99f6bd581

Release Notes:

- agent: Added support for pasting images as context

---------

Co-authored-by: Danilo Leal <daniloleal09@gmail.com>
This commit is contained in:
Bennet Bo Fenner 2025-04-22 11:01:01 +02:00 committed by GitHub
parent 3357736aea
commit eca6d5a04e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 407 additions and 99 deletions

View file

@ -6,7 +6,7 @@ use crate::context::{AssistantContext, format_context_as_string};
use crate::tool_compatibility::{IncompatibleToolsState, IncompatibleToolsTooltip};
use buffer_diff::BufferDiff;
use collections::HashSet;
use editor::actions::MoveUp;
use editor::actions::{MoveUp, Paste};
use editor::{
ContextMenuOptions, ContextMenuPlacement, Editor, EditorElement, EditorEvent, EditorMode,
EditorStyle, MultiBuffer,
@ -14,8 +14,8 @@ use editor::{
use file_icons::FileIcons;
use fs::Fs;
use gpui::{
Animation, AnimationExt, App, Entity, EventEmitter, Focusable, Subscription, Task, TextStyle,
WeakEntity, linear_color_stop, linear_gradient, point, pulsating_between,
Animation, AnimationExt, App, ClipboardEntry, Entity, EventEmitter, Focusable, Subscription,
Task, TextStyle, WeakEntity, linear_color_stop, linear_gradient, point, pulsating_between,
};
use language::{Buffer, Language};
use language_model::{ConfiguredModel, LanguageModelRegistry, LanguageModelRequestMessage};
@ -271,6 +271,7 @@ impl MessageEditor {
let refresh_task =
refresh_context_store_text(self.context_store.clone(), &HashSet::default(), cx);
let wait_for_images = self.context_store.read(cx).wait_for_images(cx);
let thread = self.thread.clone();
let context_store = self.context_store.clone();
@ -280,6 +281,7 @@ impl MessageEditor {
cx.spawn(async move |this, cx| {
let checkpoint = checkpoint.await.ok();
refresh_task.await;
wait_for_images.await;
thread
.update(cx, |thread, cx| {
@ -293,7 +295,12 @@ impl MessageEditor {
let excerpt_ids = context_store
.context()
.iter()
.filter(|ctx| matches!(ctx, AssistantContext::Excerpt(_)))
.filter(|ctx| {
matches!(
ctx,
AssistantContext::Excerpt(_) | AssistantContext::Image(_)
)
})
.map(|ctx| ctx.id())
.collect::<Vec<_>>();
@ -370,6 +377,34 @@ impl MessageEditor {
}
}
fn paste(&mut self, _: &Paste, _: &mut Window, cx: &mut Context<Self>) {
let images = cx
.read_from_clipboard()
.map(|item| {
item.into_entries()
.filter_map(|entry| {
if let ClipboardEntry::Image(image) = entry {
Some(image)
} else {
None
}
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
if images.is_empty() {
return;
}
cx.stop_propagation();
self.context_store.update(cx, |store, cx| {
for image in images {
store.add_image(Arc::new(image), cx);
}
});
}
fn handle_review_click(&self, window: &mut Window, cx: &mut Context<Self>) {
AgentDiff::deploy(self.thread.clone(), self.workspace.clone(), window, cx).log_err();
}
@ -445,6 +480,7 @@ impl MessageEditor {
.on_action(cx.listener(Self::move_up))
.on_action(cx.listener(Self::toggle_chat_mode))
.on_action(cx.listener(Self::expand_message_editor))
.capture_action(cx.listener(Self::paste))
.gap_2()
.p_2()
.bg(editor_bg_color)