Support images in agent2 threads (#36152)
- Support adding ImageContent to messages through copy/paste and through path completions - Ensure images are fully converted to LanguageModelImageContent before sending them to the model - Update ACP crate to v0.0.24 to enable passing image paths through the protocol Release Notes: - N/A --------- Co-authored-by: Conrad Irwin <conrad.irwin@gmail.com>
This commit is contained in:
Cole Miller
GitHub
parent
e2ce787c05
commit
b1e806442a
7 changed files with 415 additions and 92 deletions
4
Cargo.lock
generated
4
Cargo.lock
generated
|
|
@ -172,9 +172,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "agent-client-protocol"
|
||||
version = "0.0.23"
|
||||
version = "0.0.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fad72b7b8ee4331b3a4c8d43c107e982a4725564b4ee658ae5c4e79d2b486e8"
|
||||
checksum = "8fd68bbbef8e424fb8a605c5f0b00c360f682c4528b0a5feb5ec928aaf5ce28e"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures 0.3.31",
|
||||
|
|
|
|||
|
|
@ -425,7 +425,7 @@ zlog_settings = { path = "crates/zlog_settings" }
|
|||
#
|
||||
|
||||
agentic-coding-protocol = "0.0.10"
|
||||
agent-client-protocol = "0.0.23"
|
||||
agent-client-protocol = "0.0.24"
|
||||
aho-corasick = "1.1"
|
||||
alacritty_terminal = { git = "https://github.com/zed-industries/alacritty.git", branch = "add-hush-login-flag" }
|
||||
any_vec = "0.14"
|
||||
|
|
|
|||
|
|
@ -443,9 +443,8 @@ impl ContentBlock {
|
|||
}),
|
||||
..
|
||||
}) => Self::resource_link_md(&uri),
|
||||
acp::ContentBlock::Image(_)
|
||||
| acp::ContentBlock::Audio(_)
|
||||
| acp::ContentBlock::Resource(_) => String::new(),
|
||||
acp::ContentBlock::Image(image) => Self::image_md(&image),
|
||||
acp::ContentBlock::Audio(_) | acp::ContentBlock::Resource(_) => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -457,6 +456,10 @@ impl ContentBlock {
|
|||
}
|
||||
}
|
||||
|
||||
fn image_md(_image: &acp::ImageContent) -> String {
|
||||
"`Image`".into()
|
||||
}
|
||||
|
||||
fn to_markdown<'a>(&'a self, cx: &'a App) -> &'a str {
|
||||
match self {
|
||||
ContentBlock::Empty => "",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use std::{
|
|||
fmt,
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
str::FromStr,
|
||||
};
|
||||
use ui::{App, IconName, SharedString};
|
||||
use url::Url;
|
||||
|
|
@ -224,6 +225,14 @@ impl MentionUri {
|
|||
}
|
||||
}
|
||||
|
||||
impl FromStr for MentionUri {
|
||||
type Err = anyhow::Error;
|
||||
|
||||
fn from_str(s: &str) -> anyhow::Result<Self> {
|
||||
Self::parse(s)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MentionLink<'a>(&'a MentionUri);
|
||||
|
||||
impl fmt::Display for MentionLink<'_> {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use std::ffi::OsStr;
|
||||
use std::ops::Range;
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
|
|
@ -8,13 +9,14 @@ use anyhow::{Context as _, Result, anyhow};
|
|||
use collections::{HashMap, HashSet};
|
||||
use editor::display_map::CreaseId;
|
||||
use editor::{CompletionProvider, Editor, ExcerptId, ToOffset as _};
|
||||
|
||||
use futures::future::try_join_all;
|
||||
use futures::future::{Shared, try_join_all};
|
||||
use futures::{FutureExt, TryFutureExt};
|
||||
use fuzzy::{StringMatch, StringMatchCandidate};
|
||||
use gpui::{App, Entity, Task, WeakEntity};
|
||||
use gpui::{App, Entity, ImageFormat, Img, Task, WeakEntity};
|
||||
use http_client::HttpClientWithUrl;
|
||||
use itertools::Itertools as _;
|
||||
use language::{Buffer, CodeLabel, HighlightId};
|
||||
use language_model::LanguageModelImage;
|
||||
use lsp::CompletionContext;
|
||||
use parking_lot::Mutex;
|
||||
use project::{
|
||||
|
|
@ -43,24 +45,43 @@ use crate::context_picker::{
|
|||
available_context_picker_entries, recent_context_picker_entries, selection_ranges,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Eq, PartialEq)]
|
||||
pub struct MentionImage {
|
||||
pub abs_path: Option<Arc<Path>>,
|
||||
pub data: SharedString,
|
||||
pub format: ImageFormat,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MentionSet {
|
||||
uri_by_crease_id: HashMap<CreaseId, MentionUri>,
|
||||
fetch_results: HashMap<Url, String>,
|
||||
fetch_results: HashMap<Url, Shared<Task<Result<String, String>>>>,
|
||||
images: HashMap<CreaseId, Shared<Task<Result<MentionImage, String>>>>,
|
||||
}
|
||||
|
||||
impl MentionSet {
|
||||
pub fn insert(&mut self, crease_id: CreaseId, uri: MentionUri) {
|
||||
pub fn insert_uri(&mut self, crease_id: CreaseId, uri: MentionUri) {
|
||||
self.uri_by_crease_id.insert(crease_id, uri);
|
||||
}
|
||||
|
||||
pub fn add_fetch_result(&mut self, url: Url, content: String) {
|
||||
pub fn add_fetch_result(&mut self, url: Url, content: Shared<Task<Result<String, String>>>) {
|
||||
self.fetch_results.insert(url, content);
|
||||
}
|
||||
|
||||
pub fn insert_image(
|
||||
&mut self,
|
||||
crease_id: CreaseId,
|
||||
task: Shared<Task<Result<MentionImage, String>>>,
|
||||
) {
|
||||
self.images.insert(crease_id, task);
|
||||
}
|
||||
|
||||
pub fn drain(&mut self) -> impl Iterator<Item = CreaseId> {
|
||||
self.fetch_results.clear();
|
||||
self.uri_by_crease_id.drain().map(|(id, _)| id)
|
||||
self.uri_by_crease_id
|
||||
.drain()
|
||||
.map(|(id, _)| id)
|
||||
.chain(self.images.drain().map(|(id, _)| id))
|
||||
}
|
||||
|
||||
pub fn clear(&mut self) {
|
||||
|
|
@ -76,7 +97,7 @@ impl MentionSet {
|
|||
window: &mut Window,
|
||||
cx: &mut App,
|
||||
) -> Task<Result<HashMap<CreaseId, Mention>>> {
|
||||
let contents = self
|
||||
let mut contents = self
|
||||
.uri_by_crease_id
|
||||
.iter()
|
||||
.map(|(&crease_id, uri)| {
|
||||
|
|
@ -85,19 +106,59 @@ impl MentionSet {
|
|||
// TODO directories
|
||||
let uri = uri.clone();
|
||||
let abs_path = abs_path.to_path_buf();
|
||||
let buffer_task = project.update(cx, |project, cx| {
|
||||
let path = project
|
||||
.find_project_path(abs_path, cx)
|
||||
.context("Failed to find project path")?;
|
||||
anyhow::Ok(project.open_buffer(path, cx))
|
||||
});
|
||||
let extension = abs_path.extension().and_then(OsStr::to_str).unwrap_or("");
|
||||
|
||||
cx.spawn(async move |cx| {
|
||||
let buffer = buffer_task?.await?;
|
||||
let content = buffer.read_with(cx, |buffer, _cx| buffer.text())?;
|
||||
if Img::extensions().contains(&extension) && !extension.contains("svg") {
|
||||
let open_image_task = project.update(cx, |project, cx| {
|
||||
let path = project
|
||||
.find_project_path(&abs_path, cx)
|
||||
.context("Failed to find project path")?;
|
||||
anyhow::Ok(project.open_image(path, cx))
|
||||
});
|
||||
|
||||
anyhow::Ok((crease_id, Mention { uri, content }))
|
||||
})
|
||||
cx.spawn(async move |cx| {
|
||||
let image_item = open_image_task?.await?;
|
||||
let (data, format) = image_item.update(cx, |image_item, cx| {
|
||||
let format = image_item.image.format;
|
||||
(
|
||||
LanguageModelImage::from_image(
|
||||
image_item.image.clone(),
|
||||
cx,
|
||||
),
|
||||
format,
|
||||
)
|
||||
})?;
|
||||
let data = cx.spawn(async move |_| {
|
||||
if let Some(data) = data.await {
|
||||
Ok(data.source)
|
||||
} else {
|
||||
anyhow::bail!("Failed to convert image")
|
||||
}
|
||||
});
|
||||
|
||||
anyhow::Ok((
|
||||
crease_id,
|
||||
Mention::Image(MentionImage {
|
||||
abs_path: Some(abs_path.as_path().into()),
|
||||
data: data.await?,
|
||||
format,
|
||||
}),
|
||||
))
|
||||
})
|
||||
} else {
|
||||
let buffer_task = project.update(cx, |project, cx| {
|
||||
let path = project
|
||||
.find_project_path(abs_path, cx)
|
||||
.context("Failed to find project path")?;
|
||||
anyhow::Ok(project.open_buffer(path, cx))
|
||||
});
|
||||
cx.spawn(async move |cx| {
|
||||
let buffer = buffer_task?.await?;
|
||||
let content = buffer.read_with(cx, |buffer, _cx| buffer.text())?;
|
||||
|
||||
anyhow::Ok((crease_id, Mention::Text { uri, content }))
|
||||
})
|
||||
}
|
||||
}
|
||||
MentionUri::Symbol {
|
||||
path, line_range, ..
|
||||
|
|
@ -130,7 +191,7 @@ impl MentionSet {
|
|||
.collect()
|
||||
})?;
|
||||
|
||||
anyhow::Ok((crease_id, Mention { uri, content }))
|
||||
anyhow::Ok((crease_id, Mention::Text { uri, content }))
|
||||
})
|
||||
}
|
||||
MentionUri::Thread { id: thread_id, .. } => {
|
||||
|
|
@ -145,7 +206,7 @@ impl MentionSet {
|
|||
thread.latest_detailed_summary_or_text().to_string()
|
||||
})?;
|
||||
|
||||
anyhow::Ok((crease_id, Mention { uri, content }))
|
||||
anyhow::Ok((crease_id, Mention::Text { uri, content }))
|
||||
})
|
||||
}
|
||||
MentionUri::TextThread { path, .. } => {
|
||||
|
|
@ -156,7 +217,7 @@ impl MentionSet {
|
|||
cx.spawn(async move |cx| {
|
||||
let context = context.await?;
|
||||
let xml = context.update(cx, |context, cx| context.to_xml(cx))?;
|
||||
anyhow::Ok((crease_id, Mention { uri, content: xml }))
|
||||
anyhow::Ok((crease_id, Mention::Text { uri, content: xml }))
|
||||
})
|
||||
}
|
||||
MentionUri::Rule { id: prompt_id, .. } => {
|
||||
|
|
@ -169,25 +230,39 @@ impl MentionSet {
|
|||
cx.spawn(async move |_| {
|
||||
// TODO: report load errors instead of just logging
|
||||
let text = text_task.await?;
|
||||
anyhow::Ok((crease_id, Mention { uri, content: text }))
|
||||
anyhow::Ok((crease_id, Mention::Text { uri, content: text }))
|
||||
})
|
||||
}
|
||||
MentionUri::Fetch { url } => {
|
||||
let Some(content) = self.fetch_results.get(&url) else {
|
||||
let Some(content) = self.fetch_results.get(&url).cloned() else {
|
||||
return Task::ready(Err(anyhow!("missing fetch result")));
|
||||
};
|
||||
Task::ready(Ok((
|
||||
crease_id,
|
||||
Mention {
|
||||
uri: uri.clone(),
|
||||
content: content.clone(),
|
||||
},
|
||||
)))
|
||||
let uri = uri.clone();
|
||||
cx.spawn(async move |_| {
|
||||
Ok((
|
||||
crease_id,
|
||||
Mention::Text {
|
||||
uri,
|
||||
content: content.await.map_err(|e| anyhow::anyhow!("{e}"))?,
|
||||
},
|
||||
))
|
||||
})
|
||||
}
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
contents.extend(self.images.iter().map(|(crease_id, image)| {
|
||||
let crease_id = *crease_id;
|
||||
let image = image.clone();
|
||||
cx.spawn(async move |_| {
|
||||
Ok((
|
||||
crease_id,
|
||||
Mention::Image(image.await.map_err(|e| anyhow::anyhow!("{e}"))?),
|
||||
))
|
||||
})
|
||||
}));
|
||||
|
||||
cx.spawn(async move |_cx| {
|
||||
let contents = try_join_all(contents).await?.into_iter().collect();
|
||||
anyhow::Ok(contents)
|
||||
|
|
@ -195,10 +270,10 @@ impl MentionSet {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Mention {
|
||||
pub uri: MentionUri,
|
||||
pub content: String,
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
pub enum Mention {
|
||||
Text { uri: MentionUri, content: String },
|
||||
Image(MentionImage),
|
||||
}
|
||||
|
||||
pub(crate) enum Match {
|
||||
|
|
@ -536,7 +611,10 @@ impl ContextPickerCompletionProvider {
|
|||
crease_ids.try_into().unwrap()
|
||||
});
|
||||
|
||||
mention_set.lock().insert(crease_id, uri);
|
||||
mention_set.lock().insert_uri(
|
||||
crease_id,
|
||||
MentionUri::Selection { path, line_range },
|
||||
);
|
||||
|
||||
current_offset += text_len + 1;
|
||||
}
|
||||
|
|
@ -786,6 +864,7 @@ impl ContextPickerCompletionProvider {
|
|||
let url_to_fetch = url_to_fetch.clone();
|
||||
let source_range = source_range.clone();
|
||||
let icon_path = icon_path.clone();
|
||||
let mention_uri = mention_uri.clone();
|
||||
Arc::new(move |_, window, cx| {
|
||||
let Some(url) = url::Url::parse(url_to_fetch.as_ref())
|
||||
.or_else(|_| url::Url::parse(&format!("https://{url_to_fetch}")))
|
||||
|
|
@ -799,6 +878,7 @@ impl ContextPickerCompletionProvider {
|
|||
let http_client = http_client.clone();
|
||||
let source_range = source_range.clone();
|
||||
let icon_path = icon_path.clone();
|
||||
let mention_uri = mention_uri.clone();
|
||||
window.defer(cx, move |window, cx| {
|
||||
let url = url.clone();
|
||||
|
||||
|
|
@ -819,17 +899,24 @@ impl ContextPickerCompletionProvider {
|
|||
let mention_set = mention_set.clone();
|
||||
let http_client = http_client.clone();
|
||||
let source_range = source_range.clone();
|
||||
|
||||
let url_string = url.to_string();
|
||||
let fetch = cx
|
||||
.background_executor()
|
||||
.spawn(async move {
|
||||
fetch_url_content(http_client, url_string)
|
||||
.map_err(|e| e.to_string())
|
||||
.await
|
||||
})
|
||||
.shared();
|
||||
mention_set.lock().add_fetch_result(url, fetch.clone());
|
||||
|
||||
window
|
||||
.spawn(cx, async move |cx| {
|
||||
if let Some(content) =
|
||||
fetch_url_content(http_client, url.to_string())
|
||||
.await
|
||||
.notify_async_err(cx)
|
||||
{
|
||||
mention_set.lock().add_fetch_result(url.clone(), content);
|
||||
if fetch.await.notify_async_err(cx).is_some() {
|
||||
mention_set
|
||||
.lock()
|
||||
.insert(crease_id, MentionUri::Fetch { url });
|
||||
.insert_uri(crease_id, mention_uri.clone());
|
||||
} else {
|
||||
// Remove crease if we failed to fetch
|
||||
editor
|
||||
|
|
@ -1121,7 +1208,9 @@ fn confirm_completion_callback(
|
|||
window,
|
||||
cx,
|
||||
) {
|
||||
mention_set.lock().insert(crease_id, mention_uri.clone());
|
||||
mention_set
|
||||
.lock()
|
||||
.insert_uri(crease_id, mention_uri.clone());
|
||||
}
|
||||
});
|
||||
false
|
||||
|
|
@ -1499,11 +1588,12 @@ mod tests {
|
|||
.into_values()
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(contents.len(), 1);
|
||||
assert_eq!(contents[0].content, "1");
|
||||
assert_eq!(
|
||||
contents[0].uri.to_uri().to_string(),
|
||||
"file:///dir/a/one.txt"
|
||||
pretty_assertions::assert_eq!(
|
||||
contents,
|
||||
[Mention::Text {
|
||||
content: "1".into(),
|
||||
uri: "file:///dir/a/one.txt".parse().unwrap()
|
||||
}]
|
||||
);
|
||||
|
||||
cx.simulate_input(" ");
|
||||
|
|
@ -1567,11 +1657,13 @@ mod tests {
|
|||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(contents.len(), 2);
|
||||
let new_mention = contents
|
||||
.iter()
|
||||
.find(|mention| mention.uri.to_uri().to_string() == "file:///dir/b/eight.txt")
|
||||
.unwrap();
|
||||
assert_eq!(new_mention.content, "8");
|
||||
pretty_assertions::assert_eq!(
|
||||
contents[1],
|
||||
Mention::Text {
|
||||
content: "8".to_string(),
|
||||
uri: "file:///dir/b/eight.txt".parse().unwrap(),
|
||||
}
|
||||
);
|
||||
|
||||
editor.update(&mut cx, |editor, cx| {
|
||||
assert_eq!(
|
||||
|
|
@ -1689,13 +1781,15 @@ mod tests {
|
|||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(contents.len(), 3);
|
||||
let new_mention = contents
|
||||
.iter()
|
||||
.find(|mention| {
|
||||
mention.uri.to_uri().to_string() == "file:///dir/a/one.txt?symbol=MySymbol#L1:1"
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(new_mention.content, "1");
|
||||
pretty_assertions::assert_eq!(
|
||||
contents[2],
|
||||
Mention::Text {
|
||||
content: "1".into(),
|
||||
uri: "file:///dir/a/one.txt?symbol=MySymbol#L1:1"
|
||||
.parse()
|
||||
.unwrap(),
|
||||
}
|
||||
);
|
||||
|
||||
cx.run_until_parked();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use crate::acp::completion_provider::ContextPickerCompletionProvider;
|
||||
use crate::acp::completion_provider::MentionImage;
|
||||
use crate::acp::completion_provider::MentionSet;
|
||||
use acp_thread::MentionUri;
|
||||
use agent::TextThreadStore;
|
||||
|
|
@ -6,30 +7,44 @@ use agent::ThreadStore;
|
|||
use agent_client_protocol as acp;
|
||||
use anyhow::Result;
|
||||
use collections::HashSet;
|
||||
use editor::ExcerptId;
|
||||
use editor::actions::Paste;
|
||||
use editor::display_map::CreaseId;
|
||||
use editor::{
|
||||
AnchorRangeExt, ContextMenuOptions, ContextMenuPlacement, Editor, EditorElement, EditorMode,
|
||||
EditorStyle, MultiBuffer,
|
||||
};
|
||||
use futures::FutureExt as _;
|
||||
use gpui::ClipboardEntry;
|
||||
use gpui::Image;
|
||||
use gpui::ImageFormat;
|
||||
use gpui::{
|
||||
AppContext, Context, Entity, EventEmitter, FocusHandle, Focusable, Task, TextStyle, WeakEntity,
|
||||
};
|
||||
use language::Buffer;
|
||||
use language::Language;
|
||||
use language_model::LanguageModelImage;
|
||||
use parking_lot::Mutex;
|
||||
use project::{CompletionIntent, Project};
|
||||
use settings::Settings;
|
||||
use std::fmt::Write;
|
||||
use std::path::Path;
|
||||
use std::rc::Rc;
|
||||
use std::sync::Arc;
|
||||
use theme::ThemeSettings;
|
||||
use ui::IconName;
|
||||
use ui::SharedString;
|
||||
use ui::{
|
||||
ActiveTheme, App, InteractiveElement, IntoElement, ParentElement, Render, Styled, TextSize,
|
||||
Window, div,
|
||||
};
|
||||
use util::ResultExt;
|
||||
use workspace::Workspace;
|
||||
use workspace::notifications::NotifyResultExt as _;
|
||||
use zed_actions::agent::Chat;
|
||||
|
||||
use super::completion_provider::Mention;
|
||||
|
||||
pub struct MessageEditor {
|
||||
editor: Entity<Editor>,
|
||||
project: Entity<Project>,
|
||||
|
|
@ -130,23 +145,41 @@ impl MessageEditor {
|
|||
continue;
|
||||
}
|
||||
|
||||
if let Some(mention) = contents.get(&crease_id) {
|
||||
let crease_range = crease.range().to_offset(&snapshot.buffer_snapshot);
|
||||
if crease_range.start > ix {
|
||||
chunks.push(text[ix..crease_range.start].into());
|
||||
}
|
||||
chunks.push(acp::ContentBlock::Resource(acp::EmbeddedResource {
|
||||
annotations: None,
|
||||
resource: acp::EmbeddedResourceResource::TextResourceContents(
|
||||
acp::TextResourceContents {
|
||||
mime_type: None,
|
||||
text: mention.content.clone(),
|
||||
uri: mention.uri.to_uri().to_string(),
|
||||
},
|
||||
),
|
||||
}));
|
||||
ix = crease_range.end;
|
||||
let Some(mention) = contents.get(&crease_id) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let crease_range = crease.range().to_offset(&snapshot.buffer_snapshot);
|
||||
if crease_range.start > ix {
|
||||
chunks.push(text[ix..crease_range.start].into());
|
||||
}
|
||||
let chunk = match mention {
|
||||
Mention::Text { uri, content } => {
|
||||
acp::ContentBlock::Resource(acp::EmbeddedResource {
|
||||
annotations: None,
|
||||
resource: acp::EmbeddedResourceResource::TextResourceContents(
|
||||
acp::TextResourceContents {
|
||||
mime_type: None,
|
||||
text: content.clone(),
|
||||
uri: uri.to_uri().to_string(),
|
||||
},
|
||||
),
|
||||
})
|
||||
}
|
||||
Mention::Image(mention_image) => {
|
||||
acp::ContentBlock::Image(acp::ImageContent {
|
||||
annotations: None,
|
||||
data: mention_image.data.to_string(),
|
||||
mime_type: mention_image.format.mime_type().into(),
|
||||
uri: mention_image
|
||||
.abs_path
|
||||
.as_ref()
|
||||
.map(|path| format!("file://{}", path.display())),
|
||||
})
|
||||
}
|
||||
};
|
||||
chunks.push(chunk);
|
||||
ix = crease_range.end;
|
||||
}
|
||||
|
||||
if ix < text.len() {
|
||||
|
|
@ -177,6 +210,56 @@ impl MessageEditor {
|
|||
cx.emit(MessageEditorEvent::Cancel)
|
||||
}
|
||||
|
||||
fn paste(&mut self, _: &Paste, window: &mut Window, cx: &mut Context<Self>) {
|
||||
let images = cx
|
||||
.read_from_clipboard()
|
||||
.map(|item| {
|
||||
item.into_entries()
|
||||
.filter_map(|entry| {
|
||||
if let ClipboardEntry::Image(image) = entry {
|
||||
Some(image)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
|
||||
if images.is_empty() {
|
||||
return;
|
||||
}
|
||||
cx.stop_propagation();
|
||||
|
||||
let replacement_text = "image";
|
||||
for image in images {
|
||||
let (excerpt_id, anchor) = self.editor.update(cx, |message_editor, cx| {
|
||||
let snapshot = message_editor.snapshot(window, cx);
|
||||
let (excerpt_id, _, snapshot) = snapshot.buffer_snapshot.as_singleton().unwrap();
|
||||
|
||||
let anchor = snapshot.anchor_before(snapshot.len());
|
||||
message_editor.edit(
|
||||
[(
|
||||
multi_buffer::Anchor::max()..multi_buffer::Anchor::max(),
|
||||
format!("{replacement_text} "),
|
||||
)],
|
||||
cx,
|
||||
);
|
||||
(*excerpt_id, anchor)
|
||||
});
|
||||
|
||||
self.insert_image(
|
||||
excerpt_id,
|
||||
anchor,
|
||||
replacement_text.len(),
|
||||
Arc::new(image),
|
||||
None,
|
||||
window,
|
||||
cx,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn insert_dragged_files(
|
||||
&self,
|
||||
paths: Vec<project::ProjectPath>,
|
||||
|
|
@ -234,6 +317,68 @@ impl MessageEditor {
|
|||
}
|
||||
}
|
||||
|
||||
fn insert_image(
|
||||
&mut self,
|
||||
excerpt_id: ExcerptId,
|
||||
crease_start: text::Anchor,
|
||||
content_len: usize,
|
||||
image: Arc<Image>,
|
||||
abs_path: Option<Arc<Path>>,
|
||||
window: &mut Window,
|
||||
cx: &mut Context<Self>,
|
||||
) {
|
||||
let Some(crease_id) = insert_crease_for_image(
|
||||
excerpt_id,
|
||||
crease_start,
|
||||
content_len,
|
||||
self.editor.clone(),
|
||||
window,
|
||||
cx,
|
||||
) else {
|
||||
return;
|
||||
};
|
||||
self.editor.update(cx, |_editor, cx| {
|
||||
let format = image.format;
|
||||
let convert = LanguageModelImage::from_image(image, cx);
|
||||
|
||||
let task = cx
|
||||
.spawn_in(window, async move |editor, cx| {
|
||||
if let Some(image) = convert.await {
|
||||
Ok(MentionImage {
|
||||
abs_path,
|
||||
data: image.source,
|
||||
format,
|
||||
})
|
||||
} else {
|
||||
editor
|
||||
.update(cx, |editor, cx| {
|
||||
let snapshot = editor.buffer().read(cx).snapshot(cx);
|
||||
let Some(anchor) =
|
||||
snapshot.anchor_in_excerpt(excerpt_id, crease_start)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
editor.display_map.update(cx, |display_map, cx| {
|
||||
display_map.unfold_intersecting(vec![anchor..anchor], true, cx);
|
||||
});
|
||||
editor.remove_creases([crease_id], cx);
|
||||
})
|
||||
.ok();
|
||||
Err("Failed to convert image".to_string())
|
||||
}
|
||||
})
|
||||
.shared();
|
||||
|
||||
cx.spawn_in(window, {
|
||||
let task = task.clone();
|
||||
async move |_, cx| task.clone().await.notify_async_err(cx)
|
||||
})
|
||||
.detach();
|
||||
|
||||
self.mention_set.lock().insert_image(crease_id, task);
|
||||
});
|
||||
}
|
||||
|
||||
pub fn set_mode(&mut self, mode: EditorMode, cx: &mut Context<Self>) {
|
||||
self.editor.update(cx, |editor, cx| {
|
||||
editor.set_mode(mode);
|
||||
|
|
@ -243,12 +388,13 @@ impl MessageEditor {
|
|||
|
||||
pub fn set_message(
|
||||
&mut self,
|
||||
message: &[acp::ContentBlock],
|
||||
message: Vec<acp::ContentBlock>,
|
||||
window: &mut Window,
|
||||
cx: &mut Context<Self>,
|
||||
) {
|
||||
let mut text = String::new();
|
||||
let mut mentions = Vec::new();
|
||||
let mut images = Vec::new();
|
||||
|
||||
for chunk in message {
|
||||
match chunk {
|
||||
|
|
@ -266,8 +412,13 @@ impl MessageEditor {
|
|||
mentions.push((start..end, mention_uri));
|
||||
}
|
||||
}
|
||||
acp::ContentBlock::Image(_)
|
||||
| acp::ContentBlock::Audio(_)
|
||||
acp::ContentBlock::Image(content) => {
|
||||
let start = text.len();
|
||||
text.push_str("image");
|
||||
let end = text.len();
|
||||
images.push((start..end, content));
|
||||
}
|
||||
acp::ContentBlock::Audio(_)
|
||||
| acp::ContentBlock::Resource(_)
|
||||
| acp::ContentBlock::ResourceLink(_) => {}
|
||||
}
|
||||
|
|
@ -293,7 +444,50 @@ impl MessageEditor {
|
|||
);
|
||||
|
||||
if let Some(crease_id) = crease_id {
|
||||
self.mention_set.lock().insert(crease_id, mention_uri);
|
||||
self.mention_set.lock().insert_uri(crease_id, mention_uri);
|
||||
}
|
||||
}
|
||||
for (range, content) in images {
|
||||
let Some(format) = ImageFormat::from_mime_type(&content.mime_type) else {
|
||||
continue;
|
||||
};
|
||||
let anchor = snapshot.anchor_before(range.start);
|
||||
let abs_path = content
|
||||
.uri
|
||||
.as_ref()
|
||||
.and_then(|uri| uri.strip_prefix("file://").map(|s| Path::new(s).into()));
|
||||
|
||||
let name = content
|
||||
.uri
|
||||
.as_ref()
|
||||
.and_then(|uri| {
|
||||
uri.strip_prefix("file://")
|
||||
.and_then(|path| Path::new(path).file_name())
|
||||
})
|
||||
.map(|name| name.to_string_lossy().to_string())
|
||||
.unwrap_or("Image".to_owned());
|
||||
let crease_id = crate::context_picker::insert_crease_for_mention(
|
||||
anchor.excerpt_id,
|
||||
anchor.text_anchor,
|
||||
range.end - range.start,
|
||||
name.into(),
|
||||
IconName::Image.path().into(),
|
||||
self.editor.clone(),
|
||||
window,
|
||||
cx,
|
||||
);
|
||||
let data: SharedString = content.data.to_string().into();
|
||||
|
||||
if let Some(crease_id) = crease_id {
|
||||
self.mention_set.lock().insert_image(
|
||||
crease_id,
|
||||
Task::ready(Ok(MentionImage {
|
||||
abs_path,
|
||||
data,
|
||||
format,
|
||||
}))
|
||||
.shared(),
|
||||
);
|
||||
}
|
||||
}
|
||||
cx.notify();
|
||||
|
|
@ -319,6 +513,7 @@ impl Render for MessageEditor {
|
|||
.key_context("MessageEditor")
|
||||
.on_action(cx.listener(Self::chat))
|
||||
.on_action(cx.listener(Self::cancel))
|
||||
.capture_action(cx.listener(Self::paste))
|
||||
.flex_1()
|
||||
.child({
|
||||
let settings = ThemeSettings::get_global(cx);
|
||||
|
|
@ -351,6 +546,26 @@ impl Render for MessageEditor {
|
|||
}
|
||||
}
|
||||
|
||||
pub(crate) fn insert_crease_for_image(
|
||||
excerpt_id: ExcerptId,
|
||||
anchor: text::Anchor,
|
||||
content_len: usize,
|
||||
editor: Entity<Editor>,
|
||||
window: &mut Window,
|
||||
cx: &mut App,
|
||||
) -> Option<CreaseId> {
|
||||
crate::context_picker::insert_crease_for_mention(
|
||||
excerpt_id,
|
||||
anchor,
|
||||
content_len,
|
||||
"Image".into(),
|
||||
IconName::Image.path().into(),
|
||||
editor,
|
||||
window,
|
||||
cx,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::Path;
|
||||
|
|
|
|||
|
|
@ -5,9 +5,10 @@ use acp_thread::{
|
|||
use acp_thread::{AgentConnection, Plan};
|
||||
use action_log::ActionLog;
|
||||
use agent::{TextThreadStore, ThreadStore};
|
||||
use agent_client_protocol as acp;
|
||||
use agent_client_protocol::{self as acp};
|
||||
use agent_servers::AgentServer;
|
||||
use agent_settings::{AgentSettings, NotifyWhenAgentWaiting};
|
||||
use anyhow::bail;
|
||||
use audio::{Audio, Sound};
|
||||
use buffer_diff::BufferDiff;
|
||||
use collections::{HashMap, HashSet};
|
||||
|
|
@ -2360,7 +2361,7 @@ impl AcpThreadView {
|
|||
window,
|
||||
cx,
|
||||
);
|
||||
editor.set_message(&chunks, window, cx);
|
||||
editor.set_message(chunks, window, cx);
|
||||
editor
|
||||
});
|
||||
let subscription =
|
||||
|
|
@ -2725,7 +2726,7 @@ impl AcpThreadView {
|
|||
let project = workspace.project().clone();
|
||||
|
||||
if !project.read(cx).is_local() {
|
||||
anyhow::bail!("failed to open active thread as markdown in remote project");
|
||||
bail!("failed to open active thread as markdown in remote project");
|
||||
}
|
||||
|
||||
let buffer = project.update(cx, |project, cx| {
|
||||
|
|
@ -2990,12 +2991,13 @@ impl AcpThreadView {
|
|||
pub(crate) fn insert_dragged_files(
|
||||
&self,
|
||||
paths: Vec<project::ProjectPath>,
|
||||
_added_worktrees: Vec<Entity<project::Worktree>>,
|
||||
added_worktrees: Vec<Entity<project::Worktree>>,
|
||||
window: &mut Window,
|
||||
cx: &mut Context<Self>,
|
||||
) {
|
||||
self.message_editor.update(cx, |message_editor, cx| {
|
||||
message_editor.insert_dragged_files(paths, window, cx);
|
||||
drop(added_worktrees);
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue