Have read_file support images (#30435)

This is very basic support for them. There are a number of other TODOs
before this is really a first-class supported feature, so not adding any
release notes for it; for now, this PR just makes it so that if
read_file tries to read a PNG (which has come up in practice), it at
least correctly sends it to Anthropic instead of messing up.

This also lays the groundwork for future PRs for more first-class
support for images in tool calls across more image file formats and LLM
providers.

Release Notes:

- N/A

---------

Co-authored-by: Agus Zubiaga <hi@aguz.me>
Co-authored-by: Agus Zubiaga <agus@zed.dev>
This commit is contained in:
Richard Feldman 2025-05-13 10:58:00 +02:00 committed by GitHub
parent f01af006e1
commit 8fdf309a4a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
30 changed files with 557 additions and 194 deletions

View file

@ -113,7 +113,7 @@ pub enum ModelVendor {
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
#[serde(tag = "type")]
pub enum ChatMessageContent {
pub enum ChatMessagePart {
#[serde(rename = "text")]
Text { text: String },
#[serde(rename = "image_url")]
@ -194,26 +194,55 @@ pub enum ToolChoice {
None,
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
#[derive(Serialize, Deserialize, Debug)]
#[serde(tag = "role", rename_all = "lowercase")]
pub enum ChatMessage {
Assistant {
content: Option<String>,
content: ChatMessageContent,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
tool_calls: Vec<ToolCall>,
},
User {
content: Vec<ChatMessageContent>,
content: ChatMessageContent,
},
System {
content: String,
},
Tool {
content: String,
content: ChatMessageContent,
tool_call_id: String,
},
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ChatMessageContent {
OnlyText(String),
Multipart(Vec<ChatMessagePart>),
}
impl ChatMessageContent {
pub fn empty() -> Self {
ChatMessageContent::Multipart(vec![])
}
}
impl From<Vec<ChatMessagePart>> for ChatMessageContent {
fn from(mut parts: Vec<ChatMessagePart>) -> Self {
if let [ChatMessagePart::Text { text }] = parts.as_mut_slice() {
ChatMessageContent::OnlyText(std::mem::take(text))
} else {
ChatMessageContent::Multipart(parts)
}
}
}
impl From<String> for ChatMessageContent {
fn from(text: String) -> Self {
ChatMessageContent::OnlyText(text)
}
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
pub struct ToolCall {
pub id: String,