Have read_file support images (#30435)
This is very basic support for them. There are a number of other TODOs before this is really a first-class supported feature, so not adding any release notes for it; for now, this PR just makes it so that if read_file tries to read a PNG (which has come up in practice), it at least correctly sends it to Anthropic instead of messing up. This also lays the groundwork for future PRs for more first-class support for images in tool calls across more image file formats and LLM providers. Release Notes: - N/A --------- Co-authored-by: Agus Zubiaga <hi@aguz.me> Co-authored-by: Agus Zubiaga <agus@zed.dev>
This commit is contained in:
parent
f01af006e1
commit
8fdf309a4a
30 changed files with 557 additions and 194 deletions
|
@ -113,7 +113,7 @@ pub enum ModelVendor {
|
|||
|
||||
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ChatMessageContent {
|
||||
pub enum ChatMessagePart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
|
@ -194,26 +194,55 @@ pub enum ToolChoice {
|
|||
None,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(tag = "role", rename_all = "lowercase")]
|
||||
pub enum ChatMessage {
|
||||
Assistant {
|
||||
content: Option<String>,
|
||||
content: ChatMessageContent,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
tool_calls: Vec<ToolCall>,
|
||||
},
|
||||
User {
|
||||
content: Vec<ChatMessageContent>,
|
||||
content: ChatMessageContent,
|
||||
},
|
||||
System {
|
||||
content: String,
|
||||
},
|
||||
Tool {
|
||||
content: String,
|
||||
content: ChatMessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatMessageContent {
|
||||
OnlyText(String),
|
||||
Multipart(Vec<ChatMessagePart>),
|
||||
}
|
||||
|
||||
impl ChatMessageContent {
|
||||
pub fn empty() -> Self {
|
||||
ChatMessageContent::Multipart(vec![])
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<ChatMessagePart>> for ChatMessageContent {
|
||||
fn from(mut parts: Vec<ChatMessagePart>) -> Self {
|
||||
if let [ChatMessagePart::Text { text }] = parts.as_mut_slice() {
|
||||
ChatMessageContent::OnlyText(std::mem::take(text))
|
||||
} else {
|
||||
ChatMessageContent::Multipart(parts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for ChatMessageContent {
|
||||
fn from(text: String) -> Self {
|
||||
ChatMessageContent::OnlyText(text)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue