Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add image input support for OpenAI models (#30639)

Browse source 
Release Notes:

- Added input image support for OpenAI models
This commit is contained in:
Agus Zubiaga 2025-05-13 17:32:42 +02:00 • committed by GitHub
parent 68afe4fdda
commit dd6594621f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 162 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

2
crates/language_models/src/provider/anthropic.rs
View file

@ -589,7 +589,7 @@ pub fn into_anthropic(
is_error: tool_result.is_error,
content: match tool_result.content {
LanguageModelToolResultContent::Text(text) => {
ToolResultContent::JustText(text.to_string())
ToolResultContent::Plain(text.to_string())
}
LanguageModelToolResultContent::Image(image) => {
ToolResultContent::Multipart(vec![ToolResultPart::Image {

79
crates/language_models/src/provider/open_ai.rs
View file

@ -15,7 +15,7 @@ use language_model::{
LanguageModelToolChoice, LanguageModelToolResultContent, LanguageModelToolUse, MessageContent,
RateLimiter, Role, StopReason,
};
use open_ai::{Model, ResponseStreamEvent, stream_completion};
use open_ai::{ImageUrl, Model, ResponseStreamEvent, stream_completion};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use settings::{Settings, SettingsStore};
@ -362,17 +362,26 @@ pub fn into_open_ai(
for message in request.messages {
for content in message.content {
match content {
MessageContent::Text(text) | MessageContent::Thinking { text, .. } => messages
.push(match message.role {
Role::User => open_ai::RequestMessage::User { content: text },
Role::Assistant => open_ai::RequestMessage::Assistant {
content: Some(text),
tool_calls: Vec::new(),
},
Role::System => open_ai::RequestMessage::System { content: text },
}),
MessageContent::Text(text) | MessageContent::Thinking { text, .. } => {
add_message_content_part(
open_ai::MessagePart::Text { text: text },
message.role,
&mut messages,
)
}
MessageContent::RedactedThinking(_) => {}
MessageContent::Image(_) => {}
MessageContent::Image(image) => {
add_message_content_part(
open_ai::MessagePart::Image {
image_url: ImageUrl {
url: image.to_base64_url(),
detail: None,
},
},
message.role,
&mut messages,
);
}
MessageContent::ToolUse(tool_use) => {
let tool_call = open_ai::ToolCall {
id: tool_use.id.to_string(),
@ -391,22 +400,30 @@ pub fn into_open_ai(
tool_calls.push(tool_call);
} else {
messages.push(open_ai::RequestMessage::Assistant {
content: None,
content: open_ai::MessageContent::empty(),
tool_calls: vec![tool_call],
});
}
}
MessageContent::ToolResult(tool_result) => {
let content = match &tool_result.content {
LanguageModelToolResultContent::Text(text) => text.to_string(),
LanguageModelToolResultContent::Image(_) => {
// TODO: Open AI image support
"[Tool responded with an image, but Zed doesn't support these in Open AI models yet]".to_string()
LanguageModelToolResultContent::Text(text) => {
vec![open_ai::MessagePart::Text {
text: text.to_string(),
}]
}
LanguageModelToolResultContent::Image(image) => {
vec![open_ai::MessagePart::Image {
image_url: ImageUrl {
url: image.to_base64_url(),
detail: None,
},
}]
}
};
messages.push(open_ai::RequestMessage::Tool {
content,
content: content.into(),
tool_call_id: tool_result.tool_use_id.to_string(),
});
}
@ -446,6 +463,34 @@ pub fn into_open_ai(
}
}
fn add_message_content_part(
new_part: open_ai::MessagePart,
role: Role,
messages: &mut Vec<open_ai::RequestMessage>,
) {
match (role, messages.last_mut()) {
(Role::User, Some(open_ai::RequestMessage::User { content }))
| (Role::Assistant, Some(open_ai::RequestMessage::Assistant { content, .. }))
| (Role::System, Some(open_ai::RequestMessage::System { content, .. })) => {
content.push_part(new_part);
}
_ => {
messages.push(match role {
Role::User => open_ai::RequestMessage::User {
content: open_ai::MessageContent::empty(),
},
Role::Assistant => open_ai::RequestMessage::Assistant {
content: open_ai::MessageContent::empty(),
tool_calls: Vec::new(),
},
Role::System => open_ai::RequestMessage::System {
content: open_ai::MessageContent::empty(),
},
});
}
}
}
pub struct OpenAiEventMapper {
tool_calls_by_index: HashMap<usize, RawToolCall>,
}