Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add tool result image support to Gemini models (#30647)

Browse source 
Release Notes:

- Add tool result image support to Gemini models
This commit is contained in:
Agus Zubiaga 2025-05-14 02:51:31 +02:00 • committed by GitHub
parent 2f26a860a9
commit a4766e296f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 44 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

63
crates/language_models/src/provider/google.rs
View file

@ -398,43 +398,68 @@ pub fn into_google(
fn map_content(content: Vec<MessageContent>) -> Vec<Part> { fn map_content(content: Vec<MessageContent>) -> Vec<Part> {
content content
.into_iter() .into_iter()
.filter_map(|content| match content { .flat_map(|content| match content {
language_model::MessageContent::Text(text) language_model::MessageContent::Text(text)
| language_model::MessageContent::Thinking { text, .. } => { | language_model::MessageContent::Thinking { text, .. } => {
if !text.is_empty() { if !text.is_empty() {
Some(Part::TextPart(google_ai::TextPart { text })) vec![Part::TextPart(google_ai::TextPart { text })]
} else { } else {
None vec![]
} }
} }
language_model::MessageContent::RedactedThinking(_) => None, language_model::MessageContent::RedactedThinking(_) => vec![],
language_model::MessageContent::Image(image) => { language_model::MessageContent::Image(image) => {
Some(Part::InlineDataPart(google_ai::InlineDataPart { vec![Part::InlineDataPart(google_ai::InlineDataPart {
inline_data: google_ai::GenerativeContentBlob { inline_data: google_ai::GenerativeContentBlob {
mime_type: "image/png".to_string(), mime_type: "image/png".to_string(),
data: image.source.to_string(), data: image.source.to_string(),
}, },
})) })]
} }
language_model::MessageContent::ToolUse(tool_use) => { language_model::MessageContent::ToolUse(tool_use) => {
Some(Part::FunctionCallPart(google_ai::FunctionCallPart { vec![Part::FunctionCallPart(google_ai::FunctionCallPart {
function_call: google_ai::FunctionCall { function_call: google_ai::FunctionCall {
name: tool_use.name.to_string(), name: tool_use.name.to_string(),
args: tool_use.input, args: tool_use.input,
}, },
})) })]
}
language_model::MessageContent::ToolResult(tool_result) => {
match tool_result.content {
language_model::LanguageModelToolResultContent::Text(txt) => {
vec![Part::FunctionResponsePart(
google_ai::FunctionResponsePart {
function_response: google_ai::FunctionResponse {
name: tool_result.tool_name.to_string(),
// The API expects a valid JSON object
response: serde_json::json!({
"output": txt
}),
},
},
)]
}
language_model::LanguageModelToolResultContent::Image(image) => {
vec![
Part::FunctionResponsePart(google_ai::FunctionResponsePart {
function_response: google_ai::FunctionResponse {
name: tool_result.tool_name.to_string(),
// The API expects a valid JSON object
response: serde_json::json!({
"output": "Tool responded with an image"
}),
},
}),
Part::InlineDataPart(google_ai::InlineDataPart {
inline_data: google_ai::GenerativeContentBlob {
mime_type: "image/png".to_string(),
data: image.source.to_string(),
},
}),
]
}
}
} }
language_model::MessageContent::ToolResult(tool_result) => Some(
Part::FunctionResponsePart(google_ai::FunctionResponsePart {
function_response: google_ai::FunctionResponse {
name: tool_result.tool_name.to_string(),
// The API expects a valid JSON object
response: serde_json::json!({
"output": tool_result.content
}),
},
}),
),
}) })
.collect() .collect()
} }