Add tool calling support for Gemini models (#27772)

Release Notes:

- N/A
This commit is contained in:
Bennet Bo Fenner 2025-03-31 17:46:42 +02:00 committed by GitHub
parent f6d58f76e4
commit c8a9a74e6a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 735 additions and 251 deletions

View file

@ -18,6 +18,7 @@ mod path_search_tool;
mod read_file_tool;
mod regex_search_tool;
mod replace;
mod schema;
mod symbol_info_tool;
mod thinking_tool;

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Context as _, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -38,9 +39,8 @@ impl Tool for BashTool {
IconName::Terminal
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(BashToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<BashToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,8 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool, ToolWorkingSet};
use futures::future::join_all;
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -162,9 +163,8 @@ impl Tool for BatchTool {
IconName::Cog
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(BatchToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<BatchToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -7,7 +7,7 @@ use assistant_tool::{ActionLog, Tool};
use collections::IndexMap;
use gpui::{App, AsyncApp, Entity, Task};
use language::{CodeLabel, Language, LanguageRegistry};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use lsp::SymbolKind;
use project::{DocumentSymbol, Project, Symbol};
use regex::{Regex, RegexBuilder};
@ -17,6 +17,7 @@ use ui::IconName;
use util::markdown::MarkdownString;
use crate::code_symbol_iter::{CodeSymbolIterator, Entry};
use crate::schema::json_schema_for;
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct CodeSymbolsInput {
@ -93,9 +94,8 @@ impl Tool for CodeSymbolsTool {
IconName::Eye
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(CodeSymbolsInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<CodeSymbolsInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::LanguageModelToolSchemaFormat;
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -53,9 +55,8 @@ impl Tool for CopyPathTool {
IconName::Clipboard
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(CopyPathToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<CopyPathToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::LanguageModelToolSchemaFormat;
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -43,9 +45,8 @@ impl Tool for CreateDirectoryTool {
IconName::Folder
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(CreateDirectoryToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<CreateDirectoryToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::LanguageModelToolSchemaFormat;
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -50,9 +52,8 @@ impl Tool for CreateFileTool {
IconName::FileCreate
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(CreateFileToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<CreateFileToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,8 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use futures::{channel::mpsc, SinkExt, StreamExt};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::{Project, ProjectPath};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -44,9 +45,8 @@ impl Tool for DeletePathTool {
IconName::FileDelete
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(DeletePathToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<DeletePathToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,8 +1,9 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language::{DiagnosticSeverity, OffsetRangeExt};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -57,9 +58,8 @@ impl Tool for DiagnosticsTool {
IconName::Warning
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(DiagnosticsToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<DiagnosticsToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -2,12 +2,14 @@ mod edit_action;
pub mod log;
use crate::replace::{replace_exact, replace_with_flexible_indent};
use crate::schema::json_schema_for;
use anyhow::{anyhow, Context, Result};
use assistant_tool::{ActionLog, Tool};
use collections::HashSet;
use edit_action::{EditAction, EditActionParser};
use futures::{channel::mpsc, SinkExt, StreamExt};
use gpui::{App, AppContext, AsyncApp, Entity, Task};
use language_model::LanguageModelToolSchemaFormat;
use language_model::{
LanguageModelRegistry, LanguageModelRequest, LanguageModelRequestMessage, MessageContent, Role,
};
@ -91,9 +93,8 @@ impl Tool for EditFilesTool {
IconName::Pencil
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(EditFilesToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<EditFilesToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -2,13 +2,14 @@ use std::cell::RefCell;
use std::rc::Rc;
use std::sync::Arc;
use crate::schema::json_schema_for;
use anyhow::{anyhow, bail, Context as _, Result};
use assistant_tool::{ActionLog, Tool};
use futures::AsyncReadExt as _;
use gpui::{App, AppContext as _, Entity, Task};
use html_to_markdown::{convert_html_to_markdown, markdown, TagHandler};
use http_client::{AsyncBody, HttpClientWithUrl};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -127,9 +128,8 @@ impl Tool for FetchTool {
IconName::Globe
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(FetchToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<FetchToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Context as _, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -140,9 +141,8 @@ impl Tool for FindReplaceFileTool {
IconName::Pencil
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(FindReplaceFileToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<FindReplaceFileToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -55,9 +56,8 @@ impl Tool for ListDirectoryTool {
IconName::Folder
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(ListDirectoryToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<ListDirectoryToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -53,9 +54,8 @@ impl Tool for MovePathTool {
IconName::ArrowRightLeft
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(MovePathToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<MovePathToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,10 +1,11 @@
use std::sync::Arc;
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use chrono::{Local, Utc};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -44,9 +45,8 @@ impl Tool for NowTool {
IconName::Info
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(NowToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<NowToolInput>(format)
}
fn ui_text(&self, _input: &serde_json::Value) -> String {

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Context as _, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -34,9 +35,8 @@ impl Tool for OpenTool {
IconName::ExternalLink
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(OpenToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<OpenToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,7 +1,8 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AppContext, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -52,9 +53,8 @@ impl Tool for PathSearchTool {
IconName::SearchCode
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(PathSearchToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<PathSearchToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,11 +1,12 @@
use std::path::Path;
use std::sync::Arc;
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use itertools::Itertools;
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -58,9 +59,8 @@ impl Tool for ReadFileTool {
IconName::Eye
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(ReadFileToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<ReadFileToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,9 +1,10 @@
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use futures::StreamExt;
use gpui::{App, Entity, Task};
use language::OffsetRangeExt;
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::{
search::{SearchQuery, SearchResult},
Project,
@ -55,9 +56,8 @@ impl Tool for RegexSearchTool {
IconName::Regex
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(RegexSearchToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<RegexSearchToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -0,0 +1,120 @@
use anyhow::Result;
use language_model::LanguageModelToolSchemaFormat;
use schemars::{
schema::{RootSchema, Schema, SchemaObject},
JsonSchema,
};
pub fn json_schema_for<T: JsonSchema>(format: LanguageModelToolSchemaFormat) -> serde_json::Value {
let schema = root_schema_for::<T>(format);
schema_to_json(&schema, format).expect("Failed to convert tool calling schema to JSON")
}
pub fn schema_to_json(
schema: &RootSchema,
format: LanguageModelToolSchemaFormat,
) -> Result<serde_json::Value> {
let mut value = serde_json::to_value(schema)?;
match format {
LanguageModelToolSchemaFormat::JsonSchema => Ok(value),
LanguageModelToolSchemaFormat::JsonSchemaSubset => {
transform_fields_to_json_schema_subset(&mut value);
Ok(value)
}
}
}
fn root_schema_for<T: JsonSchema>(format: LanguageModelToolSchemaFormat) -> RootSchema {
let mut generator = match format {
LanguageModelToolSchemaFormat::JsonSchema => schemars::SchemaGenerator::default(),
LanguageModelToolSchemaFormat::JsonSchemaSubset => {
schemars::r#gen::SchemaSettings::default()
.with(|settings| {
settings.meta_schema = None;
settings.inline_subschemas = true;
settings
.visitors
.push(Box::new(TransformToJsonSchemaSubsetVisitor));
})
.into_generator()
}
};
generator.root_schema_for::<T>()
}
#[derive(Debug, Clone)]
struct TransformToJsonSchemaSubsetVisitor;
impl schemars::visit::Visitor for TransformToJsonSchemaSubsetVisitor {
fn visit_root_schema(&mut self, root: &mut RootSchema) {
schemars::visit::visit_root_schema(self, root)
}
fn visit_schema(&mut self, schema: &mut Schema) {
schemars::visit::visit_schema(self, schema)
}
fn visit_schema_object(&mut self, schema: &mut SchemaObject) {
// Ensure that the type field is not an array, this happens when we use
// Option<T>, the type will be [T, "null"].
if let Some(instance_type) = schema.instance_type.take() {
schema.instance_type = match instance_type {
schemars::schema::SingleOrVec::Single(t) => {
Some(schemars::schema::SingleOrVec::Single(t))
}
schemars::schema::SingleOrVec::Vec(items) => items
.into_iter()
.next()
.map(schemars::schema::SingleOrVec::from),
};
}
// One of is not supported, use anyOf instead.
if let Some(subschema) = schema.subschemas.as_mut() {
if let Some(one_of) = subschema.one_of.take() {
subschema.any_of = Some(one_of);
}
}
schemars::visit::visit_schema_object(self, schema)
}
}
fn transform_fields_to_json_schema_subset(json: &mut serde_json::Value) {
if let serde_json::Value::Object(obj) = json {
if let Some(default) = obj.get("default") {
let is_null = default.is_null();
//Default is not supported, so we need to remove it.
obj.remove("default");
if is_null {
obj.insert("nullable".to_string(), serde_json::Value::Bool(true));
}
}
// If a type is not specified for an input parameter we need to add it.
if obj.contains_key("description")
&& !obj.contains_key("type")
&& !(obj.contains_key("anyOf")
|| obj.contains_key("oneOf")
|| obj.contains_key("allOf"))
{
obj.insert(
"type".to_string(),
serde_json::Value::String("string".to_string()),
);
}
//Format field is only partially supported (e.g. not uint compatibility)
obj.remove("format");
for (_, value) in obj.iter_mut() {
if let serde_json::Value::Object(_) | serde_json::Value::Array(_) = value {
transform_fields_to_json_schema_subset(value);
}
}
} else if let serde_json::Value::Array(arr) = json {
for item in arr.iter_mut() {
transform_fields_to_json_schema_subset(item);
}
}
}

View file

@ -2,7 +2,7 @@ use anyhow::{anyhow, Context as _, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, AsyncApp, Entity, Task};
use language::{self, Anchor, Buffer, BufferSnapshot, Location, Point, ToPoint, ToPointUtf16};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -10,6 +10,8 @@ use std::{fmt::Write, ops::Range, sync::Arc};
use ui::IconName;
use util::markdown::MarkdownString;
use crate::schema::json_schema_for;
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
pub struct SymbolInfoToolInput {
/// The relative path to the file containing the symbol.
@ -82,9 +84,8 @@ impl Tool for SymbolInfoTool {
IconName::Eye
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(SymbolInfoToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<SymbolInfoToolInput>(format)
}
fn ui_text(&self, input: &serde_json::Value) -> String {

View file

@ -1,9 +1,10 @@
use std::sync::Arc;
use crate::schema::json_schema_for;
use anyhow::{anyhow, Result};
use assistant_tool::{ActionLog, Tool};
use gpui::{App, Entity, Task};
use language_model::LanguageModelRequestMessage;
use language_model::{LanguageModelRequestMessage, LanguageModelToolSchemaFormat};
use project::Project;
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@ -35,9 +36,8 @@ impl Tool for ThinkingTool {
IconName::Brain
}
fn input_schema(&self) -> serde_json::Value {
let schema = schemars::schema_for!(ThinkingToolInput);
serde_json::to_value(&schema).unwrap()
fn input_schema(&self, format: LanguageModelToolSchemaFormat) -> serde_json::Value {
json_schema_for::<ThinkingToolInput>(format)
}
fn ui_text(&self, _input: &serde_json::Value) -> String {