agent: Improve initial file search quality (#29317)
This PR significantly improves the quality of the initial file search that occurs when the model doesn't yet know the full path to a file it needs to read/edit. Previously, the assertions in file_search often failed on main as the model attempted to guess full file paths. On this branch, it reliably calls `find_path` (previously `path_search`) before reading files. After getting the model to find paths first, I noticed it would try using `grep` instead of `path_search`. This motivated renaming `path_search` to `find_path` (continuing the analogy to unix commands) and adding system prompt instructions about proper tool selection. Note: I know the command is just called `find`, but that seemed too general. In my eval runs, the `file_search` example improved from 40% ± 10% to 98% ± 2%. The only assertion I'm seeing occasionally fail is "glob starts with `**` or project". We can probably add some instructions in that regard. Release Notes: - N/A
This commit is contained in:
parent
2124b7ea99
commit
8b5835de17
12 changed files with 107 additions and 28 deletions
|
@ -1,5 +1,5 @@
|
|||
use anyhow::Result;
|
||||
use assistant_tools::PathSearchToolInput;
|
||||
use assistant_tools::FindPathToolInput;
|
||||
use async_trait::async_trait;
|
||||
use regex::Regex;
|
||||
|
||||
|
@ -15,7 +15,7 @@ impl Example for FileSearchExample {
|
|||
url: "https://github.com/zed-industries/zed.git".to_string(),
|
||||
revision: "03ecb88fe30794873f191ddb728f597935b3101c".to_string(),
|
||||
language_server: None,
|
||||
max_assertions: Some(4),
|
||||
max_assertions: Some(3),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -32,21 +32,18 @@ impl Example for FileSearchExample {
|
|||
));
|
||||
|
||||
let response = cx.run_turn().await?;
|
||||
let tool_use = response.expect_tool("path_search", cx)?;
|
||||
let input = tool_use.parse_input::<PathSearchToolInput>()?;
|
||||
let tool_use = response.expect_tool("find_path", cx)?;
|
||||
let input = tool_use.parse_input::<FindPathToolInput>()?;
|
||||
|
||||
let glob = input.glob;
|
||||
cx.assert(
|
||||
glob.ends_with(FILENAME),
|
||||
format!("glob ends with `{FILENAME}`"),
|
||||
)?;
|
||||
cx.assert(glob.ends_with(FILENAME), "glob ends with file name")?;
|
||||
|
||||
let without_filename = glob.replace(FILENAME, "");
|
||||
let matches = Regex::new("(\\*\\*|zed)/(\\*\\*?/)?")
|
||||
.unwrap()
|
||||
.is_match(&without_filename);
|
||||
|
||||
cx.assert(matches, "glob starts with either `**` or `zed`")?;
|
||||
cx.assert(matches, "glob starts with `**` or project")?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue