eval: New add_arg_to_trait_method example (#29297)

Release Notes:

- N/A

---------

Co-authored-by: Richard Feldman <oss@rtfeldman.com>

crates/eval/src/examples/add_arg_to_trait_method.rs

@@ -0,0 +1,147 @@
+use std::{collections::HashSet, path::Path};
+
+use anyhow::Result;
+use assistant_tools::{CreateFileToolInput, EditFileToolInput, ReadFileToolInput};
+use async_trait::async_trait;
+
+use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion, LanguageServer};
+
+pub struct AddArgToTraitMethod;
+
+#[async_trait(?Send)]
+impl Example for AddArgToTraitMethod {
+    fn meta(&self) -> ExampleMetadata {
+        ExampleMetadata {
+            name: "add_arg_to_trait_method".to_string(),
+            url: "https://github.com/zed-industries/zed.git".to_string(),
+            revision: "f69aeb6311dde3c0b8979c293d019d66498d54f2".to_string(),
+            language_server: Some(LanguageServer {
+                file_extension: "rs".to_string(),
+                allow_preexisting_diagnostics: false,
+            }),
+            max_assertions: None,
+        }
+    }
+
+    async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> {
+        const FILENAME: &str = "assistant_tool.rs";
+        cx.push_user_message(format!(
+            r#"
+            Add a `window: Option<gpui::AnyWindowHandle>` argument to the `Tool::run` trait method in {FILENAME},
+            and update all the implementations of the trait and call sites accordingly.
+            "#
+        ));
+
+        let response = cx.run_to_end().await?;
+
+        // Reads files before it edits them
+
+        let mut read_files = HashSet::new();
+
+        for tool_use in response.tool_uses() {
+            match tool_use.name.as_str() {
+                "read_file" => {
+                    if let Ok(input) = tool_use.parse_input::<ReadFileToolInput>() {
+                        read_files.insert(input.path);
+                    }
+                }
+                "create_file" => {
+                    if let Ok(input) = tool_use.parse_input::<CreateFileToolInput>() {
+                        read_files.insert(input.path);
+                    }
+                }
+                "edit_file" => {
+                    if let Ok(input) = tool_use.parse_input::<EditFileToolInput>() {
+                        cx.assert(
+                            read_files.contains(input.path.to_str().unwrap()),
+                            format!(
+                                "Read before edit: {}",
+                                &input.path.file_stem().unwrap().to_str().unwrap()
+                            ),
+                        )
+                        .ok();
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        // Adds ignored argument to all but `batch_tool`
+
+        let add_ignored_window_paths = &[
+            "code_action_tool",
+            "code_symbols_tool",
+            "contents_tool",
+            "copy_path_tool",
+            "create_directory_tool",
+            "create_file_tool",
+            "delete_path_tool",
+            "diagnostics_tool",
+            "edit_file_tool",
+            "fetch_tool",
+            "grep_tool",
+            "list_directory_tool",
+            "move_path_tool",
+            "now_tool",
+            "open_tool",
+            "path_search_tool",
+            "read_file_tool",
+            "rename_tool",
+            "symbol_info_tool",
+            "terminal_tool",
+            "thinking_tool",
+            "web_search_tool",
+        ];
+
+        let edits = cx.edits();
+
+        for tool_name in add_ignored_window_paths {
+            let path_str = format!("crates/assistant_tools/src/{}.rs", tool_name);
+            let edits = edits.get(Path::new(&path_str));
+
+            let ignored = edits.map_or(false, |edits| {
+                edits.has_added_line("        _window: Option<gpui::AnyWindowHandle>,\n")
+            });
+            let uningored = edits.map_or(false, |edits| {
+                edits.has_added_line("        window: Option<gpui::AnyWindowHandle>,\n")
+            });
+
+            cx.assert(ignored || uningored, format!("Argument:   {}", tool_name))
+                .ok();
+
+            cx.assert(ignored, format!("`_` prefix: {}", tool_name))
+                .ok();
+        }
+
+        // Adds unignored argument to `batch_tool`
+
+        let batch_tool_edits = edits.get(Path::new("crates/assistant_tools/src/batch_tool.rs"));
+
+        cx.assert(
+            batch_tool_edits.map_or(false, |edits| {
+                edits.has_added_line("        window: Option<gpui::AnyWindowHandle>,\n")
+            }),
+            "Argument:   batch_tool",
+        )
+        .ok();
+
+        Ok(())
+    }
+
+    fn diff_assertions(&self) -> Vec<JudgeAssertion> {
+        vec![
+            JudgeAssertion {
+                id: "batch tool passes window to each".to_string(),
+                description:
+                    "batch_tool is modified to pass a clone of the window to each tool it calls."
+                        .to_string(),
+            },
+            JudgeAssertion {
+                id: "tool tests updated".to_string(),
+                description:
+                    "tool tests are updated to pass the new `window` argument (`None` is ok)."
+                        .to_string(),
+            },
+        ]
+    }
+}

crates/eval/src/examples/file_search.rs

@@ -33,7 +33,7 @@ impl Example for FileSearchExample {
 
         let response = cx.run_turn().await?;
         let tool_use = response.expect_tool("path_search", cx)?;
-        let input = tool_use.expect_input::<PathSearchToolInput>(cx)?;
+        let input = tool_use.parse_input::<PathSearchToolInput>()?;
 
         let glob = input.glob;
         cx.assert(

crates/eval/src/examples/mod.rs

@@ -11,10 +11,14 @@ use util::serde::default_true;
 
 use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion};
 
+mod add_arg_to_trait_method;
 mod file_search;
 
 pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
-    let mut threads: Vec<Rc<dyn Example>> = vec![Rc::new(file_search::FileSearchExample)];
+    let mut threads: Vec<Rc<dyn Example>> = vec![
+        Rc::new(file_search::FileSearchExample),
+        Rc::new(add_arg_to_trait_method::AddArgToTraitMethod),
+    ];
 
     for example_path in list_declarative_examples(examples_dir).unwrap() {
         threads.push(Rc::new(DeclarativeExample::load(&example_path).unwrap()));