agent: Handle attempts to use hallucinated tools (#29946)

This change:

1. Catches attempts to use missing tools. If this happens, we now send
Agent a message listing available tools, after which Agent can
gracefully recover. Prior behavior: thread would stop in a broken state.

Example of a hallucinated call and a message we send back: 

![image](https://github.com/user-attachments/assets/92a8f700-b192-4038-8c7e-0a74ca2e0146)

2. Adds evals for hallucinated tool use and imagined edits
3. Adds ability to configure a profile name in evals.



Release Notes:

- N/A

Cargo.lock

@@ -5019,6 +5019,7 @@ version = "0.1.0"
 dependencies = [
  "agent",
  "anyhow",
+ "assistant_settings",
  "assistant_tool",
  "assistant_tools",
  "async-trait",

crates/agent/src/active_thread.rs

@@ -1070,6 +1070,22 @@ impl ActiveThread {
                     cx,
                 );
             }
+            ThreadEvent::MissingToolUse {
+                tool_use_id,
+                ui_text,
+            } => {
+                self.render_tool_use_markdown(
+                    tool_use_id.clone(),
+                    ui_text,
+                    "",
+                    self.thread
+                        .read(cx)
+                        .output_for_tool(tool_use_id)
+                        .map(|output| output.clone().into())
+                        .unwrap_or("".into()),
+                    cx,
+                );
+            }
         }
     }
 

crates/agent/src/agent_diff.rs

@@ -1372,6 +1372,7 @@ impl AgentDiff {
             | ThreadEvent::StreamedAssistantThinking(_, _)
             | ThreadEvent::StreamedToolUse { .. }
             | ThreadEvent::InvalidToolInput { .. }
+            | ThreadEvent::MissingToolUse { .. }
             | ThreadEvent::MessageAdded(_)
             | ThreadEvent::MessageEdited(_)
             | ThreadEvent::MessageDeleted(_)

crates/agent/src/thread.rs

@@ -1911,12 +1911,54 @@ impl Thread {
                         cx,
                     );
                 }
+            } else {
+                self.handle_hallucinated_tool_use(
+                    tool_use.id.clone(),
+                    tool_use.name.clone(),
+                    window,
+                    cx,
+                );
             }
         }
 
         pending_tool_uses
     }
 
+    pub fn handle_hallucinated_tool_use(
+        &mut self,
+        tool_use_id: LanguageModelToolUseId,
+        hallucinated_tool_name: Arc<str>,
+        window: Option<AnyWindowHandle>,
+        cx: &mut Context<Thread>,
+    ) {
+        let available_tools = self.tools.read(cx).enabled_tools(cx);
+
+        let tool_list = available_tools
+            .iter()
+            .map(|tool| format!("- {}: {}", tool.name(), tool.description()))
+            .collect::<Vec<_>>()
+            .join("\n");
+
+        let error_message = format!(
+            "The tool '{}' doesn't exist or is not enabled. Available tools:\n{}",
+            hallucinated_tool_name, tool_list
+        );
+
+        let pending_tool_use = self.tool_use.insert_tool_output(
+            tool_use_id.clone(),
+            hallucinated_tool_name,
+            Err(anyhow!("Missing tool call: {error_message}")),
+            self.configured_model.as_ref(),
+        );
+
+        cx.emit(ThreadEvent::MissingToolUse {
+            tool_use_id: tool_use_id.clone(),
+            ui_text: error_message.into(),
+        });
+
+        self.tool_finished(tool_use_id, pending_tool_use, false, window, cx);
+    }
+
     pub fn receive_invalid_tool_json(
         &mut self,
         tool_use_id: LanguageModelToolUseId,
@@ -2574,6 +2616,10 @@ pub enum ThreadEvent {
         ui_text: Arc<str>,
         input: serde_json::Value,
     },
+    MissingToolUse {
+        tool_use_id: LanguageModelToolUseId,
+        ui_text: Arc<str>,
+    },
     InvalidToolInput {
         tool_use_id: LanguageModelToolUseId,
         ui_text: Arc<str>,

crates/eval/Cargo.toml

@@ -20,6 +20,7 @@ path = "src/explorer.rs"
 [dependencies]
 agent.workspace = true
 anyhow.workspace = true
+assistant_settings.workspace = true
 assistant_tool.workspace = true
 assistant_tools.workspace = true
 async-trait.workspace = true

crates/eval/src/example.rs

@@ -12,6 +12,7 @@ use crate::{
 };
 use agent::{ContextLoadResult, Thread, ThreadEvent};
 use anyhow::{Result, anyhow};
+use assistant_settings::AgentProfileId;
 use async_trait::async_trait;
 use buffer_diff::DiffHunkStatus;
 use collections::HashMap;
@@ -46,6 +47,7 @@ pub struct ExampleMetadata {
     pub revision: String,
     pub language_server: Option<LanguageServer>,
     pub max_assertions: Option<usize>,
+    pub profile_id: AgentProfileId,
 }
 
 #[derive(Clone, Debug)]
@@ -268,6 +270,12 @@ impl ExampleContext {
                 ThreadEvent::InvalidToolInput { .. } => {
                     println!("{log_prefix} invalid tool input");
                 }
+                ThreadEvent::MissingToolUse {
+                    tool_use_id: _,
+                    ui_text,
+                } => {
+                    println!("{log_prefix} {ui_text}");
+                }
                 ThreadEvent::ToolConfirmationNeeded => {
                     panic!(
                         "{}Bug: Tool confirmation should not be required in eval",

crates/eval/src/examples/add_arg_to_trait_method.rs

@@ -1,6 +1,7 @@
 use std::path::Path;
 
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use async_trait::async_trait;
 
 use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion, LanguageServer};
@@ -19,6 +20,7 @@ impl Example for AddArgToTraitMethod {
                 allow_preexisting_diagnostics: false,
             }),
             max_assertions: None,
+            profile_id: AgentProfileId::default(),
         }
     }
 

crates/eval/src/examples/code_block_citations.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use async_trait::async_trait;
 use markdown::PathWithRange;
 
@@ -20,6 +21,7 @@ impl Example for CodeBlockCitations {
                 allow_preexisting_diagnostics: false,
             }),
             max_assertions: None,
+            profile_id: AgentProfileId::default(),
         }
     }
 

crates/eval/src/examples/comment_translation.rs

@@ -1,5 +1,6 @@
 use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion};
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use assistant_tools::StreamingEditFileToolInput;
 use async_trait::async_trait;
 
@@ -14,6 +15,7 @@ impl Example for CommentTranslation {
             revision: "504d084e29bce4f60614bc702e91af7f7d9e60ad".to_string(),
             language_server: None,
             max_assertions: Some(1),
+            profile_id: AgentProfileId::default(),
         }
     }
 

crates/eval/src/examples/file_search.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use assistant_tools::FindPathToolInput;
 use async_trait::async_trait;
 use regex::Regex;
@@ -16,6 +17,7 @@ impl Example for FileSearchExample {
             revision: "03ecb88fe30794873f191ddb728f597935b3101c".to_string(),
             language_server: None,
             max_assertions: Some(3),
+            profile_id: AgentProfileId::default(),
         }
     }
 

crates/eval/src/examples/hallucinated_tool_calls.toml

@@ -0,0 +1,13 @@
+url = "https://github.com/jlowin/fastmcp"
+revision = "a2c1e14e5d83af1c32b76280ab368df199c4e860"
+language_extension = "py"
+
+prompt = "Write a LICENSE file just saying 'Apache 2.0' and nothing else"
+
+profile_name = "ask"
+
+[thread_assertions]
+
+no_edit_attempts = """The agent should not claim that it edited or created the file. It should not pretend making any changes."""
+
+mention_insufficient_tools = """Agent should mention that it doesn't have relevant tools needed to make the change."""

crates/eval/src/examples/mod.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use async_trait::async_trait;
 use serde::Deserialize;
 use std::collections::BTreeMap;
@@ -56,12 +57,19 @@ impl DeclarativeExample {
             None
         };
 
+        let profile_id = if let Some(profile_name) = base.profile_name {
+            AgentProfileId(profile_name.into())
+        } else {
+            AgentProfileId::default()
+        };
+
         let metadata = ExampleMetadata {
             name,
             url: base.url,
             revision: base.revision,
             language_server,
             max_assertions: None,
+            profile_id,
         };
 
         Ok(DeclarativeExample {
@@ -97,6 +105,8 @@ pub struct ExampleToml {
     pub allow_preexisting_diagnostics: bool,
     pub prompt: String,
     #[serde(default)]
+    pub profile_name: Option<String>,
+    #[serde(default)]
     pub diff_assertions: BTreeMap<String, String>,
     #[serde(default)]
     pub thread_assertions: BTreeMap<String, String>,

crates/eval/src/examples/planets.rs

@@ -1,4 +1,5 @@
 use anyhow::Result;
+use assistant_settings::AgentProfileId;
 use assistant_tool::Tool;
 use assistant_tools::{OpenTool, TerminalTool};
 use async_trait::async_trait;
@@ -16,6 +17,7 @@ impl Example for Planets {
             revision: "59e49c75214f60b4dc4a45092292061c8c26ce27".to_string(), // so effectively a blank project.
             language_server: None,
             max_assertions: None,
+            profile_id: AgentProfileId::default(),
         }
     }
 

crates/eval/src/instance.rs

@@ -307,9 +307,14 @@ impl ExampleInstance {
             std::fs::write(&last_diff_file_path, "")?;
 
             let thread_store = thread_store.await?;
+
+            let profile_id = meta.profile_id.clone();
+            thread_store.update(cx, |thread_store, cx| thread_store.load_profile_by_id(profile_id, cx)).expect("Failed to load profile");
+
             let thread =
                 thread_store.update(cx, |thread_store, cx| thread_store.create_thread(cx))?;
 
+
             thread.update(cx, |thread, _cx| {
                 let mut request_count = 0;
                 let previous_diff = Rc::new(RefCell::new("".to_string()));