agent: Include grep-related instructions in the prompt only if the tool is available (#29536)

This change updates the system prompt to conditionally include
`grep`-related instructions based on whether the `grep` tool is enabled.

Implementation details:
1. Add a `has_tool` handlebars helper.
2. Pass the `model` to all locations where the prompt is built.
3. Use `{{#if has_tool "grep"}}` in the system prompt to gate
`grep`-specific instructions.

Testing:
- Unit tests for the `hasTool` helper.
- Unit tests to verify that `grep`-related instructions are included /
omitted from the prompt as appropriate.
- Manual agent evaluation:
- Setup: Asked the Agent "List all impls of MyTrait in the project"
using a custom "No tools" profile (all tools disabled).
- Before the change: The Agent attempted to call `grep`, encountered an
error, then realized the tool was unavailable.
- After the change: The Agent immediately asked to enable a search tool.

Note: in principle, `grep`/`read_file` tool descriptions alone might be
enough, but to confirm this we need more evaluation. If it turns out to
be true, we'll be able to remove grep-specific instructions from the
system prompt and undo this change.

Release Notes:

- N/A
This commit is contained in:
Oleksiy Syvokon 2025-04-28 22:47:40 +03:00 committed by GitHub
parent 0e477e7db9
commit 99df1190a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 209 additions and 46 deletions

View file

@ -28,6 +28,7 @@ parking_lot.workspace = true
paths.workspace = true
rope.workspace = true
serde.workspace = true
serde_json.workspace = true
text.workspace = true
util.workspace = true
uuid.workspace = true

View file

@ -48,6 +48,20 @@ impl ProjectContext {
}
}
#[derive(Debug, Clone, Serialize)]
pub struct ModelContext {
pub available_tools: Vec<String>,
}
#[derive(Serialize)]
struct PromptTemplateContext {
#[serde(flatten)]
project: ProjectContext,
#[serde(flatten)]
model: ModelContext,
}
#[derive(Debug, Clone, Serialize)]
pub struct UserRulesContext {
pub uuid: UserPromptId,
@ -124,9 +138,40 @@ impl PromptBuilder {
.unwrap_or_else(|| Arc::new(Self::new(None).unwrap()))
}
/// Helper function for handlebars templates to check if a specific tool is enabled
fn has_tool_helper(
h: &handlebars::Helper,
_: &Handlebars,
ctx: &handlebars::Context,
_: &mut handlebars::RenderContext,
out: &mut dyn handlebars::Output,
) -> handlebars::HelperResult {
let tool_name = h.param(0).and_then(|v| v.value().as_str()).ok_or_else(|| {
handlebars::RenderError::new("has_tool helper: missing or invalid tool name parameter")
})?;
let enabled_tools = ctx
.data()
.get("available_tools")
.and_then(|v| v.as_array())
.map(|arr| arr.iter().filter_map(|v| v.as_str()).collect::<Vec<&str>>())
.ok_or_else(|| {
handlebars::RenderError::new(
"has_tool handlebars helper: available_tools not found or not an array",
)
})?;
if enabled_tools.contains(&tool_name) {
out.write("true")?;
}
Ok(())
}
pub fn new(loading_params: Option<PromptLoadingParams>) -> Result<Self> {
let mut handlebars = Handlebars::new();
Self::register_built_in_templates(&mut handlebars)?;
handlebars.register_helper("has_tool", Box::new(Self::has_tool_helper));
let handlebars = Arc::new(Mutex::new(handlebars));
@ -278,10 +323,16 @@ impl PromptBuilder {
pub fn generate_assistant_system_prompt(
&self,
context: &ProjectContext,
model_context: &ModelContext,
) -> Result<String, RenderError> {
let template_context = PromptTemplateContext {
project: context.clone(),
model: model_context.clone(),
};
self.handlebars
.lock()
.render("assistant_system_prompt", context)
.render("assistant_system_prompt", &template_context)
}
pub fn generate_inline_transformation_prompt(
@ -398,6 +449,7 @@ impl PromptBuilder {
#[cfg(test)]
mod test {
use super::*;
use serde_json;
use uuid::Uuid;
#[test]
@ -416,9 +468,73 @@ mod test {
contents: "Rules contents".into(),
}];
let project_context = ProjectContext::new(worktrees, default_user_rules);
PromptBuilder::new(None)
let model_context = ModelContext {
available_tools: ["grep".into()].to_vec(),
};
let prompt = PromptBuilder::new(None)
.unwrap()
.generate_assistant_system_prompt(&project_context)
.generate_assistant_system_prompt(&project_context, &model_context)
.unwrap();
assert!(
prompt.contains("Rules contents"),
"Expected default user rules to be in rendered prompt"
);
}
#[test]
fn test_assistant_system_prompt_depends_on_enabled_tools() {
let worktrees = vec![WorktreeContext {
root_name: "path".into(),
rules_file: None,
}];
let default_user_rules = vec![];
let project_context = ProjectContext::new(worktrees, default_user_rules);
let prompt_builder = PromptBuilder::new(None).unwrap();
// When the `grep` tool is enabled, it should be mentioned in the prompt
let model_context = ModelContext {
available_tools: ["grep".into()].to_vec(),
};
let prompt_with_grep = prompt_builder
.generate_assistant_system_prompt(&project_context, &model_context)
.unwrap();
assert!(
prompt_with_grep.contains("grep"),
"`grep` tool should be mentioned in prompt when the tool is enabled"
);
// When the `grep` tool is disabled, it should not be mentioned in the prompt
let model_context = ModelContext {
available_tools: [].to_vec(),
};
let prompt_without_grep = prompt_builder
.generate_assistant_system_prompt(&project_context, &model_context)
.unwrap();
assert!(
!prompt_without_grep.contains("grep"),
"`grep` tool should not be mentioned in prompt when the tool is disabled"
);
}
#[test]
fn test_has_tool_helper() {
let mut handlebars = Handlebars::new();
handlebars.register_helper("has_tool", Box::new(PromptBuilder::has_tool_helper));
handlebars
.register_template_string(
"test_template",
"{{#if (has_tool 'grep')}}grep is enabled{{else}}grep is disabled{{/if}}",
)
.unwrap();
// grep available
let data = serde_json::json!({"available_tools": ["grep", "fetch"]});
let result = handlebars.render("test_template", &data).unwrap();
assert_eq!(result, "grep is enabled");
// grep not available
let data = serde_json::json!({"available_tools": ["terminal", "fetch"]});
let result = handlebars.render("test_template", &data).unwrap();
assert_eq!(result, "grep is disabled");
}
}