agent: Include grep-related instructions in the prompt only if the tool is available (#29536)
This change updates the system prompt to conditionally include `grep`-related instructions based on whether the `grep` tool is enabled. Implementation details: 1. Add a `has_tool` handlebars helper. 2. Pass the `model` to all locations where the prompt is built. 3. Use `{{#if has_tool "grep"}}` in the system prompt to gate `grep`-specific instructions. Testing: - Unit tests for the `hasTool` helper. - Unit tests to verify that `grep`-related instructions are included / omitted from the prompt as appropriate. - Manual agent evaluation: - Setup: Asked the Agent "List all impls of MyTrait in the project" using a custom "No tools" profile (all tools disabled). - Before the change: The Agent attempted to call `grep`, encountered an error, then realized the tool was unavailable. - After the change: The Agent immediately asked to enable a search tool. Note: in principle, `grep`/`read_file` tool descriptions alone might be enough, but to confirm this we need more evaluation. If it turns out to be true, we'll be able to remove grep-specific instructions from the system prompt and undo this change. Release Notes: - N/A
This commit is contained in:
parent
0e477e7db9
commit
99df1190a9
5 changed files with 209 additions and 46 deletions
|
@ -28,6 +28,7 @@ parking_lot.workspace = true
|
|||
paths.workspace = true
|
||||
rope.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
text.workspace = true
|
||||
util.workspace = true
|
||||
uuid.workspace = true
|
||||
|
|
|
@ -48,6 +48,20 @@ impl ProjectContext {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct ModelContext {
|
||||
pub available_tools: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PromptTemplateContext {
|
||||
#[serde(flatten)]
|
||||
project: ProjectContext,
|
||||
|
||||
#[serde(flatten)]
|
||||
model: ModelContext,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct UserRulesContext {
|
||||
pub uuid: UserPromptId,
|
||||
|
@ -124,9 +138,40 @@ impl PromptBuilder {
|
|||
.unwrap_or_else(|| Arc::new(Self::new(None).unwrap()))
|
||||
}
|
||||
|
||||
/// Helper function for handlebars templates to check if a specific tool is enabled
|
||||
fn has_tool_helper(
|
||||
h: &handlebars::Helper,
|
||||
_: &Handlebars,
|
||||
ctx: &handlebars::Context,
|
||||
_: &mut handlebars::RenderContext,
|
||||
out: &mut dyn handlebars::Output,
|
||||
) -> handlebars::HelperResult {
|
||||
let tool_name = h.param(0).and_then(|v| v.value().as_str()).ok_or_else(|| {
|
||||
handlebars::RenderError::new("has_tool helper: missing or invalid tool name parameter")
|
||||
})?;
|
||||
|
||||
let enabled_tools = ctx
|
||||
.data()
|
||||
.get("available_tools")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|arr| arr.iter().filter_map(|v| v.as_str()).collect::<Vec<&str>>())
|
||||
.ok_or_else(|| {
|
||||
handlebars::RenderError::new(
|
||||
"has_tool handlebars helper: available_tools not found or not an array",
|
||||
)
|
||||
})?;
|
||||
|
||||
if enabled_tools.contains(&tool_name) {
|
||||
out.write("true")?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn new(loading_params: Option<PromptLoadingParams>) -> Result<Self> {
|
||||
let mut handlebars = Handlebars::new();
|
||||
Self::register_built_in_templates(&mut handlebars)?;
|
||||
handlebars.register_helper("has_tool", Box::new(Self::has_tool_helper));
|
||||
|
||||
let handlebars = Arc::new(Mutex::new(handlebars));
|
||||
|
||||
|
@ -278,10 +323,16 @@ impl PromptBuilder {
|
|||
pub fn generate_assistant_system_prompt(
|
||||
&self,
|
||||
context: &ProjectContext,
|
||||
model_context: &ModelContext,
|
||||
) -> Result<String, RenderError> {
|
||||
let template_context = PromptTemplateContext {
|
||||
project: context.clone(),
|
||||
model: model_context.clone(),
|
||||
};
|
||||
|
||||
self.handlebars
|
||||
.lock()
|
||||
.render("assistant_system_prompt", context)
|
||||
.render("assistant_system_prompt", &template_context)
|
||||
}
|
||||
|
||||
pub fn generate_inline_transformation_prompt(
|
||||
|
@ -398,6 +449,7 @@ impl PromptBuilder {
|
|||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
use serde_json;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
|
@ -416,9 +468,73 @@ mod test {
|
|||
contents: "Rules contents".into(),
|
||||
}];
|
||||
let project_context = ProjectContext::new(worktrees, default_user_rules);
|
||||
PromptBuilder::new(None)
|
||||
let model_context = ModelContext {
|
||||
available_tools: ["grep".into()].to_vec(),
|
||||
};
|
||||
let prompt = PromptBuilder::new(None)
|
||||
.unwrap()
|
||||
.generate_assistant_system_prompt(&project_context)
|
||||
.generate_assistant_system_prompt(&project_context, &model_context)
|
||||
.unwrap();
|
||||
assert!(
|
||||
prompt.contains("Rules contents"),
|
||||
"Expected default user rules to be in rendered prompt"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_assistant_system_prompt_depends_on_enabled_tools() {
|
||||
let worktrees = vec![WorktreeContext {
|
||||
root_name: "path".into(),
|
||||
rules_file: None,
|
||||
}];
|
||||
let default_user_rules = vec![];
|
||||
let project_context = ProjectContext::new(worktrees, default_user_rules);
|
||||
let prompt_builder = PromptBuilder::new(None).unwrap();
|
||||
|
||||
// When the `grep` tool is enabled, it should be mentioned in the prompt
|
||||
let model_context = ModelContext {
|
||||
available_tools: ["grep".into()].to_vec(),
|
||||
};
|
||||
let prompt_with_grep = prompt_builder
|
||||
.generate_assistant_system_prompt(&project_context, &model_context)
|
||||
.unwrap();
|
||||
assert!(
|
||||
prompt_with_grep.contains("grep"),
|
||||
"`grep` tool should be mentioned in prompt when the tool is enabled"
|
||||
);
|
||||
|
||||
// When the `grep` tool is disabled, it should not be mentioned in the prompt
|
||||
let model_context = ModelContext {
|
||||
available_tools: [].to_vec(),
|
||||
};
|
||||
let prompt_without_grep = prompt_builder
|
||||
.generate_assistant_system_prompt(&project_context, &model_context)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!prompt_without_grep.contains("grep"),
|
||||
"`grep` tool should not be mentioned in prompt when the tool is disabled"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_has_tool_helper() {
|
||||
let mut handlebars = Handlebars::new();
|
||||
handlebars.register_helper("has_tool", Box::new(PromptBuilder::has_tool_helper));
|
||||
handlebars
|
||||
.register_template_string(
|
||||
"test_template",
|
||||
"{{#if (has_tool 'grep')}}grep is enabled{{else}}grep is disabled{{/if}}",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// grep available
|
||||
let data = serde_json::json!({"available_tools": ["grep", "fetch"]});
|
||||
let result = handlebars.render("test_template", &data).unwrap();
|
||||
assert_eq!(result, "grep is enabled");
|
||||
|
||||
// grep not available
|
||||
let data = serde_json::json!({"available_tools": ["terminal", "fetch"]});
|
||||
let result = handlebars.render("test_template", &data).unwrap();
|
||||
assert_eq!(result, "grep is disabled");
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue