agent: Less disruptive changed file notification (#31693)

When the user edits one of the tracked files, we used to notify the
agent by inserting a user message at the end of the thread. This was
causing a few problems:
- The agent would stop doing its work and start reading changed files
- The agent would write something like, "Thank you for letting me know
about these changed files."

This fix contains two parts:
1. Changing the prompt to indicate this is a service message
2. Moving the message higher in the conversation thread

This works, but it slightly hurts caching.

We may consider making these notification messages stick in history,
trading context tokens count for the cache.

This might be related to #30906

Release Notes:

- N/A

---------

Co-authored-by: Marshall Bowers <git@maxdeviant.com>

crates/eval/src/examples/file_change_notification.rs

@@ -0,0 +1,74 @@
+use agent_settings::AgentProfileId;
+use anyhow::Result;
+use async_trait::async_trait;
+
+use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion};
+
+pub struct FileChangeNotificationExample;
+
+#[async_trait(?Send)]
+impl Example for FileChangeNotificationExample {
+    fn meta(&self) -> ExampleMetadata {
+        ExampleMetadata {
+            name: "file_change_notification".to_string(),
+            url: "https://github.com/octocat/hello-world".to_string(),
+            revision: "7fd1a60b01f91b314f59955a4e4d4e80d8edf11d".to_string(),
+            language_server: None,
+            max_assertions: Some(1),
+            profile_id: AgentProfileId::default(),
+            existing_thread_json: None,
+            max_turns: Some(3),
+        }
+    }
+
+    async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> {
+        // Track README so that the model gets notified of its changes
+        let project_path = cx.agent_thread().read_with(cx, |thread, cx| {
+            thread
+                .project()
+                .read(cx)
+                .find_project_path("README", cx)
+                .expect("README file should exist in this repo")
+        })?;
+
+        let buffer = {
+            cx.agent_thread()
+                .update(cx, |thread, cx| {
+                    thread
+                        .project()
+                        .update(cx, |project, cx| project.open_buffer(project_path, cx))
+                })?
+                .await?
+        };
+
+        cx.agent_thread().update(cx, |thread, cx| {
+            thread.action_log().update(cx, |action_log, cx| {
+                action_log.buffer_read(buffer.clone(), cx);
+            });
+        })?;
+
+        // Start conversation (specific message is not important)
+        cx.push_user_message("Find all files in this repo");
+        cx.run_turn().await?;
+
+        // Edit the README buffer - the model should get a notification on next turn
+        buffer.update(cx, |buffer, cx| {
+            buffer.edit([(0..buffer.len(), "Surprise!")], None, cx);
+        })?;
+
+        // Run for some more turns.
+        // The model shouldn't thank us for letting it know about the file change.
+        cx.run_turns(3).await?;
+
+        Ok(())
+    }
+
+    fn thread_assertions(&self) -> Vec<JudgeAssertion> {
+        vec![JudgeAssertion {
+            id: "change-file-notification".into(),
+            description:
+                "Agent should not acknowledge or mention anything about files that have been changed"
+                    .into(),
+        }]
+    }
+}

crates/eval/src/examples/mod.rs

@@ -15,6 +15,7 @@ use crate::example::{Example, ExampleContext, ExampleMetadata, JudgeAssertion};
 mod add_arg_to_trait_method;
 mod code_block_citations;
 mod comment_translation;
+mod file_change_notification;
 mod file_search;
 mod grep_params_escapement;
 mod overwrite_file;
@@ -28,6 +29,7 @@ pub fn all(examples_dir: &Path) -> Vec<Rc<dyn Example>> {
         Rc::new(planets::Planets),
         Rc::new(comment_translation::CommentTranslation),
         Rc::new(overwrite_file::FileOverwriteExample),
+        Rc::new(file_change_notification::FileChangeNotificationExample),
         Rc::new(grep_params_escapement::GrepParamsEscapementExample),
     ];
 

crates/eval/src/instance.rs

@@ -367,7 +367,13 @@ impl ExampleInstance {
                 });
             })?;
 
-            let mut example_cx = ExampleContext::new(meta.clone(), this.log_prefix.clone(), thread.clone(), model.clone(), cx.clone());
+            let mut example_cx = ExampleContext::new(
+                meta.clone(),
+                this.log_prefix.clone(),
+                thread.clone(),
+                model.clone(),
+                cx.clone(),
+            );
             let result = this.thread.conversation(&mut example_cx).await;
 
             if let Err(err) = result {