Add reminder message about system prompt (#28344)

Trying out sending the model a reminder message about code blocks in the
system prompt. If this seems to work well, we can include more specific
reminder messages, e.g. tool-specific ones.

Release Notes:

- N/A

assets/prompts/assistant_system_prompt_reminder.hbs

@@ -0,0 +1 @@
+In your response, make sure to remember and follow my instructions about how to format code blocks (and don't mention that you are remembering it, just follow the instructions).

crates/agent/src/thread.rs

@@ -997,6 +997,20 @@ impl Thread {
 
         self.attached_tracked_files_state(&mut request.messages, cx);
 
+        // Add reminder to the last user message about code blocks
+        if let Some(last_user_message) = request
+            .messages
+            .iter_mut()
+            .rev()
+            .find(|msg| msg.role == Role::User)
+        {
+            last_user_message
+                .content
+                .push(MessageContent::Text(system_prompt_reminder(
+                    &self.prompt_builder,
+                )));
+        }
+
         request
     }
 
@@ -1810,6 +1824,12 @@ impl Thread {
     }
 }
 
+pub fn system_prompt_reminder(prompt_builder: &prompt_store::PromptBuilder) -> String {
+    prompt_builder
+        .generate_assistant_system_prompt_reminder()
+        .unwrap_or_default()
+}
+
 #[derive(Debug, Clone)]
 pub enum ThreadError {
     PaymentRequired,
@@ -1879,7 +1899,7 @@ mod tests {
         )
         .await;
 
-        let (_workspace, _thread_store, thread, context_store) =
+        let (_workspace, _thread_store, thread, context_store, prompt_builder) =
             setup_test_environment(cx, project.clone()).await;
 
         add_file_to_context(&project, &context_store, "test/code.rs", cx)
@@ -1933,8 +1953,14 @@ fn main() {{
         });
 
         assert_eq!(request.messages.len(), 1);
-        let expected_full_message = format!("{}Please explain this code", expected_context);
-        assert_eq!(request.messages[0].string_contents(), expected_full_message);
+        let actual_message = request.messages[0].string_contents();
+        let expected_content = format!(
+            "{}Please explain this code{}",
+            expected_context,
+            system_prompt_reminder(&prompt_builder)
+        );
+
+        assert_eq!(actual_message, expected_content);
     }
 
     #[gpui::test]
@@ -1951,7 +1977,7 @@ fn main() {{
         )
         .await;
 
-        let (_, _thread_store, thread, context_store) =
+        let (_, _thread_store, thread, context_store, _prompt_builder) =
             setup_test_environment(cx, project.clone()).await;
 
         // Open files individually
@@ -2051,7 +2077,7 @@ fn main() {{
         )
         .await;
 
-        let (_, _thread_store, thread, _context_store) =
+        let (_, _thread_store, thread, _context_store, prompt_builder) =
             setup_test_environment(cx, project.clone()).await;
 
         // Insert user message without any context (empty context vector)
@@ -2077,11 +2103,14 @@ fn main() {{
         });
 
         assert_eq!(request.messages.len(), 1);
-        assert_eq!(
-            request.messages[0].string_contents(),
-            "What is the best way to learn Rust?"
+        let actual_message = request.messages[0].string_contents();
+        let expected_content = format!(
+            "What is the best way to learn Rust?{}",
+            system_prompt_reminder(&prompt_builder)
         );
 
+        assert_eq!(actual_message, expected_content);
+
         // Add second message, also without context
         let message2_id = thread.update(cx, |thread, cx| {
             thread.insert_user_message("Are there any good books?", vec![], None, cx)
@@ -2097,14 +2126,17 @@ fn main() {{
         });
 
         assert_eq!(request.messages.len(), 2);
-        assert_eq!(
-            request.messages[0].string_contents(),
-            "What is the best way to learn Rust?"
-        );
-        assert_eq!(
-            request.messages[1].string_contents(),
-            "Are there any good books?"
+        // First message should be the system prompt
+        assert_eq!(request.messages[0].role, Role::User);
+
+        // Second message should be the user message with prompt reminder
+        let actual_message = request.messages[1].string_contents();
+        let expected_content = format!(
+            "Are there any good books?{}",
+            system_prompt_reminder(&prompt_builder)
         );
+
+        assert_eq!(actual_message, expected_content);
     }
 
     #[gpui::test]
@@ -2117,7 +2149,7 @@ fn main() {{
         )
         .await;
 
-        let (_workspace, _thread_store, thread, context_store) =
+        let (_workspace, _thread_store, thread, context_store, prompt_builder) =
             setup_test_environment(cx, project.clone()).await;
 
         // Open buffer and add it to context
@@ -2177,11 +2209,14 @@ fn main() {{
         // The last message should be the stale buffer notification
         assert_eq!(last_message.role, Role::User);
 
-        // Check the exact content of the message
-        let expected_content = "These files changed since last read:\n- code.rs\n";
+        let actual_message = last_message.string_contents();
+        let expected_content = format!(
+            "These files changed since last read:\n- code.rs\n{}",
+            system_prompt_reminder(&prompt_builder)
+        );
+
         assert_eq!(
-            last_message.string_contents(),
-            expected_content,
+            actual_message, expected_content,
             "Last message should be exactly the stale buffer notification"
         );
     }
@@ -2219,24 +2254,27 @@ fn main() {{
         Entity<ThreadStore>,
         Entity<Thread>,
         Entity<ContextStore>,
+        Arc<PromptBuilder>,
     ) {
         let (workspace, cx) =
             cx.add_window_view(|window, cx| Workspace::test_new(project.clone(), window, cx));
 
+        let prompt_builder = Arc::new(PromptBuilder::new(None).unwrap());
+
         let thread_store = cx.update(|_, cx| {
-            ThreadStore::new(
-                project.clone(),
-                Arc::default(),
-                Arc::new(PromptBuilder::new(None).unwrap()),
-                cx,
-            )
-            .unwrap()
+            ThreadStore::new(project.clone(), Arc::default(), prompt_builder.clone(), cx).unwrap()
         });
 
         let thread = thread_store.update(cx, |store, cx| store.create_thread(cx));
         let context_store = cx.new(|_cx| ContextStore::new(project.downgrade(), None));
 
-        (workspace, thread_store, thread, context_store)
+        (
+            workspace,
+            thread_store,
+            thread,
+            context_store,
+            prompt_builder,
+        )
     }
 
     async fn add_file_to_context(

crates/prompt_store/src/prompts.rs

@@ -261,6 +261,12 @@ impl PromptBuilder {
             .render("assistant_system_prompt", context)
     }
 
+    pub fn generate_assistant_system_prompt_reminder(&self) -> Result<String, RenderError> {
+        self.handlebars
+            .lock()
+            .render("assistant_system_prompt_reminder", &())
+    }
+
     pub fn generate_inline_transformation_prompt(
         &self,
         user_prompt: String,