danger: Check for changes in prompt files (#31744)

This PR adds a Danger check to remind engineers that any changes to our
various prompts need to be verified against the LLM Worker.

When changes to the prompt files are detected, we will fail the PR with
a message:

<img width="929" alt="Screenshot 2025-05-30 at 8 40 58 AM"
src="https://github.com/user-attachments/assets/79afab4e-e799-45f1-a90e-0fd7c9a73706"
/>

Once the corresponding changes have been made (or no changes to the LLM
Worker have been determined to be necessary), including the indicated
attestation message will convert the errors into informational messages:

<img width="926" alt="Screenshot 2025-05-30 at 8 41 52 AM"
src="https://github.com/user-attachments/assets/ff51c17a-7a76-46a7-b468-a7d864d480c3"
/>

Release Notes:

- N/A

crates/agent/src/prompts/stale_files_prompt_header.txt

@@ -0,0 +1 @@
+These files changed since last read:

crates/agent/src/prompts/summarize_thread_detailed_prompt.txt

@@ -0,0 +1,6 @@
+Generate a detailed summary of this conversation. Include:
+1. A brief overview of what was discussed
+2. Key facts or information discovered
+3. Outcomes or conclusions reached
+4. Any action items or next steps if any
+Format it in Markdown with headings and bullet points.

crates/agent/src/prompts/summarize_thread_prompt.txt

@@ -0,0 +1,4 @@
+Generate a concise 3-7 word title for this conversation, omitting punctuation.
+Go straight to the title, without any preamble and prefix like `Here's a concise suggestion:...` or `Title:`.
+If the conversation is about a specific subject, include it in the title.
+Be descriptive. DO NOT speak in the first person.

crates/agent/src/thread.rs

@@ -1428,7 +1428,7 @@ impl Thread {
         messages: &mut Vec<LanguageModelRequestMessage>,
         cx: &App,
     ) {
-        const STALE_FILES_HEADER: &str = "These files changed since last read:";
+        const STALE_FILES_HEADER: &str = include_str!("./prompts/stale_files_prompt_header.txt");
 
         let mut stale_message = String::new();
 
@@ -1440,7 +1440,7 @@ impl Thread {
             };
 
             if stale_message.is_empty() {
-                write!(&mut stale_message, "{}\n", STALE_FILES_HEADER).ok();
+                write!(&mut stale_message, "{}\n", STALE_FILES_HEADER.trim()).ok();
             }
 
             writeln!(&mut stale_message, "- {}", file.path().display()).ok();
@@ -1854,10 +1854,7 @@ impl Thread {
             return;
         }
 
-        let added_user_message = "Generate a concise 3-7 word title for this conversation, omitting punctuation. \
-            Go straight to the title, without any preamble and prefix like `Here's a concise suggestion:...` or `Title:`. \
-            If the conversation is about a specific subject, include it in the title. \
-            Be descriptive. DO NOT speak in the first person.";
+        let added_user_message = include_str!("./prompts/summarize_thread_prompt.txt");
 
         let request = self.to_summarize_request(
             &model.model,
@@ -1958,12 +1955,7 @@ impl Thread {
             return;
         }
 
-        let added_user_message = "Generate a detailed summary of this conversation. Include:\n\
-             1. A brief overview of what was discussed\n\
-             2. Key facts or information discovered\n\
-             3. Outcomes or conclusions reached\n\
-             4. Any action items or next steps if any\n\
-             Format it in Markdown with headings and bullet points.";
+        let added_user_message = include_str!("./prompts/summarize_thread_detailed_prompt.txt");
 
         let request = self.to_summarize_request(
             &model,

script/danger/dangerfile.ts

@@ -1,4 +1,4 @@
-import { danger, message, warn } from "danger";
+import { danger, message, warn, fail } from "danger";
 const { prHygiene } = require("danger-plugin-pr-hygiene");
 
 prHygiene({
@@ -57,3 +57,39 @@ if (includesIssueUrl) {
     ].join("\n"),
   );
 }
+
+const PROMPT_PATHS = [
+  "assets/prompts/content_prompt.hbs",
+  "assets/prompts/terminal_assistant_prompt.hbs",
+  "crates/agent/src/prompts/stale_files_prompt_header.txt",
+  "crates/agent/src/prompts/summarize_thread_detailed_prompt.txt",
+  "crates/agent/src/prompts/summarize_thread_prompt.txt",
+  "crates/assistant_tools/src/templates/create_file_prompt.hbs",
+  "crates/assistant_tools/src/templates/edit_file_prompt.hbs",
+  "crates/git_ui/src/commit_message_prompt.txt",
+];
+
+const PROMPT_CHANGE_ATTESTATION = "I have ensured the LLM Worker works with these prompt changes.";
+
+const modifiedPrompts = danger.git.modified_files.filter((file) =>
+  PROMPT_PATHS.some((promptPath) => file.includes(promptPath)),
+);
+
+for (const promptPath of modifiedPrompts) {
+  if (body.includes(PROMPT_CHANGE_ATTESTATION)) {
+    message(
+      [
+        `This PR contains changes to "${promptPath}".`,
+        "The author has attested the LLM Worker works with the changes to this prompt.",
+      ].join("\n"),
+    );
+  } else {
+    fail(
+      [
+        `Modifying the "${promptPath}" prompt may require corresponding changes in the LLM Worker.`,
+        "If you are ensure what this entails, talk to @maxdeviant or another AI team member.",
+        `Once you have made the changes—or determined that none are necessary—add "${PROMPT_CHANGE_ATTESTATION}" to the PR description.`,
+      ].join("\n"),
+    );
+  }
+}