From 310ea43048a046c2275261befbf1acd341689362 Mon Sep 17 00:00:00 2001 From: Marshall Bowers Date: Fri, 30 May 2025 09:46:41 -0400 Subject: [PATCH] danger: Check for changes in prompt files (#31744) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds a Danger check to remind engineers that any changes to our various prompts need to be verified against the LLM Worker. When changes to the prompt files are detected, we will fail the PR with a message: Screenshot 2025-05-30 at 8 40 58 AM Once the corresponding changes have been made (or no changes to the LLM Worker have been determined to be necessary), including the indicated attestation message will convert the errors into informational messages: Screenshot 2025-05-30 at 8 41 52 AM Release Notes: - N/A --- .../src/prompts/stale_files_prompt_header.txt | 1 + .../summarize_thread_detailed_prompt.txt | 6 +++ .../src/prompts/summarize_thread_prompt.txt | 4 ++ crates/agent/src/thread.rs | 16 ++------ script/danger/dangerfile.ts | 38 ++++++++++++++++++- 5 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 crates/agent/src/prompts/stale_files_prompt_header.txt create mode 100644 crates/agent/src/prompts/summarize_thread_detailed_prompt.txt create mode 100644 crates/agent/src/prompts/summarize_thread_prompt.txt diff --git a/crates/agent/src/prompts/stale_files_prompt_header.txt b/crates/agent/src/prompts/stale_files_prompt_header.txt new file mode 100644 index 0000000000..6686aba1e2 --- /dev/null +++ b/crates/agent/src/prompts/stale_files_prompt_header.txt @@ -0,0 +1 @@ +These files changed since last read: diff --git a/crates/agent/src/prompts/summarize_thread_detailed_prompt.txt b/crates/agent/src/prompts/summarize_thread_detailed_prompt.txt new file mode 100644 index 0000000000..30fab472af --- /dev/null +++ b/crates/agent/src/prompts/summarize_thread_detailed_prompt.txt @@ -0,0 +1,6 @@ +Generate a detailed summary of this conversation. Include: +1. A brief overview of what was discussed +2. Key facts or information discovered +3. Outcomes or conclusions reached +4. Any action items or next steps if any +Format it in Markdown with headings and bullet points. diff --git a/crates/agent/src/prompts/summarize_thread_prompt.txt b/crates/agent/src/prompts/summarize_thread_prompt.txt new file mode 100644 index 0000000000..f57644433b --- /dev/null +++ b/crates/agent/src/prompts/summarize_thread_prompt.txt @@ -0,0 +1,4 @@ +Generate a concise 3-7 word title for this conversation, omitting punctuation. +Go straight to the title, without any preamble and prefix like `Here's a concise suggestion:...` or `Title:`. +If the conversation is about a specific subject, include it in the title. +Be descriptive. DO NOT speak in the first person. diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs index c00bc60bb4..0b0308340c 100644 --- a/crates/agent/src/thread.rs +++ b/crates/agent/src/thread.rs @@ -1428,7 +1428,7 @@ impl Thread { messages: &mut Vec, cx: &App, ) { - const STALE_FILES_HEADER: &str = "These files changed since last read:"; + const STALE_FILES_HEADER: &str = include_str!("./prompts/stale_files_prompt_header.txt"); let mut stale_message = String::new(); @@ -1440,7 +1440,7 @@ impl Thread { }; if stale_message.is_empty() { - write!(&mut stale_message, "{}\n", STALE_FILES_HEADER).ok(); + write!(&mut stale_message, "{}\n", STALE_FILES_HEADER.trim()).ok(); } writeln!(&mut stale_message, "- {}", file.path().display()).ok(); @@ -1854,10 +1854,7 @@ impl Thread { return; } - let added_user_message = "Generate a concise 3-7 word title for this conversation, omitting punctuation. \ - Go straight to the title, without any preamble and prefix like `Here's a concise suggestion:...` or `Title:`. \ - If the conversation is about a specific subject, include it in the title. \ - Be descriptive. DO NOT speak in the first person."; + let added_user_message = include_str!("./prompts/summarize_thread_prompt.txt"); let request = self.to_summarize_request( &model.model, @@ -1958,12 +1955,7 @@ impl Thread { return; } - let added_user_message = "Generate a detailed summary of this conversation. Include:\n\ - 1. A brief overview of what was discussed\n\ - 2. Key facts or information discovered\n\ - 3. Outcomes or conclusions reached\n\ - 4. Any action items or next steps if any\n\ - Format it in Markdown with headings and bullet points."; + let added_user_message = include_str!("./prompts/summarize_thread_detailed_prompt.txt"); let request = self.to_summarize_request( &model, diff --git a/script/danger/dangerfile.ts b/script/danger/dangerfile.ts index 227e574823..56441bea20 100644 --- a/script/danger/dangerfile.ts +++ b/script/danger/dangerfile.ts @@ -1,4 +1,4 @@ -import { danger, message, warn } from "danger"; +import { danger, message, warn, fail } from "danger"; const { prHygiene } = require("danger-plugin-pr-hygiene"); prHygiene({ @@ -57,3 +57,39 @@ if (includesIssueUrl) { ].join("\n"), ); } + +const PROMPT_PATHS = [ + "assets/prompts/content_prompt.hbs", + "assets/prompts/terminal_assistant_prompt.hbs", + "crates/agent/src/prompts/stale_files_prompt_header.txt", + "crates/agent/src/prompts/summarize_thread_detailed_prompt.txt", + "crates/agent/src/prompts/summarize_thread_prompt.txt", + "crates/assistant_tools/src/templates/create_file_prompt.hbs", + "crates/assistant_tools/src/templates/edit_file_prompt.hbs", + "crates/git_ui/src/commit_message_prompt.txt", +]; + +const PROMPT_CHANGE_ATTESTATION = "I have ensured the LLM Worker works with these prompt changes."; + +const modifiedPrompts = danger.git.modified_files.filter((file) => + PROMPT_PATHS.some((promptPath) => file.includes(promptPath)), +); + +for (const promptPath of modifiedPrompts) { + if (body.includes(PROMPT_CHANGE_ATTESTATION)) { + message( + [ + `This PR contains changes to "${promptPath}".`, + "The author has attested the LLM Worker works with the changes to this prompt.", + ].join("\n"), + ); + } else { + fail( + [ + `Modifying the "${promptPath}" prompt may require corresponding changes in the LLM Worker.`, + "If you are ensure what this entails, talk to @maxdeviant or another AI team member.", + `Once you have made the changes—or determined that none are necessary—add "${PROMPT_CHANGE_ATTESTATION}" to the PR description.`, + ].join("\n"), + ); + } +}