Remove edit action markers literals from source (#27778)

Edit action markers look like git conflicts and can trip up tooling used
to resolve git conflicts. This PR creates them programmatically so that
they don't appear in source code.

Release Notes:

- N/A
This commit is contained in:
Agus Zubiaga 2025-03-31 10:48:35 -03:00 committed by GitHub
parent 9b40770e9f
commit 9b44bacc28
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 203 additions and 147 deletions

View file

@ -59,6 +59,20 @@ enum State {
CloseFence,
}
/// used to avoid having source code that looks like git-conflict markers
macro_rules! marker_sym {
($char:expr) => {
concat!($char, $char, $char, $char, $char, $char, $char)
};
}
const SEARCH_MARKER: &str = concat!(marker_sym!('<'), " SEARCH");
const DIVIDER: &str = marker_sym!('=');
const NL_DIVIDER: &str = concat!("\n", marker_sym!('='));
const REPLACE_MARKER: &str = concat!(marker_sym!('>'), " REPLACE");
const NL_REPLACE_MARKER: &str = concat!("\n", marker_sym!('>'), " REPLACE");
const FENCE: &str = "```";
impl EditActionParser {
/// Creates a new `EditActionParser`
pub fn new() -> Self {
@ -87,13 +101,6 @@ impl EditActionParser {
pub fn parse_chunk(&mut self, input: &str) -> Vec<(EditAction, String)> {
use State::*;
const FENCE: &[u8] = b"```";
const SEARCH_MARKER: &[u8] = b"<<<<<<< SEARCH";
const DIVIDER: &[u8] = b"=======";
const NL_DIVIDER: &[u8] = b"\n=======";
const REPLACE_MARKER: &[u8] = b">>>>>>> REPLACE";
const NL_REPLACE_MARKER: &[u8] = b"\n>>>>>>> REPLACE";
let mut actions = Vec::new();
for byte in input.bytes() {
@ -227,7 +234,7 @@ impl EditActionParser {
self.to_state(State::Default);
}
fn expect_marker(&mut self, byte: u8, marker: &'static [u8], trailing_newline: bool) -> bool {
fn expect_marker(&mut self, byte: u8, marker: &'static str, trailing_newline: bool) -> bool {
match self.match_marker(byte, marker, trailing_newline) {
MarkerMatch::Complete => true,
MarkerMatch::Partial => false,
@ -245,7 +252,7 @@ impl EditActionParser {
}
}
fn extend_block_range(&mut self, byte: u8, marker: &[u8], nl_marker: &[u8]) -> bool {
fn extend_block_range(&mut self, byte: u8, marker: &str, nl_marker: &str) -> bool {
let marker = if self.block_range.is_empty() {
// do not require another newline if block is empty
marker
@ -287,7 +294,7 @@ impl EditActionParser {
}
}
fn match_marker(&mut self, byte: u8, marker: &[u8], trailing_newline: bool) -> MarkerMatch {
fn match_marker(&mut self, byte: u8, marker: &str, trailing_newline: bool) -> MarkerMatch {
if trailing_newline && self.marker_ix >= marker.len() {
if byte == b'\n' {
MarkerMatch::Complete
@ -296,7 +303,7 @@ impl EditActionParser {
} else {
MarkerMatch::None
}
} else if byte == marker[self.marker_ix] {
} else if byte == marker.as_bytes()[self.marker_ix] {
self.marker_ix += 1;
if self.marker_ix < marker.len() || trailing_newline {
@ -321,7 +328,7 @@ enum MarkerMatch {
pub struct ParseError {
line: usize,
column: usize,
expected: &'static [u8],
expected: &'static str,
found: u8,
}
@ -330,10 +337,7 @@ impl std::fmt::Display for ParseError {
write!(
f,
"input:{}:{}: Expected marker {:?}, found {:?}",
self.line,
self.column,
String::from_utf8_lossy(self.expected),
self.found as char
self.line, self.column, self.expected, self.found as char
)
}
}
@ -344,20 +348,26 @@ mod tests {
use rand::prelude::*;
use util::line_endings;
const WRONG_MARKER: &str = concat!(marker_sym!('<'), " WRONG_MARKER");
#[test]
fn test_simple_edit_action() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```
<<<<<<< SEARCH
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 1);
@ -373,18 +383,22 @@ fn replacement() {}
#[test]
fn test_with_language_tag() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```rust
<<<<<<< SEARCH
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 1);
@ -400,22 +414,26 @@ fn replacement() {}
#[test]
fn test_with_surrounding_text() {
let input = r#"Here's a modification I'd like to make to the file:
// Construct test input using format with multiline string literals
let input = format!(
r#"Here's a modification I'd like to make to the file:
src/main.rs
```rust
<<<<<<< SEARCH
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
This change makes the function better.
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 1);
@ -431,29 +449,33 @@ This change makes the function better.
#[test]
fn test_multiple_edit_actions() {
let input = r#"First change:
// Construct test input using format with multiline string literals
let input = format!(
r#"First change:
src/main.rs
```
<<<<<<< SEARCH
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
Second change:
src/utils.rs
```rust
<<<<<<< SEARCH
fn old_util() -> bool { false }
=======
fn new_util() -> bool { true }
>>>>>>> REPLACE
{}
fn old_util() -> bool {{ false }}
{}
fn new_util() -> bool {{ true }}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER, SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 2);
@ -480,32 +502,36 @@ fn new_util() -> bool { true }
#[test]
fn test_multiline() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```rust
<<<<<<< SEARCH
fn original() {
{}
fn original() {{
println!("This is the original function");
let x = 42;
if x > 0 {
if x > 0 {{
println!("Positive number");
}
}
=======
fn replacement() {
}}
}}
{}
fn replacement() {{
println!("This is the replacement function");
let x = 100;
if x > 50 {
if x > 50 {{
println!("Large number");
} else {
}} else {{
println!("Small number");
}
}
>>>>>>> REPLACE
}}
}}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 1);
@ -523,21 +549,25 @@ fn replacement() {
#[test]
fn test_write_action() {
let input = r#"Create a new main.rs file:
// Construct test input using format with multiline string literals
let input = format!(
r#"Create a new main.rs file:
src/main.rs
```rust
<<<<<<< SEARCH
=======
fn new_function() {
{}
{}
fn new_function() {{
println!("This function is being added");
}
>>>>>>> REPLACE
}}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_no_errors(&parser);
assert_eq!(actions.len(), 1);
@ -553,16 +583,20 @@ fn new_function() {
#[test]
fn test_empty_replace() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```rust
<<<<<<< SEARCH
fn this_will_be_deleted() {
{}
fn this_will_be_deleted() {{
println!("Deleting this function");
}
=======
>>>>>>> REPLACE
}}
{}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(&input);
@ -597,16 +631,20 @@ fn this_will_be_deleted() {
#[test]
fn test_empty_both() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```rust
<<<<<<< SEARCH
=======
>>>>>>> REPLACE
{}
{}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_eq!(actions.len(), 1);
assert_eq!(
@ -621,31 +659,24 @@ fn this_will_be_deleted() {
#[test]
fn test_resumability() {
let input_part1 = r#"src/main.rs
```rust
<<<<<<< SEARCH
fn ori"#;
// Construct test input using format with multiline string literals
let input_part1 = format!("src/main.rs\n```rust\n{}\nfn ori", SEARCH_MARKER);
let input_part2 = r#"ginal() {}
=======
fn replacement() {}"#;
let input_part2 = format!("ginal() {{}}\n{}\nfn replacement() {{}}", DIVIDER);
let input_part3 = r#"
>>>>>>> REPLACE
```
"#;
let input_part3 = format!("\n{}\n```\n", REPLACE_MARKER);
let mut parser = EditActionParser::new();
let actions1 = parser.parse_chunk(input_part1);
let actions1 = parser.parse_chunk(&input_part1);
assert_no_errors(&parser);
assert_eq!(actions1.len(), 0);
let actions2 = parser.parse_chunk(input_part2);
let actions2 = parser.parse_chunk(&input_part2);
// No actions should be complete yet
assert_no_errors(&parser);
assert_eq!(actions2.len(), 0);
let actions3 = parser.parse_chunk(input_part3);
let actions3 = parser.parse_chunk(&input_part3);
// The third chunk should complete the action
assert_no_errors(&parser);
assert_eq!(actions3.len(), 1);
@ -663,18 +694,17 @@ fn replacement() {}"#;
#[test]
fn test_parser_state_preservation() {
let mut parser = EditActionParser::new();
let actions1 = parser.parse_chunk("src/main.rs\n```rust\n<<<<<<< SEARCH\n");
let first_chunk = format!("src/main.rs\n```rust\n{}\n", SEARCH_MARKER);
let actions1 = parser.parse_chunk(&first_chunk);
// Check parser is in the correct state
assert_no_errors(&parser);
assert_eq!(parser.state, State::SearchBlock);
assert_eq!(
parser.action_source,
b"src/main.rs\n```rust\n<<<<<<< SEARCH\n"
);
assert_eq!(parser.action_source, first_chunk.as_bytes());
// Continue parsing
let actions2 = parser.parse_chunk("original code\n=======\n");
let second_chunk = format!("original code\n{}\n", DIVIDER);
let actions2 = parser.parse_chunk(&second_chunk);
assert_no_errors(&parser);
assert_eq!(parser.state, State::ReplaceBlock);
@ -683,7 +713,8 @@ fn replacement() {}"#;
b"original code"
);
let actions3 = parser.parse_chunk("replacement code\n>>>>>>> REPLACE\n```\n");
let third_chunk = format!("replacement code\n{}\n```\n", REPLACE_MARKER);
let actions3 = parser.parse_chunk(&third_chunk);
// After complete parsing, state should reset
assert_no_errors(&parser);
@ -699,18 +730,21 @@ fn replacement() {}"#;
#[test]
fn test_invalid_search_marker() {
let input = r#"src/main.rs
let input = format!(
r#"src/main.rs
```rust
<<<<<<< WRONG_MARKER
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
"#;
"#,
WRONG_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
assert_eq!(actions.len(), 0);
assert_eq!(parser.errors().len(), 1);
@ -718,33 +752,40 @@ fn replacement() {}
assert_eq!(
error.to_string(),
"input:3:9: Expected marker \"<<<<<<< SEARCH\", found 'W'"
format!(
"input:3:9: Expected marker \"{}\", found 'W'",
SEARCH_MARKER
)
);
}
#[test]
fn test_missing_closing_fence() {
let input = r#"src/main.rs
// Construct test input using format with multiline string literals
let input = format!(
r#"src/main.rs
```rust
<<<<<<< SEARCH
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
<!-- Missing closing fence -->
src/utils.rs
```rust
<<<<<<< SEARCH
fn utils_func() {}
=======
fn new_utils_func() {}
>>>>>>> REPLACE
{}
fn utils_func() {{}}
{}
fn new_utils_func() {{}}
{}
```
"#;
"#,
SEARCH_MARKER, DIVIDER, REPLACE_MARKER, SEARCH_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
let actions = parser.parse_chunk(input);
let actions = parser.parse_chunk(&input);
// Only the second block should be parsed
assert_eq!(actions.len(), 1);
@ -851,38 +892,53 @@ fn new_utils_func() {}
// The system prompt includes some text that would produce errors
assert_eq!(
errors[0].to_string(),
"input:102:1: Expected marker \"<<<<<<< SEARCH\", found '3'"
format!(
"input:102:1: Expected marker \"{}\", found '3'",
SEARCH_MARKER
)
);
#[cfg(not(windows))]
assert_eq!(
errors[1].to_string(),
"input:109:0: Expected marker \"<<<<<<< SEARCH\", found '\\n'"
format!(
"input:109:0: Expected marker \"{}\", found '\\n'",
SEARCH_MARKER
)
);
#[cfg(windows)]
assert_eq!(
errors[1].to_string(),
"input:108:1: Expected marker \"<<<<<<< SEARCH\", found '\\r'"
format!(
"input:108:1: Expected marker \"{}\", found '\\r'",
SEARCH_MARKER
)
);
}
#[test]
fn test_print_error() {
let input = r#"src/main.rs
let input = format!(
r#"src/main.rs
```rust
<<<<<<< WRONG_MARKER
fn original() {}
=======
fn replacement() {}
>>>>>>> REPLACE
{}
fn original() {{}}
{}
fn replacement() {{}}
{}
```
"#;
"#,
WRONG_MARKER, DIVIDER, REPLACE_MARKER
);
let mut parser = EditActionParser::new();
parser.parse_chunk(input);
parser.parse_chunk(&input);
assert_eq!(parser.errors().len(), 1);
let error = &parser.errors()[0];
let expected_error = r#"input:3:9: Expected marker "<<<<<<< SEARCH", found 'W'"#;
let expected_error = format!(
r#"input:3:9: Expected marker "{}", found 'W'"#,
SEARCH_MARKER
);
assert_eq!(format!("{}", error), expected_error);
}

View file

@ -65,7 +65,7 @@ pub struct FindReplaceFileToolInput {
/// <example>
/// If a file contains this code:
///
/// ```rust
/// ```ignore
/// fn check_user_permissions(user_id: &str) -> Result<bool> {
/// // Check if user exists first
/// let user = database.find_user(user_id)?;
@ -83,7 +83,7 @@ pub struct FindReplaceFileToolInput {
/// Your find string should include at least 3 lines of context before and after the part
/// you want to change:
///
/// ```
/// ```ignore
/// fn check_user_permissions(user_id: &str) -> Result<bool> {
/// // Check if user exists first
/// let user = database.find_user(user_id)?;
@ -100,7 +100,7 @@ pub struct FindReplaceFileToolInput {
///
/// And your replace string might look like:
///
/// ```
/// ```ignore
/// fn check_user_permissions(user_id: &str) -> Result<bool> {
/// // Check if user exists first
/// let user = database.find_user(user_id)?;