Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make eval more resilient to bad input from LLM (#29703)

Browse source 
I saw a slice panic (for begin > end) in a debug build of the eval. This
should just be a failed assertion, not a panic that takes out the whole
eval run!

Release Notes:

- N/A
This commit is contained in:
Richard Feldman 2025-04-30 18:13:45 -04:00 • committed by GitHub
parent 92dd6b67c7
commit afeb3d4fd9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 51 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

92
crates/eval/src/examples/code_block_citations.rs
View file

@ -98,54 +98,64 @@ impl Example for CodeBlockCitations {
if let Some(content_len) = content_len {
// + 1 because there's a newline character after the citation.
let content =
&text[(citation.len() + 1)..content_len - (citation.len() + 1)];
let start_index = citation.len() + 1;
let end_index = content_len.saturating_sub(start_index);
// deindent (trim the start of each line) because sometimes the model
// chooses to deindent its code snippets for the sake of readability,
// which in markdown is not only reasonable but usually desirable.
cx.assert(
deindent(&buffer_text)
.trim()
.contains(deindent(&content).trim()),
"Code block content was found in file",
)
.ok();
if let Some(range) = path_range.range {
let start_line_index = range.start.line.saturating_sub(1);
let line_count =
range.end.line.saturating_sub(start_line_index);
let mut snippet = buffer_text
.lines()
.skip(start_line_index as usize)
.take(line_count as usize)
.collect::<Vec<&str>>()
.join("\n");
if let Some(start_col) = range.start.col {
snippet = snippet[start_col as usize..].to_string();
}
if let Some(end_col) = range.end.col {
let last_line = snippet.lines().last().unwrap();
snippet = snippet
[..snippet.len() - last_line.len() + end_col as usize]
.to_string();
}
if cx
.assert(
start_index <= end_index,
"Code block had a valid citation",
)
.is_ok()
{
let content = &text[start_index..end_index];
// deindent (trim the start of each line) because sometimes the model
// chooses to deindent its code snippets for the sake of readability,
// which in markdown is not only reasonable but usually desirable.
cx.assert_eq(
deindent(snippet.as_str()).trim(),
deindent(content).trim(),
format!(
"Code block was at {:?}-{:?}",
range.start, range.end
),
cx.assert(
deindent(&buffer_text)
.trim()
.contains(deindent(&content).trim()),
"Code block content was found in file",
)
.ok();
if let Some(range) = path_range.range {
let start_line_index = range.start.line.saturating_sub(1);
let line_count =
range.end.line.saturating_sub(start_line_index);
let mut snippet = buffer_text
.lines()
.skip(start_line_index as usize)
.take(line_count as usize)
.collect::<Vec<&str>>()
.join("\n");
if let Some(start_col) = range.start.col {
snippet = snippet[start_col as usize..].to_string();
}
if let Some(end_col) = range.end.col {
let last_line = snippet.lines().last().unwrap();
snippet = snippet[..snippet.len() - last_line.len()
+ end_col as usize]
.to_string();
}
// deindent (trim the start of each line) because sometimes the model
// chooses to deindent its code snippets for the sake of readability,
// which in markdown is not only reasonable but usually desirable.
cx.assert_eq(
deindent(snippet.as_str()).trim(),
deindent(content).trim(),
format!(
"Code block was at {:?}-{:?}",
range.start, range.end
),
)
.ok();
}
}
}
}