Code block evals (#29619)
Add a targeted eval for code block formatting, and revise the system prompt accordingly. ### Eval before, n=8 <img width="728" alt="eval before" src="https://github.com/user-attachments/assets/552b6146-3d26-4eaa-86f9-9fc36c0cadf2" /> ### Eval after prompt change, n=8 (excluding the new evals, so just testing the prompt change) <img width="717" alt="eval after" src="https://github.com/user-attachments/assets/c78c7a54-4c65-470c-b135-8691584cd73e" /> Release Notes: - N/A
This commit is contained in:
parent
2508e491d5
commit
d7004030b3
10 changed files with 536 additions and 134 deletions
|
@ -1,6 +1,8 @@
|
|||
pub mod parser;
|
||||
mod path_range;
|
||||
|
||||
pub use path_range::{LineCol, PathWithRange};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashSet;
|
||||
use std::iter;
|
||||
|
|
|
@ -32,6 +32,20 @@ impl LineCol {
|
|||
}
|
||||
|
||||
impl PathWithRange {
|
||||
// Note: We could try out this as an alternative, and see how it does on evals.
|
||||
//
|
||||
// The closest to a standard way of including a filename is this:
|
||||
// ```rust filename="path/to/file.rs#42:43"
|
||||
// ```
|
||||
//
|
||||
// or, alternatively,
|
||||
// ```rust filename="path/to/file.rs" lines="42:43"
|
||||
// ```
|
||||
//
|
||||
// Examples where it's used this way:
|
||||
// - https://mdxjs.com/guides/syntax-highlighting/#syntax-highlighting-with-the-meta-field
|
||||
// - https://docusaurus.io/docs/markdown-features/code-blocks
|
||||
// - https://spec.commonmark.org/0.31.2/#example-143
|
||||
pub fn new(str: impl AsRef<str>) -> Self {
|
||||
let str = str.as_ref();
|
||||
// Sometimes the model will include a language at the start,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue