diff --git a/crates/eval/src/eval.rs b/crates/eval/src/eval.rs index 975d547442..eb2c07e0dc 100644 --- a/crates/eval/src/eval.rs +++ b/crates/eval/src/eval.rs @@ -110,13 +110,15 @@ fn main() { continue; } - examples.push((example_path, example)); + println!("{}> Logging to {:?}", example.name, example.log_file_path); + + examples.push(example); } let mut repo_urls = HashSet::new(); let mut clone_tasks = Vec::new(); - for (_, example) in examples.iter() { + for example in examples.iter() { let repo_url = example.base.url.clone(); if repo_urls.insert(repo_url.clone()) { let repo_path = repo_path_for_url(&repo_url); @@ -149,25 +151,22 @@ fn main() { future::join_all(clone_tasks).await; - for (_, example) in examples.iter() { + for example in examples.iter() { example.setup().await?; } let tasks = examples .into_iter() - .map(|(example_path, example)| { + .map(|example| { let app_state = app_state.clone(); let model = model.clone(); cx.spawn(async move |cx| { - ( - example_path, - run_example(example, model, app_state, cx).await, - ) + (run_example(&example, model, app_state, cx).await, example) }) }) .collect::>(); - let results: Vec<(PathBuf, Result)> = future::join_all(tasks).await; + let results: Vec<(Result, Example)> = future::join_all(tasks).await; println!("\n\n"); println!("========================================"); @@ -177,11 +176,11 @@ fn main() { let mut judge_scores = Vec::new(); - for (example_path, result) in results { - let example_name = example_path.file_name().unwrap().to_string_lossy(); + for (result, example) in results { + println!("📜 {:<30}: {:?}", example.name, example.log_file_path); match result { Err(err) => { - println!("💥 {:<30}: {:?}", example_name, err); + println!("💥 {:<30}: {:?}", example.name, err); } Ok(judge_output) => { const SCORES: [&str; 6] = ["💀", "😭", "😔", "😐", "🙂", "🤩"]; @@ -189,7 +188,7 @@ fn main() { println!( "{} {:<30}: {}", SCORES[judge_output.score.min(5) as usize], - example_name, + example.name, judge_output.score, ); judge_scores.push(judge_output.score); @@ -212,7 +211,7 @@ fn main() { } async fn run_example( - mut example: Example, + example: &Example, model: Arc, app_state: Arc, cx: &mut AsyncApp, diff --git a/crates/eval/src/example.rs b/crates/eval/src/example.rs index 75be01c9d6..531b1ea275 100644 --- a/crates/eval/src/example.rs +++ b/crates/eval/src/example.rs @@ -58,6 +58,8 @@ pub struct Example { pub criteria: String, /// Markdown log file to append to pub log_file: Arc>, + /// Path to markdown log file + pub log_file_path: PathBuf, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -102,6 +104,7 @@ impl Example { prompt: fs::read_to_string(prompt_path.clone())?, criteria: fs::read_to_string(criteria_path.clone())?, log_file, + log_file_path, }) } @@ -400,7 +403,7 @@ impl Example { } pub async fn judge( - &mut self, + &self, model: Arc, repository_diff: String, cx: &AsyncApp,