diff --git a/crates/eval/src/eval.rs b/crates/eval/src/eval.rs index ca6b11be0a..96fb7e7253 100644 --- a/crates/eval/src/eval.rs +++ b/crates/eval/src/eval.rs @@ -271,12 +271,12 @@ fn main() { match judge_result { Ok(judge_output) => { const SCORES: [&str; 6] = ["💀", "😭", "😔", "😐", "🙂", "🤩"]; + let score: u32 = judge_output.score; + let score_index = (score.min(5)) as usize; println!( "{} {}{}", - SCORES[judge_output.score.min(5) as usize], - example.log_prefix, - judge_output.score, + SCORES[score_index], example.log_prefix, judge_output.score, ); judge_scores.push(judge_output.score); } @@ -304,7 +304,6 @@ fn main() { std::thread::sleep(std::time::Duration::from_secs(2)); - // Flush telemetry events before exiting app_state.client.telemetry().flush_events(); cx.update(|cx| cx.quit()) @@ -330,7 +329,6 @@ async fn run_example( for round in 0..judge_repetitions { let judge_result = example.judge(model.clone(), diff.clone(), round, cx).await; - // Log telemetry for this judge result if let Ok(judge_output) = &judge_result { let cohort_id = example .output_file_path @@ -339,6 +337,9 @@ async fn run_example( .map(|name| name.to_string_lossy().to_string()) .unwrap_or(chrono::Local::now().format("%Y-%m-%d_%H-%M-%S").to_string()); + let path = std::path::Path::new("."); + let commit_id = get_current_commit_id(path).await.unwrap_or_default(); + telemetry::event!( "Agent Eval Completed", cohort_id = cohort_id, @@ -353,7 +354,8 @@ async fn run_example( model_provider = model.provider_id().to_string(), repository_url = example.base.url.clone(), repository_revision = example.base.revision.clone(), - diagnostics_summary = run_output.diagnostics + diagnostics_summary = run_output.diagnostics, + commit_id = commit_id ); } @@ -524,3 +526,13 @@ pub fn authenticate_model_provider( let model_provider = model_registry.provider(&provider_id).unwrap(); model_provider.authenticate(cx) } + +pub async fn get_current_commit_id(repo_path: &Path) -> Option { + (run_git(repo_path, &["rev-parse", "HEAD"]).await).ok() +} + +pub fn get_current_commit_id_sync(repo_path: &Path) -> String { + futures::executor::block_on(async { + get_current_commit_id(repo_path).await.unwrap_or_default() + }) +}