agent: Add git commit ID to the eval telemetry data (#28895)
Release Notes: - N/A
This commit is contained in:
parent
10507f9a4c
commit
8de53bd89f
1 changed files with 18 additions and 6 deletions
|
@ -271,12 +271,12 @@ fn main() {
|
|||
match judge_result {
|
||||
Ok(judge_output) => {
|
||||
const SCORES: [&str; 6] = ["💀", "😭", "😔", "😐", "🙂", "🤩"];
|
||||
let score: u32 = judge_output.score;
|
||||
let score_index = (score.min(5)) as usize;
|
||||
|
||||
println!(
|
||||
"{} {}{}",
|
||||
SCORES[judge_output.score.min(5) as usize],
|
||||
example.log_prefix,
|
||||
judge_output.score,
|
||||
SCORES[score_index], example.log_prefix, judge_output.score,
|
||||
);
|
||||
judge_scores.push(judge_output.score);
|
||||
}
|
||||
|
@ -304,7 +304,6 @@ fn main() {
|
|||
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
|
||||
// Flush telemetry events before exiting
|
||||
app_state.client.telemetry().flush_events();
|
||||
|
||||
cx.update(|cx| cx.quit())
|
||||
|
@ -330,7 +329,6 @@ async fn run_example(
|
|||
for round in 0..judge_repetitions {
|
||||
let judge_result = example.judge(model.clone(), diff.clone(), round, cx).await;
|
||||
|
||||
// Log telemetry for this judge result
|
||||
if let Ok(judge_output) = &judge_result {
|
||||
let cohort_id = example
|
||||
.output_file_path
|
||||
|
@ -339,6 +337,9 @@ async fn run_example(
|
|||
.map(|name| name.to_string_lossy().to_string())
|
||||
.unwrap_or(chrono::Local::now().format("%Y-%m-%d_%H-%M-%S").to_string());
|
||||
|
||||
let path = std::path::Path::new(".");
|
||||
let commit_id = get_current_commit_id(path).await.unwrap_or_default();
|
||||
|
||||
telemetry::event!(
|
||||
"Agent Eval Completed",
|
||||
cohort_id = cohort_id,
|
||||
|
@ -353,7 +354,8 @@ async fn run_example(
|
|||
model_provider = model.provider_id().to_string(),
|
||||
repository_url = example.base.url.clone(),
|
||||
repository_revision = example.base.revision.clone(),
|
||||
diagnostics_summary = run_output.diagnostics
|
||||
diagnostics_summary = run_output.diagnostics,
|
||||
commit_id = commit_id
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -524,3 +526,13 @@ pub fn authenticate_model_provider(
|
|||
let model_provider = model_registry.provider(&provider_id).unwrap();
|
||||
model_provider.authenticate(cx)
|
||||
}
|
||||
|
||||
pub async fn get_current_commit_id(repo_path: &Path) -> Option<String> {
|
||||
(run_git(repo_path, &["rev-parse", "HEAD"]).await).ok()
|
||||
}
|
||||
|
||||
pub fn get_current_commit_id_sync(repo_path: &Path) -> String {
|
||||
futures::executor::block_on(async {
|
||||
get_current_commit_id(repo_path).await.unwrap_or_default()
|
||||
})
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue