Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

evals: Make LLMs configurable in edit_agent evals (#30813)

Browse source 
Release Notes:

- N/A
This commit is contained in:
Oleksiy Syvokon 2025-05-16 14:10:15 +03:00 • committed by GitHub
parent dcf7f714f7
commit 5112fcebeb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 46 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

29
crates/assistant_tools/src/edit_agent/evals.rs
View file

@ -15,7 +15,7 @@ use gpui::{AppContext, TestAppContext};
use indoc::{formatdoc, indoc};
use language_model::{
LanguageModelRegistry, LanguageModelRequestTool, LanguageModelToolResult,
LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId,
LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, SelectedModel,
};
use project::Project;
use rand::prelude::*;
@ -25,6 +25,7 @@ use std::{
cmp::Reverse,
fmt::{self, Display},
io::Write as _,
str::FromStr,
sync::mpsc,
};
use util::path;
@ -1216,7 +1217,7 @@ fn report_progress(evaluated_count: usize, failed_count: usize, iterations: usiz
passed_count as f64 / evaluated_count as f64
};
print!(
"\r\x1b[KEvaluated {}/{} ({:.2}%)",
"\r\x1b[KEvaluated {}/{} ({:.2}% passed)",
evaluated_count,
iterations,
passed_ratio * 100.0
@ -1255,13 +1256,21 @@ impl EditAgentTest {
fs.insert_tree("/root", json!({})).await;
let project = Project::test(fs.clone(), [path!("/root").as_ref()], cx).await;
let agent_model = SelectedModel::from_str(
&std::env::var("ZED_AGENT_MODEL")
.unwrap_or("anthropic/claude-3-7-sonnet-latest".into()),
)
.unwrap();
let judge_model = SelectedModel::from_str(
&std::env::var("ZED_JUDGE_MODEL")
.unwrap_or("anthropic/claude-3-7-sonnet-latest".into()),
)
.unwrap();
let (agent_model, judge_model) = cx
.update(|cx| {
cx.spawn(async move |cx| {
let agent_model =
Self::load_model("anthropic", "claude-3-7-sonnet-latest", cx).await;
let judge_model =
Self::load_model("anthropic", "claude-3-7-sonnet-latest", cx).await;
let agent_model = Self::load_model(&agent_model, cx).await;
let judge_model = Self::load_model(&judge_model, cx).await;
(agent_model.unwrap(), judge_model.unwrap())
})
})
@ -1276,15 +1285,17 @@ impl EditAgentTest {
}
async fn load_model(
provider: &str,
id: &str,
selected_model: &SelectedModel,
cx: &mut AsyncApp,
) -> Result<Arc<dyn LanguageModel>> {
let (provider, model) = cx.update(|cx| {
let models = LanguageModelRegistry::read_global(cx);
let model = models
.available_models(cx)
.find(|model| model.provider_id().0 == provider && model.id().0 == id)
.find(|model| {
model.provider_id() == selected_model.provider
&& model.id() == selected_model.model
})
.unwrap();
let provider = models.provider(&model.provider_id()).unwrap();
(provider, model)