diff --git a/crates/eval/src/example.rs b/crates/eval/src/example.rs index cafc5d996f..fa5e95807e 100644 --- a/crates/eval/src/example.rs +++ b/crates/eval/src/example.rs @@ -49,6 +49,7 @@ pub struct ExampleMetadata { pub max_assertions: Option, pub profile_id: AgentProfileId, pub existing_thread_json: Option, + pub max_turns: Option, } #[derive(Clone, Debug)] diff --git a/crates/eval/src/examples/add_arg_to_trait_method.rs b/crates/eval/src/examples/add_arg_to_trait_method.rs index b9f306f841..9c538f9260 100644 --- a/crates/eval/src/examples/add_arg_to_trait_method.rs +++ b/crates/eval/src/examples/add_arg_to_trait_method.rs @@ -22,6 +22,7 @@ impl Example for AddArgToTraitMethod { max_assertions: None, profile_id: AgentProfileId::default(), existing_thread_json: None, + max_turns: None, } } diff --git a/crates/eval/src/examples/code_block_citations.rs b/crates/eval/src/examples/code_block_citations.rs index f0c2074ce5..2239ccdfdd 100644 --- a/crates/eval/src/examples/code_block_citations.rs +++ b/crates/eval/src/examples/code_block_citations.rs @@ -23,6 +23,7 @@ impl Example for CodeBlockCitations { max_assertions: None, profile_id: AgentProfileId::default(), existing_thread_json: None, + max_turns: None, } } diff --git a/crates/eval/src/examples/comment_translation.rs b/crates/eval/src/examples/comment_translation.rs index 3a4999bc85..b6c9f7376f 100644 --- a/crates/eval/src/examples/comment_translation.rs +++ b/crates/eval/src/examples/comment_translation.rs @@ -17,6 +17,7 @@ impl Example for CommentTranslation { max_assertions: Some(1), profile_id: AgentProfileId::default(), existing_thread_json: None, + max_turns: None, } } diff --git a/crates/eval/src/examples/file_search.rs b/crates/eval/src/examples/file_search.rs index 9056326db9..f1a482a41a 100644 --- a/crates/eval/src/examples/file_search.rs +++ b/crates/eval/src/examples/file_search.rs @@ -19,6 +19,7 @@ impl Example for FileSearchExample { max_assertions: Some(3), profile_id: AgentProfileId::default(), existing_thread_json: None, + max_turns: None, } } diff --git a/crates/eval/src/examples/mod.rs b/crates/eval/src/examples/mod.rs index edf3265186..5968ee2fd0 100644 --- a/crates/eval/src/examples/mod.rs +++ b/crates/eval/src/examples/mod.rs @@ -82,6 +82,7 @@ impl DeclarativeExample { max_assertions: None, profile_id, existing_thread_json, + max_turns: base.max_turns, }; Ok(DeclarativeExample { @@ -124,6 +125,8 @@ pub struct ExampleToml { pub thread_assertions: BTreeMap, #[serde(default)] pub existing_thread_path: Option, + #[serde(default)] + pub max_turns: Option, } #[async_trait(?Send)] @@ -134,7 +137,8 @@ impl Example for DeclarativeExample { async fn conversation(&self, cx: &mut ExampleContext) -> Result<()> { cx.push_user_message(&self.prompt); - let _ = cx.run_to_end().await; + let max_turns = self.metadata.max_turns.unwrap_or(1000); + let _ = cx.run_turns(max_turns).await; Ok(()) } diff --git a/crates/eval/src/examples/overwrite_file.rs b/crates/eval/src/examples/overwrite_file.rs index 57c83a40f7..df0b75294c 100644 --- a/crates/eval/src/examples/overwrite_file.rs +++ b/crates/eval/src/examples/overwrite_file.rs @@ -31,6 +31,7 @@ impl Example for FileOverwriteExample { max_assertions: Some(1), profile_id: AgentProfileId::default(), existing_thread_json: Some(thread_json.to_string()), + max_turns: None, } } diff --git a/crates/eval/src/examples/planets.rs b/crates/eval/src/examples/planets.rs index 9363c4ac9a..f3a69332d2 100644 --- a/crates/eval/src/examples/planets.rs +++ b/crates/eval/src/examples/planets.rs @@ -19,6 +19,7 @@ impl Example for Planets { max_assertions: None, profile_id: AgentProfileId::default(), existing_thread_json: None, + max_turns: None, } }