diff --git a/crates/assistant_tools/src/edit_agent/evals.rs b/crates/assistant_tools/src/edit_agent/evals.rs
index 023febbdae..d19cf922fc 100644
--- a/crates/assistant_tools/src/edit_agent/evals.rs
+++ b/crates/assistant_tools/src/edit_agent/evals.rs
@@ -1634,15 +1634,20 @@ impl EditAgentTest {
 }
 
 async fn retry_on_rate_limit<R>(mut request: impl AsyncFnMut() -> Result<R>) -> Result<R> {
+    let mut attempt = 0;
     loop {
+        attempt += 1;
         match request().await {
             Ok(result) => return Ok(result),
             Err(err) => match err.downcast::<LanguageModelCompletionError>() {
                 Ok(err) => match err {
                     LanguageModelCompletionError::RateLimit(duration) => {
-                        // Wait until after we are allowed to try again
-                        eprintln!("Rate limit exceeded. Waiting for {duration:?}...",);
-                        Timer::after(duration).await;
+                        // Wait for the duration supplied, with some jitter to avoid all requests being made at the same time.
+                        let jitter = duration.mul_f64(rand::thread_rng().gen_range(0.0..0.5));
+                        eprintln!(
+                            "Attempt #{attempt}: Rate limit exceeded. Retry after {duration:?} + jitter of {jitter:?}"
+                        );
+                        Timer::after(duration + jitter).await;
                         continue;
                     }
                     _ => return Err(err.into()),