Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

eval: Add jitter to retry attempts (#32542)

Browse source 
Adds some jitter to avoid the issue that all requests will retry at
roughly the same time in eval where we have a lot of concurrent
requests.

Release Notes:

- N/A
This commit is contained in:
Ben Brandt 2025-06-11 14:56:23 +02:00 • committed by GitHub
parent 6c4728f00f
commit 2ecc24eb26
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
crates/assistant_tools/src/edit_agent/evals.rs
View file

@ -1634,15 +1634,20 @@ impl EditAgentTest {
} }
async fn retry_on_rate_limit<R>(mut request: impl AsyncFnMut() -> Result<R>) -> Result<R> { async fn retry_on_rate_limit<R>(mut request: impl AsyncFnMut() -> Result<R>) -> Result<R> {
let mut attempt = 0;
loop { loop {
attempt += 1;
match request().await { match request().await {
Ok(result) => return Ok(result), Ok(result) => return Ok(result),
Err(err) => match err.downcast::<LanguageModelCompletionError>() { Err(err) => match err.downcast::<LanguageModelCompletionError>() {
Ok(err) => match err { Ok(err) => match err {
LanguageModelCompletionError::RateLimit(duration) => { LanguageModelCompletionError::RateLimit(duration) => {
// Wait until after we are allowed to try again // Wait for the duration supplied, with some jitter to avoid all requests being made at the same time.
eprintln!("Rate limit exceeded. Waiting for {duration:?}...",); let jitter = duration.mul_f64(rand::thread_rng().gen_range(0.0..0.5));
Timer::after(duration).await; eprintln!(
"Attempt #{attempt}: Rate limit exceeded. Retry after {duration:?} + jitter of {jitter:?}"
);
Timer::after(duration + jitter).await;
continue; continue;
} }
_ => return Err(err.into()), _ => return Err(err.into()),