Make the default repetitions be 8, and concurrency 4 (#29576)
This is based on having observed that there is a lot of variation between runs on `n=1` and `n=3`. * With `n=8` two runs on the same branch give answers that seem close enough to be reasonably consistent. * With higher concurrency, trying to run this many repetitions seems to lead language servers to time out a lot, causing evals to fail. Release Notes: - N/A
This commit is contained in:
parent
399eced884
commit
04c68dc0cf
2 changed files with 3 additions and 3 deletions
|
@ -52,10 +52,10 @@ struct Args {
|
|||
#[arg(long, value_delimiter = ',', default_value = "rs,ts")]
|
||||
languages: Vec<String>,
|
||||
/// How many times to run each example.
|
||||
#[arg(long, default_value = "1")]
|
||||
#[arg(long, default_value = "8")]
|
||||
repetitions: usize,
|
||||
/// Maximum number of examples to run concurrently.
|
||||
#[arg(long, default_value = "10")]
|
||||
#[arg(long, default_value = "4")]
|
||||
concurrency: usize,
|
||||
}
|
||||
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue