eval: Add eval unit tests as a CI job (#32152)
We run the unit evals once a day in the middle of the night, and trigger a Slack post if it fails. Release Notes: - N/A --------- Co-authored-by: Oleksiy Syvokon <oleksiy.syvokon@gmail.com>
This commit is contained in:
parent
fa9da6ad5b
commit
dda614091a
2 changed files with 86 additions and 1 deletions
|
@ -1351,7 +1351,7 @@ fn eval(iterations: usize, expected_pass_ratio: f32, mut eval: EvalInput) {
|
|||
|
||||
let mismatched_tag_ratio =
|
||||
cumulative_parser_metrics.mismatched_tags as f32 / cumulative_parser_metrics.tags as f32;
|
||||
if mismatched_tag_ratio > 0.05 {
|
||||
if mismatched_tag_ratio > 0.10 {
|
||||
for eval_output in eval_outputs {
|
||||
println!("{}", eval_output);
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue