eval: Add eval unit tests as a CI job (#32152)

We run the unit evals once a day in the middle of the night, and trigger
a Slack post if it fails.


Release Notes:

- N/A

---------

Co-authored-by: Oleksiy Syvokon <oleksiy.syvokon@gmail.com>
This commit is contained in:
Ben Brandt 2025-06-05 15:16:27 +02:00 committed by GitHub
parent fa9da6ad5b
commit dda614091a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 86 additions and 1 deletions

View file

@ -1351,7 +1351,7 @@ fn eval(iterations: usize, expected_pass_ratio: f32, mut eval: EvalInput) {
let mismatched_tag_ratio =
cumulative_parser_metrics.mismatched_tags as f32 / cumulative_parser_metrics.tags as f32;
if mismatched_tag_ratio > 0.05 {
if mismatched_tag_ratio > 0.10 {
for eval_output in eval_outputs {
println!("{}", eval_output);
}