Add Danger check for changes to eval fixtures (#33852)

Also, revert unintentional changes to fixtures.

Changes to test fixtures are intentional and necessary.

Release Notes:

- N/A
This commit is contained in:
Oleksiy Syvokon 2025-07-03 17:27:49 +03:00 committed by GitHub
parent 12ab53b804
commit cc0d8a411e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 41 additions and 188 deletions

View file

@ -94,3 +94,24 @@ for (const promptPath of modifiedPrompts) {
);
}
}
const FIXTURE_CHANGE_ATTESTATION = "Changes to test fixtures are intentional and necessary.";
const FIXTURES_PATHS = ["crates/assistant_tools/src/edit_agent/evals/fixtures"];
const modifiedFixtures = danger.git.modified_files.filter((file) =>
FIXTURES_PATHS.some((fixturePath) => file.includes(fixturePath)),
);
if (modifiedFixtures.length > 0) {
if (!body.includes(FIXTURE_CHANGE_ATTESTATION)) {
const modifiedFixturesStr = modifiedFixtures.map((path) => "`" + path + "`").join(", ");
fail(
[
`This PR modifies eval or test fixtures (${modifiedFixturesStr}), which are typically expected to remain unchanged.`,
"If these changes are intentional and required, please add the following attestation to your PR description: ",
`"${FIXTURE_CHANGE_ATTESTATION}"`,
].join("\n\n"),
);
}
}