Add Danger check for changes to eval fixtures (#33852)
Also, revert unintentional changes to fixtures. Changes to test fixtures are intentional and necessary. Release Notes: - N/A
This commit is contained in:
parent
12ab53b804
commit
cc0d8a411e
2 changed files with 41 additions and 188 deletions
|
@ -94,3 +94,24 @@ for (const promptPath of modifiedPrompts) {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
const FIXTURE_CHANGE_ATTESTATION = "Changes to test fixtures are intentional and necessary.";
|
||||
|
||||
const FIXTURES_PATHS = ["crates/assistant_tools/src/edit_agent/evals/fixtures"];
|
||||
|
||||
const modifiedFixtures = danger.git.modified_files.filter((file) =>
|
||||
FIXTURES_PATHS.some((fixturePath) => file.includes(fixturePath)),
|
||||
);
|
||||
|
||||
if (modifiedFixtures.length > 0) {
|
||||
if (!body.includes(FIXTURE_CHANGE_ATTESTATION)) {
|
||||
const modifiedFixturesStr = modifiedFixtures.map((path) => "`" + path + "`").join(", ");
|
||||
fail(
|
||||
[
|
||||
`This PR modifies eval or test fixtures (${modifiedFixturesStr}), which are typically expected to remain unchanged.`,
|
||||
"If these changes are intentional and required, please add the following attestation to your PR description: ",
|
||||
`"${FIXTURE_CHANGE_ATTESTATION}"`,
|
||||
].join("\n\n"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue