Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add eval for gpt-engineer

Browse source
This commit is contained in:
KCaverly 2023-09-12 21:27:35 -04:00
parent 0d14bbbf5b
commit d4fbe99052
1 changed files with 114 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

114
crates/semantic_index/eval/gpt-engineer.json Normal file
View file

@ -0,0 +1,114 @@
{
"repo": "https://github.com/AntonOsika/gpt-engineer.git",
"commit": "7735a6445bae3611c62f521e6464c67c957f87c2",
"assertions": [
{
"query": "How do I contribute to this project?",
"matches": [
".github/CONTRIBUTING.md:1",
"ROADMAP.md:48"
]
},
{
"query": "What version of the openai package is active?",
"matches": [
"pyproject.toml:14"
]
},
{
"query": "Ask user for clarification",
"matches": [
"gpt-engineer/steps.py:69"
]
},
{
"query": "generate tests for python code",
"matches": [
"gpt-engineer/steps.py:153"
]
},
{
"query": "get item from database based on key",
"matches": [
"gpt-engineer/db.py:42",
"gpt-engineer/db.py:68"
]
},
{
"query": "prompt user to select files",
"matches": [
"gpt-engineer/file_selector.py:171",
"gpt-engineer/file_selector.py:306",
"gpt-engineer/file_selector.py:289",
"gpt-engineer/file_selector.py:234"
]
},
{
"query": "send to rudderstack",
"matches": [
"gpt-engineer/collect.py:11",
"gpt-engineer/collect.py:38"
]
},
{
"query": "parse code blocks from chat messages",
"matches": [
"gpt-engineer/chat_to_files.py:10",
"docs/intro/chat_parsing.md:1"
]
},
{
"query": "how do I use the docker cli?",
"matches": [
"docker/README.md:1"
]
},
{
"query": "ask the user if the code ran successfully?",
"matches": [
"gpt-engineer/learning.py:54"
]
},
{
"query": "how is consent granted by the user?",
"matches": [
"gpt-engineer/learning.py:107",
"gpt-engineer/learning.py:130",
"gpt-engineer/learning.py:152"
]
},
{
"query": "what are all the different steps the agent can take?",
"matches": [
"docs/intro/steps_module.md:1",
"gpt-engineer/steps.py:391"
]
},
{
"query": "ask the user for clarification?",
"matches": [
"gpt-engineer/steps.py:69"
]
},
{
"query": "what models are available?",
"matches": [
"gpt-engineer/ai.py:315",
"gpt-engineer/ai.py:341",
"docs/open-models.md:1"
]
},
{
"query": "what is the current focus of the project?",
"matches": [
"ROADMAP.md:11"
]
},
{
"query": "does the agent know how to fix code?",
"matches": [
"gpt-engineer/steps.py:367"
]
}
]
}