Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add ndcg@k to evaluate metrics

Browse source
This commit is contained in:
KCaverly 2023-09-15 10:36:21 -04:00
parent 3a661c5977
commit 04bd107ada
2 changed files with 88 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

44
crates/semantic_index/eval/gpt-engineer.json
View file

@ -1,5 +1,5 @@
{
"repo": "https://github.com/AntonOsika/gpt-engineer.git",
"repo": "https://github.com/AntonOsika/gpt_engineer.git",
"commit": "7735a6445bae3611c62f521e6464c67c957f87c2",
"assertions": [
{
@ -12,48 +12,48 @@
{
"query": "What version of the openai package is active?",
"matches": [
"pyprojet.toml:14"
"pyproject.toml:14"
]
},
{
"query": "Ask user for clarification",
"matches": [
"gpt-engineer/steps.py:69"
"gpt_engineer/steps.py:69"
]
},
{
"query": "generate tests for python code",
"matches": [
"gpt-engineer/steps.py:153"
"gpt_engineer/steps.py:153"
]
},
{
"query": "get item from database based on key",
"matches": [
"gpt-engineer/db.py:42",
"gpt-engineer/db.py:68"
"gpt_engineer/db.py:42",
"gpt_engineer/db.py:68"
]
},
{
"query": "prompt user to select files",
"matches": [
"gpt-engineer/file_selector.py:171",
"gpt-engineer/file_selector.py:306",
"gpt-engineer/file_selector.py:289",
"gpt-engineer/file_selector.py:234"
"gpt_engineer/file_selector.py:171",
"gpt_engineer/file_selector.py:306",
"gpt_engineer/file_selector.py:289",
"gpt_engineer/file_selector.py:234"
]
},
{
"query": "send to rudderstack",
"matches": [
"gpt-engineer/collect.py:11",
"gpt-engineer/collect.py:38"
"gpt_engineer/collect.py:11",
"gpt_engineer/collect.py:38"
]
},
{
"query": "parse code blocks from chat messages",
"matches": [
"gpt-engineer/chat_to_files.py:10",
"gpt_engineer/chat_to_files.py:10",
"docs/intro/chat_parsing.md:1"
]
},
@ -66,35 +66,35 @@
{
"query": "ask the user if the code ran successfully?",
"matches": [
"gpt-engineer/learning.py:54"
"gpt_engineer/learning.py:54"
]
},
{
"query": "how is consent granted by the user?",
"matches": [
"gpt-engineer/learning.py:107",
"gpt-engineer/learning.py:130",
"gpt-engineer/learning.py:152"
"gpt_engineer/learning.py:107",
"gpt_engineer/learning.py:130",
"gpt_engineer/learning.py:152"
]
},
{
"query": "what are all the different steps the agent can take?",
"matches": [
"docs/intro/steps_module.md:1",
"gpt-engineer/steps.py:391"
"gpt_engineer/steps.py:391"
]
},
{
"query": "ask the user for clarification?",
"matches": [
"gpt-engineer/steps.py:69"
"gpt_engineer/steps.py:69"
]
},
{
"query": "what models are available?",
"matches": [
"gpt-engineer/ai.py:315",
"gpt-engineer/ai.py:341",
"gpt_engineer/ai.py:315",
"gpt_engineer/ai.py:341",
"docs/open-models.md:1"
]
},
@ -107,7 +107,7 @@
{
"query": "does the agent know how to fix code?",
"matches": [
"gpt-engineer/steps.py:367"
"gpt_engineer/steps.py:367"
]
}
]