Add an eval
binary that evaluates our semantic index against CodeSearchNet (#17375)
This PR is the beginning of an evaluation framework for our AI features. Right now, we're evaluating our semantic search feature against the [CodeSearchNet](https://github.com/github/CodeSearchNet) code search dataset. This dataset is very limited (for the most part, only 1 known good search result per repo) but it has surfaced some problems with our search already. Release Notes: - N/A --------- Co-authored-by: Jason <jason@zed.dev> Co-authored-by: Jason Mancuso <7891333+jvmncs@users.noreply.github.com> Co-authored-by: Nathan <nathan@zed.dev> Co-authored-by: Richard <richard@zed.dev>
This commit is contained in:
parent
06a13c2983
commit
d3d3a093b4
14 changed files with 881 additions and 144 deletions
17
.github/workflows/ci.yml
vendored
17
.github/workflows/ci.yml
vendored
|
@ -101,7 +101,7 @@ jobs:
|
|||
timeout-minutes: 60
|
||||
name: (Linux) Run Clippy and tests
|
||||
runs-on:
|
||||
- hosted-linux-x86-1
|
||||
- buildjet-16vcpu-ubuntu-2204
|
||||
steps:
|
||||
- name: Add Rust to the PATH
|
||||
run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
||||
|
@ -111,6 +111,11 @@ jobs:
|
|||
with:
|
||||
clean: false
|
||||
|
||||
- name: Cache dependencies
|
||||
uses: swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2
|
||||
with:
|
||||
save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Install Linux dependencies
|
||||
run: ./script/linux
|
||||
|
||||
|
@ -264,7 +269,7 @@ jobs:
|
|||
timeout-minutes: 60
|
||||
name: Create a Linux bundle
|
||||
runs-on:
|
||||
- hosted-linux-x86-1
|
||||
- buildjet-16vcpu-ubuntu-2204
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') || contains(github.event.pull_request.labels.*.name, 'run-bundling') }}
|
||||
needs: [linux_tests]
|
||||
env:
|
||||
|
@ -279,9 +284,6 @@ jobs:
|
|||
- name: Install Linux dependencies
|
||||
run: ./script/linux
|
||||
|
||||
- name: Limit target directory size
|
||||
run: script/clear-target-dir-if-larger-than 100
|
||||
|
||||
- name: Determine version and release channel
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
run: |
|
||||
|
@ -335,7 +337,7 @@ jobs:
|
|||
timeout-minutes: 60
|
||||
name: Create arm64 Linux bundle
|
||||
runs-on:
|
||||
- hosted-linux-arm-1
|
||||
- buildjet-16vcpu-ubuntu-2204-arm
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') || contains(github.event.pull_request.labels.*.name, 'run-bundling') }}
|
||||
needs: [linux_tests]
|
||||
env:
|
||||
|
@ -350,9 +352,6 @@ jobs:
|
|||
- name: Install Linux dependencies
|
||||
run: ./script/linux
|
||||
|
||||
- name: Limit target directory size
|
||||
run: script/clear-target-dir-if-larger-than 100
|
||||
|
||||
- name: Determine version and release channel
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
run: |
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue