Add an eval binary that evaluates our semantic index against CodeSearchNet (#17375)

This PR is the beginning of an evaluation framework for our AI features.
Right now, we're evaluating our semantic search feature against the
[CodeSearchNet](https://github.com/github/CodeSearchNet) code search
dataset. This dataset is very limited (for the most part, only 1 known
good search result per repo) but it has surfaced some problems with our
search already.

Release Notes:

- N/A

---------

Co-authored-by: Jason <jason@zed.dev>
Co-authored-by: Jason Mancuso <7891333+jvmncs@users.noreply.github.com>
Co-authored-by: Nathan <nathan@zed.dev>
Co-authored-by: Richard <richard@zed.dev>
This commit is contained in:
Max Brunsfeld 2024-09-17 12:44:33 -07:00 committed by GitHub
parent 06a13c2983
commit d3d3a093b4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 881 additions and 144 deletions

View file

@ -101,7 +101,7 @@ jobs:
timeout-minutes: 60
name: (Linux) Run Clippy and tests
runs-on:
- hosted-linux-x86-1
- buildjet-16vcpu-ubuntu-2204
steps:
- name: Add Rust to the PATH
run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH
@ -111,6 +111,11 @@ jobs:
with:
clean: false
- name: Cache dependencies
uses: swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Install Linux dependencies
run: ./script/linux
@ -264,7 +269,7 @@ jobs:
timeout-minutes: 60
name: Create a Linux bundle
runs-on:
- hosted-linux-x86-1
- buildjet-16vcpu-ubuntu-2204
if: ${{ startsWith(github.ref, 'refs/tags/v') || contains(github.event.pull_request.labels.*.name, 'run-bundling') }}
needs: [linux_tests]
env:
@ -279,9 +284,6 @@ jobs:
- name: Install Linux dependencies
run: ./script/linux
- name: Limit target directory size
run: script/clear-target-dir-if-larger-than 100
- name: Determine version and release channel
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
run: |
@ -335,7 +337,7 @@ jobs:
timeout-minutes: 60
name: Create arm64 Linux bundle
runs-on:
- hosted-linux-arm-1
- buildjet-16vcpu-ubuntu-2204-arm
if: ${{ startsWith(github.ref, 'refs/tags/v') || contains(github.event.pull_request.labels.*.name, 'run-bundling') }}
needs: [linux_tests]
env:
@ -350,9 +352,6 @@ jobs:
- name: Install Linux dependencies
run: ./script/linux
- name: Limit target directory size
run: script/clear-target-dir-if-larger-than 100
- name: Determine version and release channel
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
run: |