ci: Switch from BuildJet to GitHub runners (#35826)

In response to an ongoing BuildJet outage, consider migrating CI to
GitHub hosted runners.

Also includes revert of (causing flaky tests):
- https://github.com/zed-industries/zed/pull/35741

Downsides:
- Cost (2x)
- Force migration to Ubuntu 22.04 from 20.04 will bump our glibc minimum
from 2.31 to 2.35. Which would break RHEL 9.x (glibc 2.34), Ubuntu 20.04
(EOL) and derivatives.

Release Notes:

- N/A

.github/actionlint.yml

@@ -5,6 +5,11 @@ self-hosted-runner:
     # GitHub-hosted Runners
     - github-8vcpu-ubuntu-2404
     - github-16vcpu-ubuntu-2404
+    - github-32vcpu-ubuntu-2404
+    - github-8vcpu-ubuntu-2204
+    - github-16vcpu-ubuntu-2204
+    - github-32vcpu-ubuntu-2204
+    - github-16vcpu-ubuntu-2204-arm
     - windows-2025-16
     - windows-2025-32
     - windows-2025-64

.github/actions/build_docs/action.yml

@@ -13,7 +13,7 @@ runs:
       uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
       with:
         save-if: ${{ github.ref == 'refs/heads/main' }}
-        cache-provider: "buildjet"
+        # cache-provider: "buildjet"
 
     - name: Install Linux dependencies
       shell: bash -euxo pipefail {0}

.github/workflows/bump_patch_version.yml

@@ -16,7 +16,7 @@ jobs:
   bump_patch_version:
     if: github.repository_owner == 'zed-industries'
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Checkout code
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4

.github/workflows/ci.yml

@@ -137,7 +137,7 @@ jobs:
       github.repository_owner == 'zed-industries' &&
       needs.job_spec.outputs.run_tests == 'true'
     runs-on:
-      - buildjet-8vcpu-ubuntu-2204
+      - github-8vcpu-ubuntu-2204
     steps:
       - name: Checkout repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -168,7 +168,7 @@ jobs:
     needs: [job_spec]
     if: github.repository_owner == 'zed-industries'
     runs-on:
-      - buildjet-8vcpu-ubuntu-2204
+      - github-8vcpu-ubuntu-2204
     steps:
       - name: Checkout repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -221,7 +221,7 @@ jobs:
       github.repository_owner == 'zed-industries' &&
       (needs.job_spec.outputs.run_tests == 'true' || needs.job_spec.outputs.run_docs == 'true')
     runs-on:
-      - buildjet-8vcpu-ubuntu-2204
+      - github-8vcpu-ubuntu-2204
     steps:
       - name: Checkout repo
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
@@ -328,7 +328,7 @@ jobs:
       github.repository_owner == 'zed-industries' &&
       needs.job_spec.outputs.run_tests == 'true'
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Add Rust to the PATH
         run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
@@ -342,7 +342,7 @@ jobs:
         uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-          cache-provider: "buildjet"
+          # cache-provider: "buildjet"
 
       - name: Install Linux dependencies
         run: ./script/linux
@@ -380,7 +380,7 @@ jobs:
       github.repository_owner == 'zed-industries' &&
       needs.job_spec.outputs.run_tests == 'true'
     runs-on:
-      - buildjet-8vcpu-ubuntu-2204
+      - github-8vcpu-ubuntu-2204
     steps:
       - name: Add Rust to the PATH
         run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
@@ -394,7 +394,7 @@ jobs:
         uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-          cache-provider: "buildjet"
+          # cache-provider: "buildjet"
 
       - name: Install Clang & Mold
         run: ./script/remote-server && ./script/install-mold 2.34.0
@@ -597,7 +597,8 @@ jobs:
     timeout-minutes: 60
     name: Linux x86_x64 release bundle
     runs-on:
-      - buildjet-16vcpu-ubuntu-2004 # ubuntu 20.04 for minimal glibc
+      - github-16vcpu-ubuntu-2204
+      # - buildjet-16vcpu-ubuntu-2004 # ubuntu 20.04 for minimal glibc
     if: |
       startsWith(github.ref, 'refs/tags/v')
       || contains(github.event.pull_request.labels.*.name, 'run-bundling')
@@ -650,7 +651,7 @@ jobs:
     timeout-minutes: 60
     name: Linux arm64 release bundle
     runs-on:
-      - buildjet-32vcpu-ubuntu-2204-arm
+      - github-16vcpu-ubuntu-2204-arm
     if: |
       startsWith(github.ref, 'refs/tags/v')
       || contains(github.event.pull_request.labels.*.name, 'run-bundling')

.github/workflows/deploy_cloudflare.yml

@@ -9,7 +9,7 @@ jobs:
   deploy-docs:
     name: Deploy Docs
     if: github.repository_owner == 'zed-industries'
-    runs-on: buildjet-16vcpu-ubuntu-2204
+    runs-on: github-16vcpu-ubuntu-2204
 
     steps:
       - name: Checkout repo

.github/workflows/deploy_collab.yml

@@ -61,7 +61,7 @@ jobs:
       - style
       - tests
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Install doctl
         uses: digitalocean/action-doctl@v2
@@ -94,7 +94,7 @@ jobs:
     needs:
       - publish
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
 
     steps:
       - name: Checkout repo

.github/workflows/eval.yml

@@ -32,7 +32,7 @@ jobs:
       github.repository_owner == 'zed-industries' &&
       (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-eval'))
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Add Rust to the PATH
         run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
@@ -46,7 +46,7 @@ jobs:
         uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-          cache-provider: "buildjet"
+          # cache-provider: "buildjet"
 
       - name: Install Linux dependencies
         run: ./script/linux

.github/workflows/nix.yml

@@ -20,7 +20,7 @@ jobs:
       matrix:
         system:
           - os: x86 Linux
-            runner: buildjet-16vcpu-ubuntu-2204
+            runner: github-16vcpu-ubuntu-2204
             install_nix: true
           - os: arm Mac
             runner: [macOS, ARM64, test]

.github/workflows/randomized_tests.yml

@@ -20,7 +20,7 @@ jobs:
     name: Run randomized tests
     if: github.repository_owner == 'zed-industries'
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Install Node
         uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4

.github/workflows/release_nightly.yml

@@ -128,7 +128,8 @@ jobs:
     name: Create a Linux *.tar.gz bundle for x86
     if: github.repository_owner == 'zed-industries'
     runs-on:
-      - buildjet-16vcpu-ubuntu-2004
+      - github-16vcpu-ubuntu-2204
+      # - buildjet-16vcpu-ubuntu-2004
     needs: tests
     steps:
       - name: Checkout repo
@@ -168,7 +169,7 @@ jobs:
     name: Create a Linux *.tar.gz bundle for ARM
     if: github.repository_owner == 'zed-industries'
     runs-on:
-      - buildjet-32vcpu-ubuntu-2204-arm
+      - github-16vcpu-ubuntu-2204-arm
     needs: tests
     steps:
       - name: Checkout repo

.github/workflows/unit_evals.yml

@@ -23,7 +23,7 @@ jobs:
     timeout-minutes: 60
     name: Run unit evals
     runs-on:
-      - buildjet-16vcpu-ubuntu-2204
+      - github-16vcpu-ubuntu-2204
     steps:
       - name: Add Rust to the PATH
         run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
@@ -37,7 +37,7 @@ jobs:
         uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2
         with:
           save-if: ${{ github.ref == 'refs/heads/main' }}
-          cache-provider: "buildjet"
+          # cache-provider: "buildjet"
 
       - name: Install Linux dependencies
         run: ./script/linux

crates/fs/src/fs_watcher.rs

@@ -1,9 +1,6 @@
 use notify::EventKind;
 use parking_lot::Mutex;
-use std::{
+use std::sync::{Arc, OnceLock};
-    collections::HashMap,
-    sync::{Arc, OnceLock},
-};
 use util::{ResultExt, paths::SanitizedPath};
 
 use crate::{PathEvent, PathEventKind, Watcher};
@@ -11,7 +8,6 @@ use crate::{PathEvent, PathEventKind, Watcher};
 pub struct FsWatcher {
     tx: smol::channel::Sender<()>,
     pending_path_events: Arc<Mutex<Vec<PathEvent>>>,
-    registrations: Mutex<HashMap<Arc<std::path::Path>, WatcherRegistrationId>>,
 }
 
 impl FsWatcher {
@@ -22,24 +18,10 @@ impl FsWatcher {
         Self {
             tx,
             pending_path_events,
-            registrations: Default::default(),
         }
     }
 }
 
-impl Drop for FsWatcher {
-    fn drop(&mut self) {
-        let mut registrations = self.registrations.lock();
-        let registrations = registrations.drain();
-
-        let _ = global(|g| {
-            for (_, registration) in registrations {
-                g.remove(registration);
-            }
-        });
-    }
-}
-
 impl Watcher for FsWatcher {
     fn add(&self, path: &std::path::Path) -> anyhow::Result<()> {
         let root_path = SanitizedPath::from(path);
@@ -47,136 +29,75 @@ impl Watcher for FsWatcher {
         let tx = self.tx.clone();
         let pending_paths = self.pending_path_events.clone();
 
-        let path: Arc<std::path::Path> = path.into();
+        use notify::Watcher;
 
-        if self.registrations.lock().contains_key(&path) {
+        global({
-            return Ok(());
-        }
-
-        let registration_id = global({
-            let path = path.clone();
             |g| {
-                g.add(
+                g.add(move |event: &notify::Event| {
-                    path,
+                    let kind = match event.kind {
-                    notify::RecursiveMode::NonRecursive,
+                        EventKind::Create(_) => Some(PathEventKind::Created),
-                    move |event: &notify::Event| {
+                        EventKind::Modify(_) => Some(PathEventKind::Changed),
-                        let kind = match event.kind {
+                        EventKind::Remove(_) => Some(PathEventKind::Removed),
-                            EventKind::Create(_) => Some(PathEventKind::Created),
+                        _ => None,
-                            EventKind::Modify(_) => Some(PathEventKind::Changed),
+                    };
-                            EventKind::Remove(_) => Some(PathEventKind::Removed),
+                    let mut path_events = event
-                            _ => None,
+                        .paths
-                        };
+                        .iter()
-                        let mut path_events = event
+                        .filter_map(|event_path| {
-                            .paths
+                            let event_path = SanitizedPath::from(event_path);
-                            .iter()
+                            event_path.starts_with(&root_path).then(|| PathEvent {
-                            .filter_map(|event_path| {
+                                path: event_path.as_path().to_path_buf(),
-                                let event_path = SanitizedPath::from(event_path);
+                                kind,
-                                event_path.starts_with(&root_path).then(|| PathEvent {
-                                    path: event_path.as_path().to_path_buf(),
-                                    kind,
-                                })
                             })
-                            .collect::<Vec<_>>();
+                        })
+                        .collect::<Vec<_>>();
 
-                        if !path_events.is_empty() {
+                    if !path_events.is_empty() {
-                            path_events.sort();
+                        path_events.sort();
-                            let mut pending_paths = pending_paths.lock();
+                        let mut pending_paths = pending_paths.lock();
-                            if pending_paths.is_empty() {
+                        if pending_paths.is_empty() {
-                                tx.try_send(()).ok();
+                            tx.try_send(()).ok();
-                            }
-                            util::extend_sorted(
-                                &mut *pending_paths,
-                                path_events,
-                                usize::MAX,
-                                |a, b| a.path.cmp(&b.path),
-                            );
                         }
-                    },
+                        util::extend_sorted(
-                )
+                            &mut *pending_paths,
+                            path_events,
+                            usize::MAX,
+                            |a, b| a.path.cmp(&b.path),
+                        );
+                    }
+                })
             }
-        })??;
+        })?;
 
-        self.registrations.lock().insert(path, registration_id);
+        global(|g| {
+            g.watcher
+                .lock()
+                .watch(path, notify::RecursiveMode::NonRecursive)
+        })??;
 
         Ok(())
     }
 
     fn remove(&self, path: &std::path::Path) -> anyhow::Result<()> {
-        let Some(registration) = self.registrations.lock().remove(path) else {
+        use notify::Watcher;
-            return Ok(());
+        Ok(global(|w| w.watcher.lock().unwatch(path))??)
-        };
-
-        global(|w| w.remove(registration))
     }
 }
 
-#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Hash)]
-pub struct WatcherRegistrationId(u32);
-
-struct WatcherRegistrationState {
-    callback: Box<dyn Fn(&notify::Event) + Send + Sync>,
-    path: Arc<std::path::Path>,
-}
-
-struct WatcherState {
-    // two mutexes because calling watcher.add triggers an watcher.event, which needs watchers.
-    #[cfg(target_os = "linux")]
-    watcher: notify::INotifyWatcher,
-    #[cfg(target_os = "freebsd")]
-    watcher: notify::KqueueWatcher,
-    #[cfg(target_os = "windows")]
-    watcher: notify::ReadDirectoryChangesWatcher,
-
-    watchers: HashMap<WatcherRegistrationId, WatcherRegistrationState>,
-    path_registrations: HashMap<Arc<std::path::Path>, u32>,
-    last_registration: WatcherRegistrationId,
-}
-
 pub struct GlobalWatcher {
-    state: Mutex<WatcherState>,
+    // two mutexes because calling watcher.add triggers an watcher.event, which needs watchers.
+    #[cfg(target_os = "linux")]
+    pub(super) watcher: Mutex<notify::INotifyWatcher>,
+    #[cfg(target_os = "freebsd")]
+    pub(super) watcher: Mutex<notify::KqueueWatcher>,
+    #[cfg(target_os = "windows")]
+    pub(super) watcher: Mutex<notify::ReadDirectoryChangesWatcher>,
+    pub(super) watchers: Mutex<Vec<Box<dyn Fn(&notify::Event) + Send + Sync>>>,
 }
 
 impl GlobalWatcher {
-    #[must_use]
+    pub(super) fn add(&self, cb: impl Fn(&notify::Event) + Send + Sync + 'static) {
-    fn add(
+        self.watchers.lock().push(Box::new(cb))
-        &self,
-        path: Arc<std::path::Path>,
-        mode: notify::RecursiveMode,
-        cb: impl Fn(&notify::Event) + Send + Sync + 'static,
-    ) -> anyhow::Result<WatcherRegistrationId> {
-        use notify::Watcher;
-        let mut state = self.state.lock();
-
-        state.watcher.watch(&path, mode)?;
-
-        let id = state.last_registration;
-        state.last_registration = WatcherRegistrationId(id.0 + 1);
-
-        let registration_state = WatcherRegistrationState {
-            callback: Box::new(cb),
-            path: path.clone(),
-        };
-        state.watchers.insert(id, registration_state);
-        *state.path_registrations.entry(path.clone()).or_insert(0) += 1;
-
-        Ok(id)
-    }
-
-    pub fn remove(&self, id: WatcherRegistrationId) {
-        use notify::Watcher;
-        let mut state = self.state.lock();
-        let Some(registration_state) = state.watchers.remove(&id) else {
-            return;
-        };
-
-        let Some(count) = state.path_registrations.get_mut(&registration_state.path) else {
-            return;
-        };
-        *count -= 1;
-        if *count == 0 {
-            state.watcher.unwatch(&registration_state.path).log_err();
-            state.path_registrations.remove(&registration_state.path);
-        }
     }
 }
 
@@ -193,10 +114,8 @@ fn handle_event(event: Result<notify::Event, notify::Error>) {
         return;
     };
     global::<()>(move |watcher| {
-        let state = watcher.state.lock();
+        for f in watcher.watchers.lock().iter() {
-        for registration in state.watchers.values() {
+            f(&event)
-            let callback = &registration.callback;
-            callback(&event);
         }
     })
     .log_err();
@@ -205,12 +124,8 @@ fn handle_event(event: Result<notify::Event, notify::Error>) {
 pub fn global<T>(f: impl FnOnce(&GlobalWatcher) -> T) -> anyhow::Result<T> {
     let result = FS_WATCHER_INSTANCE.get_or_init(|| {
         notify::recommended_watcher(handle_event).map(|file_watcher| GlobalWatcher {
-            state: Mutex::new(WatcherState {
+            watcher: Mutex::new(file_watcher),
-                watcher: file_watcher,
+            watchers: Default::default(),
-                watchers: Default::default(),
-                path_registrations: Default::default(),
-                last_registration: Default::default(),
-            }),
         })
     });
     match result {