Add minidump crash reporting (#35263)

- [x] Handle uploading minidumps from the remote_server
- [x] Associate minidumps with panics with some sort of ID (we don't use
session_id on the remote)
  - [x] Update the protobufs and client/server code to request panics
- [x] Upload minidumps with no corresponding panic
- [x] Fill in panic info when there _is_ a corresponding panic
- [x] Use an env var for the sentry endpoint instead of hardcoding it

Release Notes:

- Zed now generates minidumps for crash reporting

---------

Co-authored-by: Max Brunsfeld <maxbrunsfeld@gmail.com>

.github/workflows/ci.yml

@@ -24,6 +24,7 @@ env:
   DIGITALOCEAN_SPACES_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }}
   DIGITALOCEAN_SPACES_SECRET_KEY: ${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }}
   ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
+  ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
 
 jobs:
   job_spec:

.github/workflows/nix.yml

@@ -29,6 +29,7 @@ jobs:
     runs-on: ${{ matrix.system.runner }}
     env:
       ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
+      ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
       ZED_CLOUD_PROVIDER_ADDITIONAL_MODELS_JSON: ${{ secrets.ZED_CLOUD_PROVIDER_ADDITIONAL_MODELS_JSON }}
       GIT_LFS_SKIP_SMUDGE: 1 # breaks the livekit rust sdk examples which we don't actually depend on
     steps:

.github/workflows/release_nightly.yml

@@ -13,6 +13,7 @@ env:
   CARGO_INCREMENTAL: 0
   RUST_BACKTRACE: 1
   ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
+  ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
   DIGITALOCEAN_SPACES_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }}
   DIGITALOCEAN_SPACES_SECRET_KEY: ${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }}
 

Cargo.lock

@@ -1172,7 +1172,7 @@ dependencies = [
  "serde_json",
  "serde_path_to_error",
  "serde_qs 0.10.1",
- "smart-default",
+ "smart-default 0.6.0",
  "smol_str 0.1.24",
  "thiserror 1.0.69",
  "tokio",
@@ -3927,6 +3927,42 @@ dependencies = [
  "target-lexicon 0.13.2",
 ]
 
+[[package]]
+name = "crash-context"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031ed29858d90cfdf27fe49fae28028a1f20466db97962fa2f4ea34809aeebf3"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "mach2",
+]
+
+[[package]]
+name = "crash-handler"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2066907075af649bcb8bcb1b9b986329b243677e6918b2d920aa64b0aac5ace3"
+dependencies = [
+ "cfg-if",
+ "crash-context",
+ "libc",
+ "mach2",
+ "parking_lot",
+]
+
+[[package]]
+name = "crashes"
+version = "0.1.0"
+dependencies = [
+ "crash-handler",
+ "log",
+ "minidumper",
+ "paths",
+ "smol",
+ "workspace-hack",
+]
+
 [[package]]
 name = "crc"
 version = "3.2.1"
@@ -4453,6 +4489,15 @@ dependencies = [
  "zlog",
 ]
 
+[[package]]
+name = "debugid"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
+dependencies = [
+ "uuid",
+]
+
 [[package]]
 name = "deepseek"
 version = "0.1.0"
@@ -7235,6 +7280,17 @@ dependencies = [
  "workspace-hack",
 ]
 
+[[package]]
+name = "goblin"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b363a30c165f666402fe6a3024d3bec7ebc898f96a4a23bd1c99f8dbf3f4f47"
+dependencies = [
+ "log",
+ "plain",
+ "scroll",
+]
+
 [[package]]
 name = "google_ai"
 version = "0.1.0"
@@ -7850,6 +7906,7 @@ dependencies = [
  "http-body 1.0.1",
  "log",
  "parking_lot",
+ "reqwest 0.12.15 (git+https://github.com/zed-industries/reqwest.git?rev=951c770a32f1998d6e999cef3e59e0013e6c4415)",
  "serde",
  "serde_json",
  "url",
@@ -10080,6 +10137,63 @@ dependencies = [
  "unicase",
 ]
 
+[[package]]
+name = "minidump-common"
+version = "0.21.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c4d14bcca0fd3ed165a03000480aaa364c6860c34e900cb2dafdf3b95340e77"
+dependencies = [
+ "bitflags 2.9.0",
+ "debugid",
+ "num-derive",
+ "num-traits",
+ "range-map",
+ "scroll",
+ "smart-default 0.7.1",
+]
+
+[[package]]
+name = "minidump-writer"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abcd9c8a1e6e1e9d56ce3627851f39a17ea83e17c96bc510f29d7e43d78a7d"
+dependencies = [
+ "bitflags 2.9.0",
+ "byteorder",
+ "cfg-if",
+ "crash-context",
+ "goblin",
+ "libc",
+ "log",
+ "mach2",
+ "memmap2",
+ "memoffset",
+ "minidump-common",
+ "nix 0.28.0",
+ "procfs-core",
+ "scroll",
+ "tempfile",
+ "thiserror 1.0.69",
+]
+
+[[package]]
+name = "minidumper"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b4ebc9d1f8847ec1d078f78b35ed598e0ebefa1f242d5f83cd8d7f03960a7d1"
+dependencies = [
+ "cfg-if",
+ "crash-context",
+ "libc",
+ "log",
+ "minidump-writer",
+ "parking_lot",
+ "polling",
+ "scroll",
+ "thiserror 1.0.69",
+ "uds",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -12069,6 +12183,12 @@ version = "0.3.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
 
+[[package]]
+name = "plain"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
+
 [[package]]
 name = "plist"
 version = "1.7.1"
@@ -12329,6 +12449,16 @@ dependencies = [
  "yansi",
 ]
 
+[[package]]
+name = "procfs-core"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
+dependencies = [
+ "bitflags 2.9.0",
+ "hex",
+]
+
 [[package]]
 name = "prodash"
 version = "29.0.2"
@@ -12979,6 +13109,15 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "range-map"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12a5a2d6c7039059af621472a4389be1215a816df61aa4d531cfe85264aee95f"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "rangemap"
 version = "1.5.1"
@@ -13321,6 +13460,8 @@ dependencies = [
  "clap",
  "client",
  "clock",
+ "crash-handler",
+ "crashes",
  "dap",
  "dap_adapters",
  "debug_adapter_extension",
@@ -13344,6 +13485,7 @@ dependencies = [
  "libc",
  "log",
  "lsp",
+ "minidumper",
  "node_runtime",
  "paths",
  "project",
@@ -13532,6 +13674,7 @@ dependencies = [
  "js-sys",
  "log",
  "mime",
+ "mime_guess",
  "once_cell",
  "percent-encoding",
  "pin-project-lite",
@@ -14260,6 +14403,26 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "scroll"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6"
+dependencies = [
+ "scroll_derive",
+]
+
+[[package]]
+name = "scroll_derive"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "scrypt"
 version = "0.11.0"
@@ -15005,6 +15168,17 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "smart-default"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.101",
+]
+
 [[package]]
 name = "smol"
 version = "2.0.2"
@@ -17288,6 +17462,15 @@ version = "0.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 
+[[package]]
+name = "uds"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "885c31f06fce836457fe3ef09a59f83fe8db95d270b11cd78f40a4666c4d1661"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "uds_windows"
 version = "1.1.0"
@@ -19743,9 +19926,11 @@ dependencies = [
  "lyon_path",
  "md-5",
  "memchr",
+ "mime_guess",
  "miniz_oxide",
  "mio 1.0.3",
  "naga",
+ "nix 0.28.0",
  "nix 0.29.0",
  "nom",
  "num-bigint",
@@ -20217,6 +20402,7 @@ dependencies = [
  "command_palette",
  "component",
  "copilot",
+ "crashes",
  "dap",
  "dap_adapters",
  "db",
@@ -20284,6 +20470,7 @@ dependencies = [
  "release_channel",
  "remote",
  "repl",
+ "reqwest 0.12.15 (git+https://github.com/zed-industries/reqwest.git?rev=951c770a32f1998d6e999cef3e59e0013e6c4415)",
  "reqwest_client",
  "rope",
  "search",

Cargo.toml

@@ -40,6 +40,7 @@ members = [
     "crates/component",
     "crates/context_server",
     "crates/copilot",
+    "crates/crashes",
     "crates/credentials_provider",
     "crates/dap",
     "crates/dap_adapters",
@@ -266,6 +267,7 @@ command_palette_hooks = { path = "crates/command_palette_hooks" }
 component = { path = "crates/component" }
 context_server = { path = "crates/context_server" }
 copilot = { path = "crates/copilot" }
+crashes = { path = "crates/crashes" }
 credentials_provider = { path = "crates/credentials_provider" }
 dap = { path = "crates/dap" }
 dap_adapters = { path = "crates/dap_adapters" }
@@ -466,6 +468,7 @@ core-foundation = "0.10.0"
 core-foundation-sys = "0.8.6"
 core-video = { version = "0.4.3", features = ["metal"] }
 cpal = "0.16"
+crash-handler = "0.6"
 criterion = { version = "0.5", features = ["html_reports"] }
 ctor = "0.4.0"
 dap-types = { git = "https://github.com/zed-industries/dap-types", rev = "1b461b310481d01e02b2603c16d7144b926339f8" }
@@ -513,6 +516,7 @@ log = { version = "0.4.16", features = ["kv_unstable_serde", "serde"] }
 lsp-types = { git = "https://github.com/zed-industries/lsp-types", rev = "39f629bdd03d59abd786ed9fc27e8bca02c0c0ec" }
 markup5ever_rcdom = "0.3.0"
 metal = "0.29"
+minidumper = "0.8"
 moka = { version = "0.12.10", features = ["sync"] }
 naga = { version = "25.0", features = ["wgsl-in"] }
 nanoid = "0.4"
@@ -552,6 +556,7 @@ reqwest = { git = "https://github.com/zed-industries/reqwest.git", rev = "951c77
     "charset",
     "http2",
     "macos-system-configuration",
+    "multipart",
     "rustls-tls-native-roots",
     "socks",
     "stream",

crates/client/src/telemetry.rs

@@ -74,6 +74,12 @@ static ZED_CLIENT_CHECKSUM_SEED: LazyLock<Option<Vec<u8>>> = LazyLock::new(|| {
         })
 });
 
+pub static SENTRY_MINIDUMP_ENDPOINT: LazyLock<Option<String>> = LazyLock::new(|| {
+    option_env!("SENTRY_MINIDUMP_ENDPOINT")
+        .map(|s| s.to_owned())
+        .or_else(|| env::var("SENTRY_MINIDUMP_ENDPOINT").ok())
+});
+
 static DOTNET_PROJECT_FILES_REGEX: LazyLock<Regex> = LazyLock::new(|| {
     Regex::new(r"^(global\.json|Directory\.Build\.props|.*\.(csproj|fsproj|vbproj|sln))$").unwrap()
 });

crates/crashes/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "crashes"
+version = "0.1.0"
+publish.workspace = true
+edition.workspace = true
+license = "GPL-3.0-or-later"
+
+[dependencies]
+crash-handler.workspace = true
+log.workspace = true
+minidumper.workspace = true
+paths.workspace = true
+smol.workspace = true
+workspace-hack.workspace = true
+
+[lints]
+workspace = true
+
+[lib]
+path = "src/crashes.rs"

crates/crashes/LICENSE-GPL

@@ -0,0 +1 @@
+../../LICENSE-GPL

crates/crashes/src/crashes.rs

@@ -0,0 +1,172 @@
+use crash_handler::CrashHandler;
+use log::info;
+use minidumper::{Client, LoopAction, MinidumpBinary};
+
+use std::{
+    env,
+    fs::File,
+    io,
+    path::{Path, PathBuf},
+    process::{self, Command},
+    sync::{
+        OnceLock,
+        atomic::{AtomicBool, Ordering},
+    },
+    thread,
+    time::Duration,
+};
+
+// set once the crash handler has initialized and the client has connected to it
+pub static CRASH_HANDLER: AtomicBool = AtomicBool::new(false);
+// set when the first minidump request is made to avoid generating duplicate crash reports
+pub static REQUESTED_MINIDUMP: AtomicBool = AtomicBool::new(false);
+const CRASH_HANDLER_TIMEOUT: Duration = Duration::from_secs(60);
+
+pub async fn init(id: String) {
+    let exe = env::current_exe().expect("unable to find ourselves");
+    let zed_pid = process::id();
+    // TODO: we should be able to get away with using 1 crash-handler process per machine,
+    // but for now we append the PID of the current process which makes it unique per remote
+    // server or interactive zed instance. This solves an issue where occasionally the socket
+    // used by the crash handler isn't destroyed correctly which causes it to stay on the file
+    // system and block further attempts to initialize crash handlers with that socket path.
+    let socket_name = paths::temp_dir().join(format!("zed-crash-handler-{zed_pid}"));
+    #[allow(unused)]
+    let server_pid = Command::new(exe)
+        .arg("--crash-handler")
+        .arg(&socket_name)
+        .spawn()
+        .expect("unable to spawn server process")
+        .id();
+    info!("spawning crash handler process");
+
+    let mut elapsed = Duration::ZERO;
+    let retry_frequency = Duration::from_millis(100);
+    let mut maybe_client = None;
+    while maybe_client.is_none() {
+        if let Ok(client) = Client::with_name(socket_name.as_path()) {
+            maybe_client = Some(client);
+            info!("connected to crash handler process after {elapsed:?}");
+            break;
+        }
+        elapsed += retry_frequency;
+        smol::Timer::after(retry_frequency).await;
+    }
+    let client = maybe_client.unwrap();
+    client.send_message(1, id).unwrap(); // set session id on the server
+
+    let client = std::sync::Arc::new(client);
+    let handler = crash_handler::CrashHandler::attach(unsafe {
+        let client = client.clone();
+        crash_handler::make_crash_event(move |crash_context: &crash_handler::CrashContext| {
+            // only request a minidump once
+            let res = if REQUESTED_MINIDUMP
+                .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
+                .is_ok()
+            {
+                client.send_message(2, "mistakes were made").unwrap();
+                client.ping().unwrap();
+                client.request_dump(crash_context).is_ok()
+            } else {
+                true
+            };
+            crash_handler::CrashEventResult::Handled(res)
+        })
+    })
+    .expect("failed to attach signal handler");
+
+    #[cfg(target_os = "linux")]
+    {
+        handler.set_ptracer(Some(server_pid));
+    }
+    CRASH_HANDLER.store(true, Ordering::Release);
+    std::mem::forget(handler);
+    info!("crash handler registered");
+
+    loop {
+        client.ping().ok();
+        smol::Timer::after(Duration::from_secs(10)).await;
+    }
+}
+
+pub struct CrashServer {
+    session_id: OnceLock<String>,
+}
+
+impl minidumper::ServerHandler for CrashServer {
+    fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> {
+        let err_message = "Need to send a message with the ID upon starting the crash handler";
+        let dump_path = paths::logs_dir()
+            .join(self.session_id.get().expect(err_message))
+            .with_extension("dmp");
+        let file = File::create(&dump_path)?;
+        Ok((file, dump_path))
+    }
+
+    fn on_minidump_created(&self, result: Result<MinidumpBinary, minidumper::Error>) -> LoopAction {
+        match result {
+            Ok(mut md_bin) => {
+                use io::Write;
+                let _ = md_bin.file.flush();
+                info!("wrote minidump to disk {:?}", md_bin.path);
+            }
+            Err(e) => {
+                info!("failed to write minidump: {:#}", e);
+            }
+        }
+        LoopAction::Exit
+    }
+
+    fn on_message(&self, kind: u32, buffer: Vec<u8>) {
+        let message = String::from_utf8(buffer).expect("invalid utf-8");
+        info!("kind: {kind}, message: {message}",);
+        if kind == 1 {
+            self.session_id
+                .set(message)
+                .expect("session id already initialized");
+        }
+    }
+
+    fn on_client_disconnected(&self, clients: usize) -> LoopAction {
+        info!("client disconnected, {clients} remaining");
+        if clients == 0 {
+            LoopAction::Exit
+        } else {
+            LoopAction::Continue
+        }
+    }
+}
+
+pub fn handle_panic() {
+    // wait 500ms for the crash handler process to start up
+    // if it's still not there just write panic info and no minidump
+    let retry_frequency = Duration::from_millis(100);
+    for _ in 0..5 {
+        if CRASH_HANDLER.load(Ordering::Acquire) {
+            log::error!("triggering a crash to generate a minidump...");
+            #[cfg(target_os = "linux")]
+            CrashHandler.simulate_signal(crash_handler::Signal::Trap as u32);
+            #[cfg(not(target_os = "linux"))]
+            CrashHandler.simulate_exception(None);
+            break;
+        }
+        thread::sleep(retry_frequency);
+    }
+}
+
+pub fn crash_server(socket: &Path) {
+    let Ok(mut server) = minidumper::Server::with_name(socket) else {
+        log::info!("Couldn't create socket, there may already be a running crash server");
+        return;
+    };
+    let ab = AtomicBool::new(false);
+    server
+        .run(
+            Box::new(CrashServer {
+                session_id: OnceLock::new(),
+            }),
+            &ab,
+            Some(CRASH_HANDLER_TIMEOUT),
+        )
+        .expect("failed to run server");
+}

crates/http_client/Cargo.toml

@@ -24,6 +24,7 @@ http.workspace = true
 http-body.workspace = true
 log.workspace = true
 parking_lot.workspace = true
+reqwest.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 url.workspace = true

crates/http_client/src/async_body.rs

@@ -88,6 +88,17 @@ impl From<&'static str> for AsyncBody {
     }
 }
 
+impl TryFrom<reqwest::Body> for AsyncBody {
+    type Error = anyhow::Error;
+
+    fn try_from(value: reqwest::Body) -> Result<Self, Self::Error> {
+        value
+            .as_bytes()
+            .ok_or_else(|| anyhow::anyhow!("Underlying data is a stream"))
+            .map(|bytes| Self::from_bytes(Bytes::copy_from_slice(bytes)))
+    }
+}
+
 impl<T: Into<Self>> From<Option<T>> for AsyncBody {
     fn from(body: Option<T>) -> Self {
         match body {

crates/http_client/src/http_client.rs

@@ -7,7 +7,10 @@ use derive_more::Deref;
 use http::HeaderValue;
 pub use http::{self, Method, Request, Response, StatusCode, Uri};
 
-use futures::future::BoxFuture;
+use futures::{
+    FutureExt as _,
+    future::{self, BoxFuture},
+};
 use http::request::Builder;
 use parking_lot::Mutex;
 #[cfg(feature = "test-support")]
@@ -89,6 +92,14 @@ pub trait HttpClient: 'static + Send + Sync {
     fn as_fake(&self) -> &FakeHttpClient {
         panic!("called as_fake on {}", type_name::<Self>())
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        _url: &str,
+        _request: reqwest::multipart::Form,
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
+        future::ready(Err(anyhow!("not implemented"))).boxed()
+    }
 }
 
 /// An [`HttpClient`] that may have a proxy.
@@ -140,31 +151,13 @@ impl HttpClient for HttpClientWithProxy {
     fn as_fake(&self) -> &FakeHttpClient {
         self.client.as_fake()
     }
-}
 
-impl HttpClient for Arc<HttpClientWithProxy> {
+    fn send_multipart_form<'a>(
-    fn send(
+        &'a self,
-        &self,
+        url: &str,
-        req: Request<AsyncBody>,
+        form: reqwest::multipart::Form,
-    ) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
-        self.client.send(req)
+        self.client.send_multipart_form(url, form)
-    }
-
-    fn user_agent(&self) -> Option<&HeaderValue> {
-        self.client.user_agent()
-    }
-
-    fn proxy(&self) -> Option<&Url> {
-        self.proxy.as_ref()
-    }
-
-    fn type_name(&self) -> &'static str {
-        self.client.type_name()
-    }
-
-    #[cfg(feature = "test-support")]
-    fn as_fake(&self) -> &FakeHttpClient {
-        self.client.as_fake()
     }
 }
 
@@ -275,32 +268,6 @@ impl HttpClientWithUrl {
     }
 }
 
-impl HttpClient for Arc<HttpClientWithUrl> {
-    fn send(
-        &self,
-        req: Request<AsyncBody>,
-    ) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
-        self.client.send(req)
-    }
-
-    fn user_agent(&self) -> Option<&HeaderValue> {
-        self.client.user_agent()
-    }
-
-    fn proxy(&self) -> Option<&Url> {
-        self.client.proxy.as_ref()
-    }
-
-    fn type_name(&self) -> &'static str {
-        self.client.type_name()
-    }
-
-    #[cfg(feature = "test-support")]
-    fn as_fake(&self) -> &FakeHttpClient {
-        self.client.as_fake()
-    }
-}
-
 impl HttpClient for HttpClientWithUrl {
     fn send(
         &self,
@@ -325,6 +292,14 @@ impl HttpClient for HttpClientWithUrl {
     fn as_fake(&self) -> &FakeHttpClient {
         self.client.as_fake()
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        url: &str,
+        request: reqwest::multipart::Form,
+    ) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
+        self.client.send_multipart_form(url, request)
+    }
 }
 
 pub fn read_proxy_from_env() -> Option<Url> {

crates/proto/proto/app.proto

@@ -79,11 +79,16 @@ message OpenServerSettings {
     uint64 project_id = 1;
 }
 
-message GetPanicFiles {
+message GetCrashFiles {
 }
 
-message GetPanicFilesResponse {
+message GetCrashFilesResponse {
-    repeated string file_contents = 2;
+    repeated CrashReport crashes = 1;
+}
+
+message CrashReport {
+    optional string panic_contents = 1;
+    optional bytes minidump_contents = 2;
 }
 
 message Extension {

crates/proto/proto/zed.proto

@@ -294,9 +294,6 @@ message Envelope {
         GetPathMetadata get_path_metadata = 278;
         GetPathMetadataResponse get_path_metadata_response = 279;
 
-        GetPanicFiles get_panic_files = 280;
-        GetPanicFilesResponse get_panic_files_response = 281;
-
         CancelLanguageServerWork cancel_language_server_work = 282;
 
         LspExtOpenDocs lsp_ext_open_docs = 283;
@@ -402,7 +399,10 @@ message Envelope {
         StashPop stash_pop = 358;
 
         GetDefaultBranch get_default_branch = 359;
-        GetDefaultBranchResponse get_default_branch_response = 360; // current max
+        GetDefaultBranchResponse get_default_branch_response = 360;
+
+        GetCrashFiles get_crash_files = 361;
+        GetCrashFilesResponse get_crash_files_response = 362; // current max
     }
 
     reserved 87 to 88;
@@ -423,6 +423,7 @@ message Envelope {
     reserved 270;
     reserved 247 to 254;
     reserved 255 to 256;
+    reserved 280 to 281;
 }
 
 message Hello {

crates/proto/src/proto.rs

@@ -99,8 +99,8 @@ messages!(
     (GetHoverResponse, Background),
     (GetNotifications, Foreground),
     (GetNotificationsResponse, Foreground),
-    (GetPanicFiles, Background),
+    (GetCrashFiles, Background),
-    (GetPanicFilesResponse, Background),
+    (GetCrashFilesResponse, Background),
     (GetPathMetadata, Background),
     (GetPathMetadataResponse, Background),
     (GetPermalinkToLine, Foreground),
@@ -462,7 +462,7 @@ request_messages!(
     (ActivateToolchain, Ack),
     (ActiveToolchain, ActiveToolchainResponse),
     (GetPathMetadata, GetPathMetadataResponse),
-    (GetPanicFiles, GetPanicFilesResponse),
+    (GetCrashFiles, GetCrashFilesResponse),
     (CancelLanguageServerWork, Ack),
     (SyncExtensions, SyncExtensionsResponse),
     (InstallExtension, Ack),

crates/remote_server/Cargo.toml

@@ -67,8 +67,11 @@ watch.workspace = true
 worktree.workspace = true
 
 [target.'cfg(not(windows))'.dependencies]
+crashes.workspace = true
+crash-handler.workspace = true
 fork.workspace = true
 libc.workspace = true
+minidumper.workspace = true
 
 [dev-dependencies]
 assistant_tool.workspace = true

crates/remote_server/src/main.rs

@@ -12,6 +12,10 @@ struct Cli {
     /// by having Zed act like netcat communicating over a Unix socket.
     #[arg(long, hide = true)]
     askpass: Option<String>,
+    /// Used for recording minidumps on crashes by having the server run a separate
+    /// process communicating over a socket.
+    #[arg(long, hide = true)]
+    crash_handler: Option<PathBuf>,
     /// Used for loading the environment from the project.
     #[arg(long, hide = true)]
     printenv: bool,
@@ -58,6 +62,11 @@ fn main() {
         return;
     }
 
+    if let Some(socket) = &cli.crash_handler {
+        crashes::crash_server(socket.as_path());
+        return;
+    }
+
     if cli.printenv {
         util::shell_env::print_env();
         return;

crates/remote_server/src/unix.rs

@@ -17,6 +17,7 @@ use node_runtime::{NodeBinaryOptions, NodeRuntime};
 use paths::logs_dir;
 use project::project_settings::ProjectSettings;
 
+use proto::CrashReport;
 use release_channel::{AppVersion, RELEASE_CHANNEL, ReleaseChannel};
 use remote::proxy::ProxyLaunchError;
 use remote::ssh_session::ChannelClient;
@@ -33,6 +34,7 @@ use smol::io::AsyncReadExt;
 
 use smol::Async;
 use smol::{net::unix::UnixListener, stream::StreamExt as _};
+use std::collections::HashMap;
 use std::ffi::OsStr;
 use std::ops::ControlFlow;
 use std::str::FromStr;
@@ -109,8 +111,9 @@ fn init_logging_server(log_file_path: PathBuf) -> Result<Receiver<Vec<u8>>> {
     Ok(rx)
 }
 
-fn init_panic_hook() {
+fn init_panic_hook(session_id: String) {
-    std::panic::set_hook(Box::new(|info| {
+    std::panic::set_hook(Box::new(move |info| {
+        crashes::handle_panic();
         let payload = info
             .payload()
             .downcast_ref::<&str>()
@@ -171,9 +174,11 @@ fn init_panic_hook() {
             architecture: env::consts::ARCH.into(),
             panicked_on: Utc::now().timestamp_millis(),
             backtrace,
-            system_id: None,            // Set on SSH client
+            system_id: None,       // Set on SSH client
-            installation_id: None,      // Set on SSH client
+            installation_id: None, // Set on SSH client
-            session_id: "".to_string(), // Set on SSH client
+
+            // used on this end to associate panics with minidumps, but will be replaced on the SSH client
+            session_id: session_id.clone(),
         };
 
         if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
@@ -194,44 +199,69 @@ fn init_panic_hook() {
     }));
 }
 
-fn handle_panic_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
+fn handle_crash_files_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
     let client: AnyProtoClient = client.clone().into();
     client.add_request_handler(
         project.downgrade(),
-        |_, _: TypedEnvelope<proto::GetPanicFiles>, _cx| async move {
+        |_, _: TypedEnvelope<proto::GetCrashFiles>, _cx| async move {
+            let mut crashes = Vec::new();
+            let mut minidumps_by_session_id = HashMap::new();
             let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
-            let mut panic_files = Vec::new();
             while let Some(child) = children.next().await {
                 let child = child?;
                 let child_path = child.path();
 
-                if child_path.extension() != Some(OsStr::new("panic")) {
+                let extension = child_path.extension();
-                    continue;
+                if extension == Some(OsStr::new("panic")) {
+                    let filename = if let Some(filename) = child_path.file_name() {
+                        filename.to_string_lossy()
+                    } else {
+                        continue;
+                    };
+
+                    if !filename.starts_with("zed") {
+                        continue;
+                    }
+
+                    let file_contents = smol::fs::read_to_string(&child_path)
+                        .await
+                        .context("error reading panic file")?;
+
+                    crashes.push(proto::CrashReport {
+                        panic_contents: Some(file_contents),
+                        minidump_contents: None,
+                    });
+                } else if extension == Some(OsStr::new("dmp")) {
+                    let session_id = child_path.file_stem().unwrap().to_string_lossy();
+                    minidumps_by_session_id
+                        .insert(session_id.to_string(), smol::fs::read(&child_path).await?);
                 }
-                let filename = if let Some(filename) = child_path.file_name() {
-                    filename.to_string_lossy()
-                } else {
-                    continue;
-                };
-
-                if !filename.starts_with("zed") {
-                    continue;
-                }
-
-                let file_contents = smol::fs::read_to_string(&child_path)
-                    .await
-                    .context("error reading panic file")?;
-
-                panic_files.push(file_contents);
 
                 // We've done what we can, delete the file
-                std::fs::remove_file(child_path)
+                smol::fs::remove_file(&child_path)
+                    .await
                     .context("error removing panic")
                     .log_err();
             }
-            anyhow::Ok(proto::GetPanicFilesResponse {
+
-                file_contents: panic_files,
+            for crash in &mut crashes {
-            })
+                let panic: telemetry_events::Panic =
+                    serde_json::from_str(crash.panic_contents.as_ref().unwrap())?;
+                if let dump @ Some(_) = minidumps_by_session_id.remove(&panic.session_id) {
+                    crash.minidump_contents = dump;
+                }
+            }
+
+            crashes.extend(
+                minidumps_by_session_id
+                    .into_values()
+                    .map(|dmp| CrashReport {
+                        panic_contents: None,
+                        minidump_contents: Some(dmp),
+                    }),
+            );
+
+            anyhow::Ok(proto::GetCrashFilesResponse { crashes })
         },
     );
 }
@@ -409,7 +439,12 @@ pub fn execute_run(
         ControlFlow::Continue(_) => {}
     }
 
-    init_panic_hook();
+    let app = gpui::Application::headless();
+    let id = std::process::id().to_string();
+    app.background_executor()
+        .spawn(crashes::init(id.clone()))
+        .detach();
+    init_panic_hook(id);
     let log_rx = init_logging_server(log_file)?;
     log::info!(
         "starting up. pid_file: {:?}, stdin_socket: {:?}, stdout_socket: {:?}, stderr_socket: {:?}",
@@ -425,7 +460,7 @@ pub fn execute_run(
     let listeners = ServerListeners::new(stdin_socket, stdout_socket, stderr_socket)?;
 
     let git_hosting_provider_registry = Arc::new(GitHostingProviderRegistry::new());
-    gpui::Application::headless().run(move |cx| {
+    app.run(move |cx| {
         settings::init(cx);
         let app_version = AppVersion::load(env!("ZED_PKG_VERSION"));
         release_channel::init(app_version, cx);
@@ -486,7 +521,7 @@ pub fn execute_run(
             )
         });
 
-        handle_panic_requests(&project, &session);
+        handle_crash_files_requests(&project, &session);
 
         cx.background_spawn(async move { cleanup_old_binaries() })
             .detach();
@@ -530,12 +565,15 @@ impl ServerPaths {
 
 pub fn execute_proxy(identifier: String, is_reconnecting: bool) -> Result<()> {
     init_logging_proxy();
-    init_panic_hook();
-
-    log::info!("starting proxy process. PID: {}", std::process::id());
 
     let server_paths = ServerPaths::new(&identifier)?;
 
+    let id = std::process::id().to_string();
+    smol::spawn(crashes::init(id.clone())).detach();
+    init_panic_hook(id);
+
+    log::info!("starting proxy process. PID: {}", std::process::id());
+
     let server_pid = check_pid_file(&server_paths.pid_file)?;
     let server_running = server_pid.is_some();
     if is_reconnecting {

crates/reqwest_client/src/reqwest_client.rs

@@ -4,14 +4,13 @@ use std::{any::type_name, borrow::Cow, mem, pin::Pin, task::Poll, time::Duration
 
 use anyhow::anyhow;
 use bytes::{BufMut, Bytes, BytesMut};
-use futures::{AsyncRead, TryStreamExt as _};
+use futures::{AsyncRead, FutureExt as _, TryStreamExt as _};
 use http_client::{RedirectPolicy, Url, http};
 use regex::Regex;
 use reqwest::{
     header::{HeaderMap, HeaderValue},
     redirect,
 };
-use smol::future::FutureExt;
 
 const DEFAULT_CAPACITY: usize = 4096;
 static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
@@ -274,6 +273,26 @@ impl http_client::HttpClient for ReqwestClient {
         }
         .boxed()
     }
+
+    fn send_multipart_form<'a>(
+        &'a self,
+        url: &str,
+        form: reqwest::multipart::Form,
+    ) -> futures::future::BoxFuture<'a, anyhow::Result<http_client::Response<http_client::AsyncBody>>>
+    {
+        let response = self.client.post(url).multipart(form).send();
+        self.handle
+            .spawn(async move {
+                let response = response.await?;
+                let mut builder = http::response::Builder::new().status(response.status());
+                for (k, v) in response.headers() {
+                    builder = builder.header(k, v)
+                }
+                Ok(builder.body(response.bytes().await?.into())?)
+            })
+            .map(|e| e?)
+            .boxed()
+    }
 }
 
 #[cfg(test)]

crates/zed/Cargo.toml

@@ -45,6 +45,7 @@ collections.workspace = true
 command_palette.workspace = true
 component.workspace = true
 copilot.workspace = true
+crashes.workspace = true
 dap_adapters.workspace = true
 db.workspace = true
 debug_adapter_extension.workspace = true
@@ -117,6 +118,7 @@ recent_projects.workspace = true
 release_channel.workspace = true
 remote.workspace = true
 repl.workspace = true
+reqwest.workspace = true
 reqwest_client.workspace = true
 rope.workspace = true
 search.workspace = true

crates/zed/src/main.rs

@@ -172,6 +172,12 @@ pub fn main() {
 
     let args = Args::parse();
 
+    // `zed --crash-handler` Makes zed operate in minidump crash handler mode
+    if let Some(socket) = &args.crash_handler {
+        crashes::crash_server(socket.as_path());
+        return;
+    }
+
     // `zed --askpass` Makes zed operate in nc/netcat mode for use with askpass
     if let Some(socket) = &args.askpass {
         askpass::main(socket);
@@ -264,6 +270,9 @@ pub fn main() {
     let session_id = Uuid::new_v4().to_string();
     let session = app.background_executor().block(Session::new());
 
+    app.background_executor()
+        .spawn(crashes::init(session_id.clone()))
+        .detach();
     reliability::init_panic_hook(
         app_version,
         app_commit_sha.clone(),
@@ -1185,6 +1194,11 @@ struct Args {
     #[arg(long, hide = true)]
     nc: Option<String>,
 
+    /// Used for recording minidumps on crashes by having Zed run a separate
+    /// process communicating over a socket.
+    #[arg(long, hide = true)]
+    crash_handler: Option<PathBuf>,
+
     /// Run zed in the foreground, only used on Windows, to match the behavior on macOS.
     #[arg(long)]
     #[cfg(target_os = "windows")]

crates/zed/src/reliability.rs

@@ -2,21 +2,32 @@ use crate::stdout_is_a_pty;
 use anyhow::{Context as _, Result};
 use backtrace::{self, Backtrace};
 use chrono::Utc;
-use client::{TelemetrySettings, telemetry};
+use client::{
+    TelemetrySettings,
+    telemetry::{self, SENTRY_MINIDUMP_ENDPOINT},
+};
 use db::kvp::KEY_VALUE_STORE;
+use futures::AsyncReadExt;
 use gpui::{App, AppContext as _, SemanticVersion};
 use http_client::{self, HttpClient, HttpClientWithUrl, HttpRequestExt, Method};
 use paths::{crashes_dir, crashes_retired_dir};
 use project::Project;
 use release_channel::{AppCommitSha, RELEASE_CHANNEL, ReleaseChannel};
+use reqwest::multipart::{Form, Part};
 use settings::Settings;
 use smol::stream::StreamExt;
 use std::{
     env,
     ffi::{OsStr, c_void},
-    sync::{Arc, atomic::Ordering},
+    fs,
+    io::Write,
+    panic,
+    sync::{
+        Arc,
+        atomic::{AtomicU32, Ordering},
+    },
+    thread,
 };
-use std::{io::Write, panic, sync::atomic::AtomicU32, thread};
 use telemetry_events::{LocationData, Panic, PanicRequest};
 use url::Url;
 use util::ResultExt;
@@ -37,9 +48,10 @@ pub fn init_panic_hook(
         if prior_panic_count > 0 {
             // Give the panic-ing thread time to write the panic file
             loop {
-                std::thread::yield_now();
+                thread::yield_now();
             }
         }
+        crashes::handle_panic();
 
         let thread = thread::current();
         let thread_name = thread.name().unwrap_or("<unnamed>");
@@ -136,9 +148,8 @@ pub fn init_panic_hook(
             if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
                 let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
                 let panic_file_path = paths::logs_dir().join(format!("zed-{timestamp}.panic"));
-                let panic_file = std::fs::OpenOptions::new()
+                let panic_file = fs::OpenOptions::new()
-                    .append(true)
+                    .create_new(true)
-                    .create(true)
                     .open(&panic_file_path)
                     .log_err();
                 if let Some(mut panic_file) = panic_file {
@@ -205,27 +216,31 @@ pub fn init(
         if let Some(ssh_client) = project.ssh_client() {
             ssh_client.update(cx, |client, cx| {
                 if TelemetrySettings::get_global(cx).diagnostics {
-                    let request = client.proto_client().request(proto::GetPanicFiles {});
+                    let request = client.proto_client().request(proto::GetCrashFiles {});
                     cx.background_spawn(async move {
-                        let panic_files = request.await?;
+                        let crash_files = request.await?;
-                        for file in panic_files.file_contents {
+                        for crash in crash_files.crashes {
-                            let panic: Option<Panic> = serde_json::from_str(&file)
+                            let mut panic: Option<Panic> = crash
-                                .log_err()
+                                .panic_contents
-                                .or_else(|| {
+                                .and_then(|s| serde_json::from_str(&s).log_err());
-                                    file.lines()
-                                        .next()
-                                        .and_then(|line| serde_json::from_str(line).ok())
-                                })
-                                .unwrap_or_else(|| {
-                                    log::error!("failed to deserialize panic file {:?}", file);
-                                    None
-                                });
 
-                            if let Some(mut panic) = panic {
+                            if let Some(panic) = panic.as_mut() {
                                 panic.session_id = session_id.clone();
                                 panic.system_id = system_id.clone();
                                 panic.installation_id = installation_id.clone();
+                            }
 
+                            if let Some(minidump) = crash.minidump_contents {
+                                upload_minidump(
+                                    http_client.clone(),
+                                    minidump.clone(),
+                                    panic.as_ref(),
+                                )
+                                .await
+                                .log_err();
+                            }
+
+                            if let Some(panic) = panic {
                                 upload_panic(&http_client, &panic_report_url, panic, &mut None)
                                     .await?;
                             }
@@ -510,6 +525,22 @@ async fn upload_previous_panics(
                 });
 
             if let Some(panic) = panic {
+                let minidump_path = paths::logs_dir()
+                    .join(&panic.session_id)
+                    .with_extension("dmp");
+                if minidump_path.exists() {
+                    let minidump = smol::fs::read(&minidump_path)
+                        .await
+                        .context("Failed to read minidump")?;
+                    if upload_minidump(http.clone(), minidump, Some(&panic))
+                        .await
+                        .log_err()
+                        .is_some()
+                    {
+                        fs::remove_file(minidump_path).ok();
+                    }
+                }
+
                 if !upload_panic(&http, &panic_report_url, panic, &mut most_recent_panic).await? {
                     continue;
                 }
@@ -517,13 +548,75 @@ async fn upload_previous_panics(
         }
 
         // We've done what we can, delete the file
-        std::fs::remove_file(child_path)
+        fs::remove_file(child_path)
             .context("error removing panic")
             .log_err();
     }
+
+    // loop back over the directory again to upload any minidumps that are missing panics
+    let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
+    while let Some(child) = children.next().await {
+        let child = child?;
+        let child_path = child.path();
+        if child_path.extension() != Some(OsStr::new("dmp")) {
+            continue;
+        }
+        if upload_minidump(
+            http.clone(),
+            smol::fs::read(&child_path)
+                .await
+                .context("Failed to read minidump")?,
+            None,
+        )
+        .await
+        .log_err()
+        .is_some()
+        {
+            fs::remove_file(child_path).ok();
+        }
+    }
+
     Ok(most_recent_panic)
 }
 
+async fn upload_minidump(
+    http: Arc<HttpClientWithUrl>,
+    minidump: Vec<u8>,
+    panic: Option<&Panic>,
+) -> Result<()> {
+    let sentry_upload_url = SENTRY_MINIDUMP_ENDPOINT
+        .to_owned()
+        .ok_or_else(|| anyhow::anyhow!("Minidump endpoint not set"))?;
+
+    let mut form = Form::new()
+        .part(
+            "upload_file_minidump",
+            Part::bytes(minidump)
+                .file_name("minidump.dmp")
+                .mime_str("application/octet-stream")?,
+        )
+        .text("platform", "rust");
+    if let Some(panic) = panic {
+        form = form.text(
+            "release",
+            format!("{}-{}", panic.release_channel, panic.app_version),
+        );
+        // TODO: tack on more fields
+    }
+
+    let mut response_text = String::new();
+    let mut response = http.send_multipart_form(&sentry_upload_url, form).await?;
+    response
+        .body_mut()
+        .read_to_string(&mut response_text)
+        .await?;
+    if !response.status().is_success() {
+        anyhow::bail!("failed to upload minidump: {response_text}");
+    }
+    log::info!("Uploaded minidump. event id: {response_text}");
+    Ok(())
+}
+
 async fn upload_panic(
     http: &Arc<HttpClientWithUrl>,
     panic_report_url: &Url,

docs/src/development/debugging-crashes.md

@@ -6,6 +6,7 @@ When an app crashes,
 
 - macOS creates a `.ips` file in `~/Library/Logs/DiagnosticReports`. You can view these using the built in Console app (`cmd-space Console`) under "Crash Reports".
 - Linux creates a core dump. See the [man pages](https://man7.org/linux/man-pages/man5/core.5.html) for pointers to how your system might be configured to manage core dumps.
+- Windows doesn't create crash reports by default, but can be configured to create "minidump" memory dumps upon applications crashing.
 
 If you have enabled Zed's telemetry these will be uploaded to us when you restart the app. They end up in a [Slack channel (internal only)](https://zed-industries.slack.com/archives/C04S6T1T7TQ).
 

docs/src/telemetry.md

@@ -21,7 +21,7 @@ The telemetry settings can also be configured via the welcome screen, which can
 
 Telemetry is sent from the application to our servers. Data is proxied through our servers to enable us to easily switch analytics services. We currently use:
 
-- [Axiom](https://axiom.co): Cloud-monitoring service - stores diagnostic events
+- [Sentry](https://sentry.io): Crash-monitoring service - stores diagnostic events
 - [Snowflake](https://snowflake.com): Data warehouse - stores both diagnostic and metric events
 - [Hex](https://www.hex.tech): Dashboards and data exploration - accesses data stored in Snowflake
 - [Amplitude](https://www.amplitude.com): Dashboards and data exploration - accesses data stored in Snowflake
@@ -30,9 +30,9 @@ Telemetry is sent from the application to our servers. Data is proxied through o
 
 ### Diagnostics
 
-Diagnostic events include debug information (stack traces) from crash reports. Reports are sent on the first application launch after the crash occurred. We've built dashboards that allow us to visualize the frequency and severity of issues experienced by users. Having these reports sent automatically allows us to begin implementing fixes without the user needing to file a report in our issue tracker. The plots in the dashboards also give us an informal measurement of the stability of Zed.
+Crash reports consist of a [minidump](https://learn.microsoft.com/en-us/windows/win32/debug/minidump-files) and some extra debug information. Reports are sent on the first application launch after the crash occurred. We've built dashboards that allow us to visualize the frequency and severity of issues experienced by users. Having these reports sent automatically allows us to begin implementing fixes without the user needing to file a report in our issue tracker. The plots in the dashboards also give us an informal measurement of the stability of Zed.
 
-You can see what data is sent when a panic occurs by inspecting the `Panic` struct in [crates/telemetry_events/src/telemetry_events.rs](https://github.com/zed-industries/zed/blob/main/crates/telemetry_events/src/telemetry_events.rs) in the Zed repo. You can find additional information in the [Debugging Crashes](./development/debugging-crashes.md) documentation.
+You can see what extra data is sent alongside the minidump in the `Panic` struct in [crates/telemetry_events/src/telemetry_events.rs](https://github.com/zed-industries/zed/blob/main/crates/telemetry_events/src/telemetry_events.rs) in the Zed repo. You can find additional information in the [Debugging Crashes](./development/debugging-crashes.md) documentation.
 
 ### Client-Side Usage Data {#client-metrics}
 

tooling/workspace-hack/Cargo.toml

@@ -82,6 +82,7 @@ lyon = { version = "1", default-features = false, features = ["extra"] }
 lyon_path = { version = "1" }
 md-5 = { version = "0.10" }
 memchr = { version = "2" }
+mime_guess = { version = "2" }
 miniz_oxide = { version = "0.8", features = ["simd"] }
 nom = { version = "7" }
 num-bigint = { version = "0.4" }
@@ -212,6 +213,7 @@ lyon = { version = "1", default-features = false, features = ["extra"] }
 lyon_path = { version = "1" }
 md-5 = { version = "0.10" }
 memchr = { version = "2" }
+mime_guess = { version = "2" }
 miniz_oxide = { version = "0.8", features = ["simd"] }
 nom = { version = "7" }
 num-bigint = { version = "0.4" }
@@ -290,7 +292,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
 hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
 itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
 naga = { version = "25", features = ["msl-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
 objc2 = { version = "0.6" }
 objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
 objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -318,7 +320,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
 hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
 itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
 naga = { version = "25", features = ["msl-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
 objc2 = { version = "0.6" }
 objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
 objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -347,7 +349,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
 hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
 itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
 naga = { version = "25", features = ["msl-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
 objc2 = { version = "0.6" }
 objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
 objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -375,7 +377,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
 hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
 itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
 naga = { version = "25", features = ["msl-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
 objc2 = { version = "0.6" }
 objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
 objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@@ -414,7 +416,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
@@ -454,7 +457,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
@@ -492,7 +496,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
@@ -532,7 +537,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
@@ -617,7 +623,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", features = ["span-locations"] }
@@ -657,7 +664,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
 linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
 mio = { version = "1", features = ["net", "os-ext"] }
 naga = { version = "25", features = ["spv-out", "wgsl-in"] }
-nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
+nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
+nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
 num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
 object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
 proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }