Add minidump crash reporting (#35263)

- [x] Handle uploading minidumps from the remote_server
- [x] Associate minidumps with panics with some sort of ID (we don't use
session_id on the remote)
  - [x] Update the protobufs and client/server code to request panics
- [x] Upload minidumps with no corresponding panic
- [x] Fill in panic info when there _is_ a corresponding panic
- [x] Use an env var for the sentry endpoint instead of hardcoding it

Release Notes:

- Zed now generates minidumps for crash reporting

---------

Co-authored-by: Max Brunsfeld <maxbrunsfeld@gmail.com>
This commit is contained in:
Julia Ryan 2025-08-04 20:19:42 -05:00 committed by GitHub
parent 07e3d53d58
commit 669c57b45f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
25 changed files with 709 additions and 135 deletions

View file

@ -24,6 +24,7 @@ env:
DIGITALOCEAN_SPACES_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }}
DIGITALOCEAN_SPACES_SECRET_KEY: ${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }}
ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
jobs:
job_spec:

View file

@ -29,6 +29,7 @@ jobs:
runs-on: ${{ matrix.system.runner }}
env:
ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
ZED_CLOUD_PROVIDER_ADDITIONAL_MODELS_JSON: ${{ secrets.ZED_CLOUD_PROVIDER_ADDITIONAL_MODELS_JSON }}
GIT_LFS_SKIP_SMUDGE: 1 # breaks the livekit rust sdk examples which we don't actually depend on
steps:

View file

@ -13,6 +13,7 @@ env:
CARGO_INCREMENTAL: 0
RUST_BACKTRACE: 1
ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
ZED_SENTRY_MINIDUMP_ENDPOINT: ${{ secrets.ZED_SENTRY_MINIDUMP_ENDPOINT }}
DIGITALOCEAN_SPACES_ACCESS_KEY: ${{ secrets.DIGITALOCEAN_SPACES_ACCESS_KEY }}
DIGITALOCEAN_SPACES_SECRET_KEY: ${{ secrets.DIGITALOCEAN_SPACES_SECRET_KEY }}

189
Cargo.lock generated
View file

@ -1172,7 +1172,7 @@ dependencies = [
"serde_json",
"serde_path_to_error",
"serde_qs 0.10.1",
"smart-default",
"smart-default 0.6.0",
"smol_str 0.1.24",
"thiserror 1.0.69",
"tokio",
@ -3927,6 +3927,42 @@ dependencies = [
"target-lexicon 0.13.2",
]
[[package]]
name = "crash-context"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "031ed29858d90cfdf27fe49fae28028a1f20466db97962fa2f4ea34809aeebf3"
dependencies = [
"cfg-if",
"libc",
"mach2",
]
[[package]]
name = "crash-handler"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2066907075af649bcb8bcb1b9b986329b243677e6918b2d920aa64b0aac5ace3"
dependencies = [
"cfg-if",
"crash-context",
"libc",
"mach2",
"parking_lot",
]
[[package]]
name = "crashes"
version = "0.1.0"
dependencies = [
"crash-handler",
"log",
"minidumper",
"paths",
"smol",
"workspace-hack",
]
[[package]]
name = "crc"
version = "3.2.1"
@ -4453,6 +4489,15 @@ dependencies = [
"zlog",
]
[[package]]
name = "debugid"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d"
dependencies = [
"uuid",
]
[[package]]
name = "deepseek"
version = "0.1.0"
@ -7235,6 +7280,17 @@ dependencies = [
"workspace-hack",
]
[[package]]
name = "goblin"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b363a30c165f666402fe6a3024d3bec7ebc898f96a4a23bd1c99f8dbf3f4f47"
dependencies = [
"log",
"plain",
"scroll",
]
[[package]]
name = "google_ai"
version = "0.1.0"
@ -7850,6 +7906,7 @@ dependencies = [
"http-body 1.0.1",
"log",
"parking_lot",
"reqwest 0.12.15 (git+https://github.com/zed-industries/reqwest.git?rev=951c770a32f1998d6e999cef3e59e0013e6c4415)",
"serde",
"serde_json",
"url",
@ -10080,6 +10137,63 @@ dependencies = [
"unicase",
]
[[package]]
name = "minidump-common"
version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c4d14bcca0fd3ed165a03000480aaa364c6860c34e900cb2dafdf3b95340e77"
dependencies = [
"bitflags 2.9.0",
"debugid",
"num-derive",
"num-traits",
"range-map",
"scroll",
"smart-default 0.7.1",
]
[[package]]
name = "minidump-writer"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abcd9c8a1e6e1e9d56ce3627851f39a17ea83e17c96bc510f29d7e43d78a7d"
dependencies = [
"bitflags 2.9.0",
"byteorder",
"cfg-if",
"crash-context",
"goblin",
"libc",
"log",
"mach2",
"memmap2",
"memoffset",
"minidump-common",
"nix 0.28.0",
"procfs-core",
"scroll",
"tempfile",
"thiserror 1.0.69",
]
[[package]]
name = "minidumper"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4ebc9d1f8847ec1d078f78b35ed598e0ebefa1f242d5f83cd8d7f03960a7d1"
dependencies = [
"cfg-if",
"crash-context",
"libc",
"log",
"minidump-writer",
"parking_lot",
"polling",
"scroll",
"thiserror 1.0.69",
"uds",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -12069,6 +12183,12 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "plain"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
[[package]]
name = "plist"
version = "1.7.1"
@ -12329,6 +12449,16 @@ dependencies = [
"yansi",
]
[[package]]
name = "procfs-core"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d3554923a69f4ce04c4a754260c338f505ce22642d3830e049a399fc2059a29"
dependencies = [
"bitflags 2.9.0",
"hex",
]
[[package]]
name = "prodash"
version = "29.0.2"
@ -12979,6 +13109,15 @@ dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "range-map"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12a5a2d6c7039059af621472a4389be1215a816df61aa4d531cfe85264aee95f"
dependencies = [
"num-traits",
]
[[package]]
name = "rangemap"
version = "1.5.1"
@ -13321,6 +13460,8 @@ dependencies = [
"clap",
"client",
"clock",
"crash-handler",
"crashes",
"dap",
"dap_adapters",
"debug_adapter_extension",
@ -13344,6 +13485,7 @@ dependencies = [
"libc",
"log",
"lsp",
"minidumper",
"node_runtime",
"paths",
"project",
@ -13532,6 +13674,7 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"once_cell",
"percent-encoding",
"pin-project-lite",
@ -14260,6 +14403,26 @@ dependencies = [
"once_cell",
]
[[package]]
name = "scroll"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ab8598aa408498679922eff7fa985c25d58a90771bd6be794434c5277eab1a6"
dependencies = [
"scroll_derive",
]
[[package]]
name = "scroll_derive"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1783eabc414609e28a5ba76aee5ddd52199f7107a0b24c2e9746a1ecc34a683d"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.101",
]
[[package]]
name = "scrypt"
version = "0.11.0"
@ -15005,6 +15168,17 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "smart-default"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.101",
]
[[package]]
name = "smol"
version = "2.0.2"
@ -17288,6 +17462,15 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "uds"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "885c31f06fce836457fe3ef09a59f83fe8db95d270b11cd78f40a4666c4d1661"
dependencies = [
"libc",
]
[[package]]
name = "uds_windows"
version = "1.1.0"
@ -19743,9 +19926,11 @@ dependencies = [
"lyon_path",
"md-5",
"memchr",
"mime_guess",
"miniz_oxide",
"mio 1.0.3",
"naga",
"nix 0.28.0",
"nix 0.29.0",
"nom",
"num-bigint",
@ -20217,6 +20402,7 @@ dependencies = [
"command_palette",
"component",
"copilot",
"crashes",
"dap",
"dap_adapters",
"db",
@ -20284,6 +20470,7 @@ dependencies = [
"release_channel",
"remote",
"repl",
"reqwest 0.12.15 (git+https://github.com/zed-industries/reqwest.git?rev=951c770a32f1998d6e999cef3e59e0013e6c4415)",
"reqwest_client",
"rope",
"search",

View file

@ -40,6 +40,7 @@ members = [
"crates/component",
"crates/context_server",
"crates/copilot",
"crates/crashes",
"crates/credentials_provider",
"crates/dap",
"crates/dap_adapters",
@ -266,6 +267,7 @@ command_palette_hooks = { path = "crates/command_palette_hooks" }
component = { path = "crates/component" }
context_server = { path = "crates/context_server" }
copilot = { path = "crates/copilot" }
crashes = { path = "crates/crashes" }
credentials_provider = { path = "crates/credentials_provider" }
dap = { path = "crates/dap" }
dap_adapters = { path = "crates/dap_adapters" }
@ -466,6 +468,7 @@ core-foundation = "0.10.0"
core-foundation-sys = "0.8.6"
core-video = { version = "0.4.3", features = ["metal"] }
cpal = "0.16"
crash-handler = "0.6"
criterion = { version = "0.5", features = ["html_reports"] }
ctor = "0.4.0"
dap-types = { git = "https://github.com/zed-industries/dap-types", rev = "1b461b310481d01e02b2603c16d7144b926339f8" }
@ -513,6 +516,7 @@ log = { version = "0.4.16", features = ["kv_unstable_serde", "serde"] }
lsp-types = { git = "https://github.com/zed-industries/lsp-types", rev = "39f629bdd03d59abd786ed9fc27e8bca02c0c0ec" }
markup5ever_rcdom = "0.3.0"
metal = "0.29"
minidumper = "0.8"
moka = { version = "0.12.10", features = ["sync"] }
naga = { version = "25.0", features = ["wgsl-in"] }
nanoid = "0.4"
@ -552,6 +556,7 @@ reqwest = { git = "https://github.com/zed-industries/reqwest.git", rev = "951c77
"charset",
"http2",
"macos-system-configuration",
"multipart",
"rustls-tls-native-roots",
"socks",
"stream",

View file

@ -74,6 +74,12 @@ static ZED_CLIENT_CHECKSUM_SEED: LazyLock<Option<Vec<u8>>> = LazyLock::new(|| {
})
});
pub static SENTRY_MINIDUMP_ENDPOINT: LazyLock<Option<String>> = LazyLock::new(|| {
option_env!("SENTRY_MINIDUMP_ENDPOINT")
.map(|s| s.to_owned())
.or_else(|| env::var("SENTRY_MINIDUMP_ENDPOINT").ok())
});
static DOTNET_PROJECT_FILES_REGEX: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(global\.json|Directory\.Build\.props|.*\.(csproj|fsproj|vbproj|sln))$").unwrap()
});

20
crates/crashes/Cargo.toml Normal file
View file

@ -0,0 +1,20 @@
[package]
name = "crashes"
version = "0.1.0"
publish.workspace = true
edition.workspace = true
license = "GPL-3.0-or-later"
[dependencies]
crash-handler.workspace = true
log.workspace = true
minidumper.workspace = true
paths.workspace = true
smol.workspace = true
workspace-hack.workspace = true
[lints]
workspace = true
[lib]
path = "src/crashes.rs"

1
crates/crashes/LICENSE-GPL Symbolic link
View file

@ -0,0 +1 @@
../../LICENSE-GPL

View file

@ -0,0 +1,172 @@
use crash_handler::CrashHandler;
use log::info;
use minidumper::{Client, LoopAction, MinidumpBinary};
use std::{
env,
fs::File,
io,
path::{Path, PathBuf},
process::{self, Command},
sync::{
OnceLock,
atomic::{AtomicBool, Ordering},
},
thread,
time::Duration,
};
// set once the crash handler has initialized and the client has connected to it
pub static CRASH_HANDLER: AtomicBool = AtomicBool::new(false);
// set when the first minidump request is made to avoid generating duplicate crash reports
pub static REQUESTED_MINIDUMP: AtomicBool = AtomicBool::new(false);
const CRASH_HANDLER_TIMEOUT: Duration = Duration::from_secs(60);
pub async fn init(id: String) {
let exe = env::current_exe().expect("unable to find ourselves");
let zed_pid = process::id();
// TODO: we should be able to get away with using 1 crash-handler process per machine,
// but for now we append the PID of the current process which makes it unique per remote
// server or interactive zed instance. This solves an issue where occasionally the socket
// used by the crash handler isn't destroyed correctly which causes it to stay on the file
// system and block further attempts to initialize crash handlers with that socket path.
let socket_name = paths::temp_dir().join(format!("zed-crash-handler-{zed_pid}"));
#[allow(unused)]
let server_pid = Command::new(exe)
.arg("--crash-handler")
.arg(&socket_name)
.spawn()
.expect("unable to spawn server process")
.id();
info!("spawning crash handler process");
let mut elapsed = Duration::ZERO;
let retry_frequency = Duration::from_millis(100);
let mut maybe_client = None;
while maybe_client.is_none() {
if let Ok(client) = Client::with_name(socket_name.as_path()) {
maybe_client = Some(client);
info!("connected to crash handler process after {elapsed:?}");
break;
}
elapsed += retry_frequency;
smol::Timer::after(retry_frequency).await;
}
let client = maybe_client.unwrap();
client.send_message(1, id).unwrap(); // set session id on the server
let client = std::sync::Arc::new(client);
let handler = crash_handler::CrashHandler::attach(unsafe {
let client = client.clone();
crash_handler::make_crash_event(move |crash_context: &crash_handler::CrashContext| {
// only request a minidump once
let res = if REQUESTED_MINIDUMP
.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed)
.is_ok()
{
client.send_message(2, "mistakes were made").unwrap();
client.ping().unwrap();
client.request_dump(crash_context).is_ok()
} else {
true
};
crash_handler::CrashEventResult::Handled(res)
})
})
.expect("failed to attach signal handler");
#[cfg(target_os = "linux")]
{
handler.set_ptracer(Some(server_pid));
}
CRASH_HANDLER.store(true, Ordering::Release);
std::mem::forget(handler);
info!("crash handler registered");
loop {
client.ping().ok();
smol::Timer::after(Duration::from_secs(10)).await;
}
}
pub struct CrashServer {
session_id: OnceLock<String>,
}
impl minidumper::ServerHandler for CrashServer {
fn create_minidump_file(&self) -> Result<(File, PathBuf), io::Error> {
let err_message = "Need to send a message with the ID upon starting the crash handler";
let dump_path = paths::logs_dir()
.join(self.session_id.get().expect(err_message))
.with_extension("dmp");
let file = File::create(&dump_path)?;
Ok((file, dump_path))
}
fn on_minidump_created(&self, result: Result<MinidumpBinary, minidumper::Error>) -> LoopAction {
match result {
Ok(mut md_bin) => {
use io::Write;
let _ = md_bin.file.flush();
info!("wrote minidump to disk {:?}", md_bin.path);
}
Err(e) => {
info!("failed to write minidump: {:#}", e);
}
}
LoopAction::Exit
}
fn on_message(&self, kind: u32, buffer: Vec<u8>) {
let message = String::from_utf8(buffer).expect("invalid utf-8");
info!("kind: {kind}, message: {message}",);
if kind == 1 {
self.session_id
.set(message)
.expect("session id already initialized");
}
}
fn on_client_disconnected(&self, clients: usize) -> LoopAction {
info!("client disconnected, {clients} remaining");
if clients == 0 {
LoopAction::Exit
} else {
LoopAction::Continue
}
}
}
pub fn handle_panic() {
// wait 500ms for the crash handler process to start up
// if it's still not there just write panic info and no minidump
let retry_frequency = Duration::from_millis(100);
for _ in 0..5 {
if CRASH_HANDLER.load(Ordering::Acquire) {
log::error!("triggering a crash to generate a minidump...");
#[cfg(target_os = "linux")]
CrashHandler.simulate_signal(crash_handler::Signal::Trap as u32);
#[cfg(not(target_os = "linux"))]
CrashHandler.simulate_exception(None);
break;
}
thread::sleep(retry_frequency);
}
}
pub fn crash_server(socket: &Path) {
let Ok(mut server) = minidumper::Server::with_name(socket) else {
log::info!("Couldn't create socket, there may already be a running crash server");
return;
};
let ab = AtomicBool::new(false);
server
.run(
Box::new(CrashServer {
session_id: OnceLock::new(),
}),
&ab,
Some(CRASH_HANDLER_TIMEOUT),
)
.expect("failed to run server");
}

View file

@ -24,6 +24,7 @@ http.workspace = true
http-body.workspace = true
log.workspace = true
parking_lot.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
url.workspace = true

View file

@ -88,6 +88,17 @@ impl From<&'static str> for AsyncBody {
}
}
impl TryFrom<reqwest::Body> for AsyncBody {
type Error = anyhow::Error;
fn try_from(value: reqwest::Body) -> Result<Self, Self::Error> {
value
.as_bytes()
.ok_or_else(|| anyhow::anyhow!("Underlying data is a stream"))
.map(|bytes| Self::from_bytes(Bytes::copy_from_slice(bytes)))
}
}
impl<T: Into<Self>> From<Option<T>> for AsyncBody {
fn from(body: Option<T>) -> Self {
match body {

View file

@ -7,7 +7,10 @@ use derive_more::Deref;
use http::HeaderValue;
pub use http::{self, Method, Request, Response, StatusCode, Uri};
use futures::future::BoxFuture;
use futures::{
FutureExt as _,
future::{self, BoxFuture},
};
use http::request::Builder;
use parking_lot::Mutex;
#[cfg(feature = "test-support")]
@ -89,6 +92,14 @@ pub trait HttpClient: 'static + Send + Sync {
fn as_fake(&self) -> &FakeHttpClient {
panic!("called as_fake on {}", type_name::<Self>())
}
fn send_multipart_form<'a>(
&'a self,
_url: &str,
_request: reqwest::multipart::Form,
) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
future::ready(Err(anyhow!("not implemented"))).boxed()
}
}
/// An [`HttpClient`] that may have a proxy.
@ -140,31 +151,13 @@ impl HttpClient for HttpClientWithProxy {
fn as_fake(&self) -> &FakeHttpClient {
self.client.as_fake()
}
}
impl HttpClient for Arc<HttpClientWithProxy> {
fn send(
&self,
req: Request<AsyncBody>,
) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
self.client.send(req)
}
fn user_agent(&self) -> Option<&HeaderValue> {
self.client.user_agent()
}
fn proxy(&self) -> Option<&Url> {
self.proxy.as_ref()
}
fn type_name(&self) -> &'static str {
self.client.type_name()
}
#[cfg(feature = "test-support")]
fn as_fake(&self) -> &FakeHttpClient {
self.client.as_fake()
fn send_multipart_form<'a>(
&'a self,
url: &str,
form: reqwest::multipart::Form,
) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
self.client.send_multipart_form(url, form)
}
}
@ -275,32 +268,6 @@ impl HttpClientWithUrl {
}
}
impl HttpClient for Arc<HttpClientWithUrl> {
fn send(
&self,
req: Request<AsyncBody>,
) -> BoxFuture<'static, anyhow::Result<Response<AsyncBody>>> {
self.client.send(req)
}
fn user_agent(&self) -> Option<&HeaderValue> {
self.client.user_agent()
}
fn proxy(&self) -> Option<&Url> {
self.client.proxy.as_ref()
}
fn type_name(&self) -> &'static str {
self.client.type_name()
}
#[cfg(feature = "test-support")]
fn as_fake(&self) -> &FakeHttpClient {
self.client.as_fake()
}
}
impl HttpClient for HttpClientWithUrl {
fn send(
&self,
@ -325,6 +292,14 @@ impl HttpClient for HttpClientWithUrl {
fn as_fake(&self) -> &FakeHttpClient {
self.client.as_fake()
}
fn send_multipart_form<'a>(
&'a self,
url: &str,
request: reqwest::multipart::Form,
) -> BoxFuture<'a, anyhow::Result<Response<AsyncBody>>> {
self.client.send_multipart_form(url, request)
}
}
pub fn read_proxy_from_env() -> Option<Url> {

View file

@ -79,11 +79,16 @@ message OpenServerSettings {
uint64 project_id = 1;
}
message GetPanicFiles {
message GetCrashFiles {
}
message GetPanicFilesResponse {
repeated string file_contents = 2;
message GetCrashFilesResponse {
repeated CrashReport crashes = 1;
}
message CrashReport {
optional string panic_contents = 1;
optional bytes minidump_contents = 2;
}
message Extension {

View file

@ -294,9 +294,6 @@ message Envelope {
GetPathMetadata get_path_metadata = 278;
GetPathMetadataResponse get_path_metadata_response = 279;
GetPanicFiles get_panic_files = 280;
GetPanicFilesResponse get_panic_files_response = 281;
CancelLanguageServerWork cancel_language_server_work = 282;
LspExtOpenDocs lsp_ext_open_docs = 283;
@ -402,7 +399,10 @@ message Envelope {
StashPop stash_pop = 358;
GetDefaultBranch get_default_branch = 359;
GetDefaultBranchResponse get_default_branch_response = 360; // current max
GetDefaultBranchResponse get_default_branch_response = 360;
GetCrashFiles get_crash_files = 361;
GetCrashFilesResponse get_crash_files_response = 362; // current max
}
reserved 87 to 88;
@ -423,6 +423,7 @@ message Envelope {
reserved 270;
reserved 247 to 254;
reserved 255 to 256;
reserved 280 to 281;
}
message Hello {

View file

@ -99,8 +99,8 @@ messages!(
(GetHoverResponse, Background),
(GetNotifications, Foreground),
(GetNotificationsResponse, Foreground),
(GetPanicFiles, Background),
(GetPanicFilesResponse, Background),
(GetCrashFiles, Background),
(GetCrashFilesResponse, Background),
(GetPathMetadata, Background),
(GetPathMetadataResponse, Background),
(GetPermalinkToLine, Foreground),
@ -462,7 +462,7 @@ request_messages!(
(ActivateToolchain, Ack),
(ActiveToolchain, ActiveToolchainResponse),
(GetPathMetadata, GetPathMetadataResponse),
(GetPanicFiles, GetPanicFilesResponse),
(GetCrashFiles, GetCrashFilesResponse),
(CancelLanguageServerWork, Ack),
(SyncExtensions, SyncExtensionsResponse),
(InstallExtension, Ack),

View file

@ -67,8 +67,11 @@ watch.workspace = true
worktree.workspace = true
[target.'cfg(not(windows))'.dependencies]
crashes.workspace = true
crash-handler.workspace = true
fork.workspace = true
libc.workspace = true
minidumper.workspace = true
[dev-dependencies]
assistant_tool.workspace = true

View file

@ -12,6 +12,10 @@ struct Cli {
/// by having Zed act like netcat communicating over a Unix socket.
#[arg(long, hide = true)]
askpass: Option<String>,
/// Used for recording minidumps on crashes by having the server run a separate
/// process communicating over a socket.
#[arg(long, hide = true)]
crash_handler: Option<PathBuf>,
/// Used for loading the environment from the project.
#[arg(long, hide = true)]
printenv: bool,
@ -58,6 +62,11 @@ fn main() {
return;
}
if let Some(socket) = &cli.crash_handler {
crashes::crash_server(socket.as_path());
return;
}
if cli.printenv {
util::shell_env::print_env();
return;

View file

@ -17,6 +17,7 @@ use node_runtime::{NodeBinaryOptions, NodeRuntime};
use paths::logs_dir;
use project::project_settings::ProjectSettings;
use proto::CrashReport;
use release_channel::{AppVersion, RELEASE_CHANNEL, ReleaseChannel};
use remote::proxy::ProxyLaunchError;
use remote::ssh_session::ChannelClient;
@ -33,6 +34,7 @@ use smol::io::AsyncReadExt;
use smol::Async;
use smol::{net::unix::UnixListener, stream::StreamExt as _};
use std::collections::HashMap;
use std::ffi::OsStr;
use std::ops::ControlFlow;
use std::str::FromStr;
@ -109,8 +111,9 @@ fn init_logging_server(log_file_path: PathBuf) -> Result<Receiver<Vec<u8>>> {
Ok(rx)
}
fn init_panic_hook() {
std::panic::set_hook(Box::new(|info| {
fn init_panic_hook(session_id: String) {
std::panic::set_hook(Box::new(move |info| {
crashes::handle_panic();
let payload = info
.payload()
.downcast_ref::<&str>()
@ -171,9 +174,11 @@ fn init_panic_hook() {
architecture: env::consts::ARCH.into(),
panicked_on: Utc::now().timestamp_millis(),
backtrace,
system_id: None, // Set on SSH client
installation_id: None, // Set on SSH client
session_id: "".to_string(), // Set on SSH client
system_id: None, // Set on SSH client
installation_id: None, // Set on SSH client
// used on this end to associate panics with minidumps, but will be replaced on the SSH client
session_id: session_id.clone(),
};
if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
@ -194,44 +199,69 @@ fn init_panic_hook() {
}));
}
fn handle_panic_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
fn handle_crash_files_requests(project: &Entity<HeadlessProject>, client: &Arc<ChannelClient>) {
let client: AnyProtoClient = client.clone().into();
client.add_request_handler(
project.downgrade(),
|_, _: TypedEnvelope<proto::GetPanicFiles>, _cx| async move {
|_, _: TypedEnvelope<proto::GetCrashFiles>, _cx| async move {
let mut crashes = Vec::new();
let mut minidumps_by_session_id = HashMap::new();
let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
let mut panic_files = Vec::new();
while let Some(child) = children.next().await {
let child = child?;
let child_path = child.path();
if child_path.extension() != Some(OsStr::new("panic")) {
continue;
let extension = child_path.extension();
if extension == Some(OsStr::new("panic")) {
let filename = if let Some(filename) = child_path.file_name() {
filename.to_string_lossy()
} else {
continue;
};
if !filename.starts_with("zed") {
continue;
}
let file_contents = smol::fs::read_to_string(&child_path)
.await
.context("error reading panic file")?;
crashes.push(proto::CrashReport {
panic_contents: Some(file_contents),
minidump_contents: None,
});
} else if extension == Some(OsStr::new("dmp")) {
let session_id = child_path.file_stem().unwrap().to_string_lossy();
minidumps_by_session_id
.insert(session_id.to_string(), smol::fs::read(&child_path).await?);
}
let filename = if let Some(filename) = child_path.file_name() {
filename.to_string_lossy()
} else {
continue;
};
if !filename.starts_with("zed") {
continue;
}
let file_contents = smol::fs::read_to_string(&child_path)
.await
.context("error reading panic file")?;
panic_files.push(file_contents);
// We've done what we can, delete the file
std::fs::remove_file(child_path)
smol::fs::remove_file(&child_path)
.await
.context("error removing panic")
.log_err();
}
anyhow::Ok(proto::GetPanicFilesResponse {
file_contents: panic_files,
})
for crash in &mut crashes {
let panic: telemetry_events::Panic =
serde_json::from_str(crash.panic_contents.as_ref().unwrap())?;
if let dump @ Some(_) = minidumps_by_session_id.remove(&panic.session_id) {
crash.minidump_contents = dump;
}
}
crashes.extend(
minidumps_by_session_id
.into_values()
.map(|dmp| CrashReport {
panic_contents: None,
minidump_contents: Some(dmp),
}),
);
anyhow::Ok(proto::GetCrashFilesResponse { crashes })
},
);
}
@ -409,7 +439,12 @@ pub fn execute_run(
ControlFlow::Continue(_) => {}
}
init_panic_hook();
let app = gpui::Application::headless();
let id = std::process::id().to_string();
app.background_executor()
.spawn(crashes::init(id.clone()))
.detach();
init_panic_hook(id);
let log_rx = init_logging_server(log_file)?;
log::info!(
"starting up. pid_file: {:?}, stdin_socket: {:?}, stdout_socket: {:?}, stderr_socket: {:?}",
@ -425,7 +460,7 @@ pub fn execute_run(
let listeners = ServerListeners::new(stdin_socket, stdout_socket, stderr_socket)?;
let git_hosting_provider_registry = Arc::new(GitHostingProviderRegistry::new());
gpui::Application::headless().run(move |cx| {
app.run(move |cx| {
settings::init(cx);
let app_version = AppVersion::load(env!("ZED_PKG_VERSION"));
release_channel::init(app_version, cx);
@ -486,7 +521,7 @@ pub fn execute_run(
)
});
handle_panic_requests(&project, &session);
handle_crash_files_requests(&project, &session);
cx.background_spawn(async move { cleanup_old_binaries() })
.detach();
@ -530,12 +565,15 @@ impl ServerPaths {
pub fn execute_proxy(identifier: String, is_reconnecting: bool) -> Result<()> {
init_logging_proxy();
init_panic_hook();
log::info!("starting proxy process. PID: {}", std::process::id());
let server_paths = ServerPaths::new(&identifier)?;
let id = std::process::id().to_string();
smol::spawn(crashes::init(id.clone())).detach();
init_panic_hook(id);
log::info!("starting proxy process. PID: {}", std::process::id());
let server_pid = check_pid_file(&server_paths.pid_file)?;
let server_running = server_pid.is_some();
if is_reconnecting {

View file

@ -4,14 +4,13 @@ use std::{any::type_name, borrow::Cow, mem, pin::Pin, task::Poll, time::Duration
use anyhow::anyhow;
use bytes::{BufMut, Bytes, BytesMut};
use futures::{AsyncRead, TryStreamExt as _};
use futures::{AsyncRead, FutureExt as _, TryStreamExt as _};
use http_client::{RedirectPolicy, Url, http};
use regex::Regex;
use reqwest::{
header::{HeaderMap, HeaderValue},
redirect,
};
use smol::future::FutureExt;
const DEFAULT_CAPACITY: usize = 4096;
static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
@ -274,6 +273,26 @@ impl http_client::HttpClient for ReqwestClient {
}
.boxed()
}
fn send_multipart_form<'a>(
&'a self,
url: &str,
form: reqwest::multipart::Form,
) -> futures::future::BoxFuture<'a, anyhow::Result<http_client::Response<http_client::AsyncBody>>>
{
let response = self.client.post(url).multipart(form).send();
self.handle
.spawn(async move {
let response = response.await?;
let mut builder = http::response::Builder::new().status(response.status());
for (k, v) in response.headers() {
builder = builder.header(k, v)
}
Ok(builder.body(response.bytes().await?.into())?)
})
.map(|e| e?)
.boxed()
}
}
#[cfg(test)]

View file

@ -45,6 +45,7 @@ collections.workspace = true
command_palette.workspace = true
component.workspace = true
copilot.workspace = true
crashes.workspace = true
dap_adapters.workspace = true
db.workspace = true
debug_adapter_extension.workspace = true
@ -117,6 +118,7 @@ recent_projects.workspace = true
release_channel.workspace = true
remote.workspace = true
repl.workspace = true
reqwest.workspace = true
reqwest_client.workspace = true
rope.workspace = true
search.workspace = true

View file

@ -172,6 +172,12 @@ pub fn main() {
let args = Args::parse();
// `zed --crash-handler` Makes zed operate in minidump crash handler mode
if let Some(socket) = &args.crash_handler {
crashes::crash_server(socket.as_path());
return;
}
// `zed --askpass` Makes zed operate in nc/netcat mode for use with askpass
if let Some(socket) = &args.askpass {
askpass::main(socket);
@ -264,6 +270,9 @@ pub fn main() {
let session_id = Uuid::new_v4().to_string();
let session = app.background_executor().block(Session::new());
app.background_executor()
.spawn(crashes::init(session_id.clone()))
.detach();
reliability::init_panic_hook(
app_version,
app_commit_sha.clone(),
@ -1185,6 +1194,11 @@ struct Args {
#[arg(long, hide = true)]
nc: Option<String>,
/// Used for recording minidumps on crashes by having Zed run a separate
/// process communicating over a socket.
#[arg(long, hide = true)]
crash_handler: Option<PathBuf>,
/// Run zed in the foreground, only used on Windows, to match the behavior on macOS.
#[arg(long)]
#[cfg(target_os = "windows")]

View file

@ -2,21 +2,32 @@ use crate::stdout_is_a_pty;
use anyhow::{Context as _, Result};
use backtrace::{self, Backtrace};
use chrono::Utc;
use client::{TelemetrySettings, telemetry};
use client::{
TelemetrySettings,
telemetry::{self, SENTRY_MINIDUMP_ENDPOINT},
};
use db::kvp::KEY_VALUE_STORE;
use futures::AsyncReadExt;
use gpui::{App, AppContext as _, SemanticVersion};
use http_client::{self, HttpClient, HttpClientWithUrl, HttpRequestExt, Method};
use paths::{crashes_dir, crashes_retired_dir};
use project::Project;
use release_channel::{AppCommitSha, RELEASE_CHANNEL, ReleaseChannel};
use reqwest::multipart::{Form, Part};
use settings::Settings;
use smol::stream::StreamExt;
use std::{
env,
ffi::{OsStr, c_void},
sync::{Arc, atomic::Ordering},
fs,
io::Write,
panic,
sync::{
Arc,
atomic::{AtomicU32, Ordering},
},
thread,
};
use std::{io::Write, panic, sync::atomic::AtomicU32, thread};
use telemetry_events::{LocationData, Panic, PanicRequest};
use url::Url;
use util::ResultExt;
@ -37,9 +48,10 @@ pub fn init_panic_hook(
if prior_panic_count > 0 {
// Give the panic-ing thread time to write the panic file
loop {
std::thread::yield_now();
thread::yield_now();
}
}
crashes::handle_panic();
let thread = thread::current();
let thread_name = thread.name().unwrap_or("<unnamed>");
@ -136,9 +148,8 @@ pub fn init_panic_hook(
if let Some(panic_data_json) = serde_json::to_string(&panic_data).log_err() {
let timestamp = chrono::Utc::now().format("%Y_%m_%d %H_%M_%S").to_string();
let panic_file_path = paths::logs_dir().join(format!("zed-{timestamp}.panic"));
let panic_file = std::fs::OpenOptions::new()
.append(true)
.create(true)
let panic_file = fs::OpenOptions::new()
.create_new(true)
.open(&panic_file_path)
.log_err();
if let Some(mut panic_file) = panic_file {
@ -205,27 +216,31 @@ pub fn init(
if let Some(ssh_client) = project.ssh_client() {
ssh_client.update(cx, |client, cx| {
if TelemetrySettings::get_global(cx).diagnostics {
let request = client.proto_client().request(proto::GetPanicFiles {});
let request = client.proto_client().request(proto::GetCrashFiles {});
cx.background_spawn(async move {
let panic_files = request.await?;
for file in panic_files.file_contents {
let panic: Option<Panic> = serde_json::from_str(&file)
.log_err()
.or_else(|| {
file.lines()
.next()
.and_then(|line| serde_json::from_str(line).ok())
})
.unwrap_or_else(|| {
log::error!("failed to deserialize panic file {:?}", file);
None
});
let crash_files = request.await?;
for crash in crash_files.crashes {
let mut panic: Option<Panic> = crash
.panic_contents
.and_then(|s| serde_json::from_str(&s).log_err());
if let Some(mut panic) = panic {
if let Some(panic) = panic.as_mut() {
panic.session_id = session_id.clone();
panic.system_id = system_id.clone();
panic.installation_id = installation_id.clone();
}
if let Some(minidump) = crash.minidump_contents {
upload_minidump(
http_client.clone(),
minidump.clone(),
panic.as_ref(),
)
.await
.log_err();
}
if let Some(panic) = panic {
upload_panic(&http_client, &panic_report_url, panic, &mut None)
.await?;
}
@ -510,6 +525,22 @@ async fn upload_previous_panics(
});
if let Some(panic) = panic {
let minidump_path = paths::logs_dir()
.join(&panic.session_id)
.with_extension("dmp");
if minidump_path.exists() {
let minidump = smol::fs::read(&minidump_path)
.await
.context("Failed to read minidump")?;
if upload_minidump(http.clone(), minidump, Some(&panic))
.await
.log_err()
.is_some()
{
fs::remove_file(minidump_path).ok();
}
}
if !upload_panic(&http, &panic_report_url, panic, &mut most_recent_panic).await? {
continue;
}
@ -517,13 +548,75 @@ async fn upload_previous_panics(
}
// We've done what we can, delete the file
std::fs::remove_file(child_path)
fs::remove_file(child_path)
.context("error removing panic")
.log_err();
}
// loop back over the directory again to upload any minidumps that are missing panics
let mut children = smol::fs::read_dir(paths::logs_dir()).await?;
while let Some(child) = children.next().await {
let child = child?;
let child_path = child.path();
if child_path.extension() != Some(OsStr::new("dmp")) {
continue;
}
if upload_minidump(
http.clone(),
smol::fs::read(&child_path)
.await
.context("Failed to read minidump")?,
None,
)
.await
.log_err()
.is_some()
{
fs::remove_file(child_path).ok();
}
}
Ok(most_recent_panic)
}
async fn upload_minidump(
http: Arc<HttpClientWithUrl>,
minidump: Vec<u8>,
panic: Option<&Panic>,
) -> Result<()> {
let sentry_upload_url = SENTRY_MINIDUMP_ENDPOINT
.to_owned()
.ok_or_else(|| anyhow::anyhow!("Minidump endpoint not set"))?;
let mut form = Form::new()
.part(
"upload_file_minidump",
Part::bytes(minidump)
.file_name("minidump.dmp")
.mime_str("application/octet-stream")?,
)
.text("platform", "rust");
if let Some(panic) = panic {
form = form.text(
"release",
format!("{}-{}", panic.release_channel, panic.app_version),
);
// TODO: tack on more fields
}
let mut response_text = String::new();
let mut response = http.send_multipart_form(&sentry_upload_url, form).await?;
response
.body_mut()
.read_to_string(&mut response_text)
.await?;
if !response.status().is_success() {
anyhow::bail!("failed to upload minidump: {response_text}");
}
log::info!("Uploaded minidump. event id: {response_text}");
Ok(())
}
async fn upload_panic(
http: &Arc<HttpClientWithUrl>,
panic_report_url: &Url,

View file

@ -6,6 +6,7 @@ When an app crashes,
- macOS creates a `.ips` file in `~/Library/Logs/DiagnosticReports`. You can view these using the built in Console app (`cmd-space Console`) under "Crash Reports".
- Linux creates a core dump. See the [man pages](https://man7.org/linux/man-pages/man5/core.5.html) for pointers to how your system might be configured to manage core dumps.
- Windows doesn't create crash reports by default, but can be configured to create "minidump" memory dumps upon applications crashing.
If you have enabled Zed's telemetry these will be uploaded to us when you restart the app. They end up in a [Slack channel (internal only)](https://zed-industries.slack.com/archives/C04S6T1T7TQ).

View file

@ -21,7 +21,7 @@ The telemetry settings can also be configured via the welcome screen, which can
Telemetry is sent from the application to our servers. Data is proxied through our servers to enable us to easily switch analytics services. We currently use:
- [Axiom](https://axiom.co): Cloud-monitoring service - stores diagnostic events
- [Sentry](https://sentry.io): Crash-monitoring service - stores diagnostic events
- [Snowflake](https://snowflake.com): Data warehouse - stores both diagnostic and metric events
- [Hex](https://www.hex.tech): Dashboards and data exploration - accesses data stored in Snowflake
- [Amplitude](https://www.amplitude.com): Dashboards and data exploration - accesses data stored in Snowflake
@ -30,9 +30,9 @@ Telemetry is sent from the application to our servers. Data is proxied through o
### Diagnostics
Diagnostic events include debug information (stack traces) from crash reports. Reports are sent on the first application launch after the crash occurred. We've built dashboards that allow us to visualize the frequency and severity of issues experienced by users. Having these reports sent automatically allows us to begin implementing fixes without the user needing to file a report in our issue tracker. The plots in the dashboards also give us an informal measurement of the stability of Zed.
Crash reports consist of a [minidump](https://learn.microsoft.com/en-us/windows/win32/debug/minidump-files) and some extra debug information. Reports are sent on the first application launch after the crash occurred. We've built dashboards that allow us to visualize the frequency and severity of issues experienced by users. Having these reports sent automatically allows us to begin implementing fixes without the user needing to file a report in our issue tracker. The plots in the dashboards also give us an informal measurement of the stability of Zed.
You can see what data is sent when a panic occurs by inspecting the `Panic` struct in [crates/telemetry_events/src/telemetry_events.rs](https://github.com/zed-industries/zed/blob/main/crates/telemetry_events/src/telemetry_events.rs) in the Zed repo. You can find additional information in the [Debugging Crashes](./development/debugging-crashes.md) documentation.
You can see what extra data is sent alongside the minidump in the `Panic` struct in [crates/telemetry_events/src/telemetry_events.rs](https://github.com/zed-industries/zed/blob/main/crates/telemetry_events/src/telemetry_events.rs) in the Zed repo. You can find additional information in the [Debugging Crashes](./development/debugging-crashes.md) documentation.
### Client-Side Usage Data {#client-metrics}

View file

@ -82,6 +82,7 @@ lyon = { version = "1", default-features = false, features = ["extra"] }
lyon_path = { version = "1" }
md-5 = { version = "0.10" }
memchr = { version = "2" }
mime_guess = { version = "2" }
miniz_oxide = { version = "0.8", features = ["simd"] }
nom = { version = "7" }
num-bigint = { version = "0.4" }
@ -212,6 +213,7 @@ lyon = { version = "1", default-features = false, features = ["extra"] }
lyon_path = { version = "1" }
md-5 = { version = "0.10" }
memchr = { version = "2" }
mime_guess = { version = "2" }
miniz_oxide = { version = "0.8", features = ["simd"] }
nom = { version = "7" }
num-bigint = { version = "0.4" }
@ -290,7 +292,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@ -318,7 +320,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@ -347,7 +349,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@ -375,7 +377,7 @@ gimli = { version = "0.31", default-features = false, features = ["read", "std",
hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring", "tls12"] }
itertools-5ef9efb8ec2df382 = { package = "itertools", version = "0.12" }
naga = { version = "25", features = ["msl-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "user"] }
objc2 = { version = "0.6" }
objc2-core-foundation = { version = "0.3", default-features = false, features = ["CFArray", "CFCGTypes", "CFData", "CFDate", "CFDictionary", "CFRunLoop", "CFString", "CFURL", "objc2", "std"] }
objc2-foundation = { version = "0.3", default-features = false, features = ["NSArray", "NSAttributedString", "NSBundle", "NSCoder", "NSData", "NSDate", "NSDictionary", "NSEnumerator", "NSError", "NSGeometry", "NSNotification", "NSNull", "NSObjCRuntime", "NSObject", "NSProcessInfo", "NSRange", "NSRunLoop", "NSString", "NSURL", "NSUndoManager", "NSValue", "objc2-core-foundation", "std"] }
@ -414,7 +416,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
@ -454,7 +457,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
@ -492,7 +496,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
@ -532,7 +537,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }
@ -617,7 +623,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", features = ["span-locations"] }
@ -657,7 +664,8 @@ linux-raw-sys-274715c4dabd11b0 = { package = "linux-raw-sys", version = "0.9", d
linux-raw-sys-9fbad63c4bcf4a8f = { package = "linux-raw-sys", version = "0.4", default-features = false, features = ["elf", "errno", "general", "if_ether", "ioctl", "net", "netlink", "no_std", "prctl", "system", "xdp"] }
mio = { version = "1", features = ["net", "os-ext"] }
naga = { version = "25", features = ["spv-out", "wgsl-in"] }
nix = { version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
nix-1f5adca70f036a62 = { package = "nix", version = "0.28", features = ["fs", "mman", "ptrace", "signal", "term", "user"] }
nix-b73a96c0a5f6a7d9 = { package = "nix", version = "0.29", features = ["fs", "pthread", "signal", "socket", "uio", "user"] }
num-bigint-dig = { version = "0.8", features = ["i128", "prime", "zeroize"] }
object = { version = "0.36", default-features = false, features = ["archive", "read_core", "unaligned", "write"] }
proc-macro2 = { version = "1", default-features = false, features = ["span-locations"] }