Add ./script/symbolicate (#8165)

This lets you get a readable backtrace from an .ips file of a crash
report.

Release Notes:

- N/A

crates/zed/src/main.rs

@@ -681,10 +681,11 @@ fn upload_panics_and_crashes(http: Arc<ZedHttpClient>, cx: &mut AppContext) {
     let telemetry_settings = *client::TelemetrySettings::get_global(cx);
     cx.background_executor()
         .spawn(async move {
-            upload_previous_panics(http.clone(), telemetry_settings)
+            let most_recent_panic = upload_previous_panics(http.clone(), telemetry_settings)
                 .await
-                .log_err();
-            upload_previous_crashes(http, telemetry_settings)
+                .log_err()
+                .flatten();
+            upload_previous_crashes(http, most_recent_panic, telemetry_settings)
                 .await
                 .log_err()
         })
@@ -695,9 +696,12 @@ fn upload_panics_and_crashes(http: Arc<ZedHttpClient>, cx: &mut AppContext) {
 async fn upload_previous_panics(
     http: Arc<ZedHttpClient>,
     telemetry_settings: client::TelemetrySettings,
-) -> Result<()> {
+) -> Result<Option<(i64, String)>> {
     let panic_report_url = http.zed_url("/api/panic");
     let mut children = smol::fs::read_dir(&*paths::LOGS_DIR).await?;
+
+    let mut most_recent_panic = None;
+
     while let Some(child) = children.next().await {
         let child = child?;
         let child_path = child.path();
@@ -720,7 +724,7 @@ async fn upload_previous_panics(
                 .await
                 .context("error reading panic file")?;
 
-            let panic = serde_json::from_str(&panic_file_content)
+            let panic: Option<Panic> = serde_json::from_str(&panic_file_content)
                 .ok()
                 .or_else(|| {
                     panic_file_content
@@ -734,6 +738,8 @@ async fn upload_previous_panics(
                 });
 
             if let Some(panic) = panic {
+                most_recent_panic = Some((panic.panicked_on, panic.payload.clone()));
+
                 let body = serde_json::to_string(&PanicRequest { panic }).unwrap();
 
                 let request = Request::post(&panic_report_url)
@@ -752,7 +758,7 @@ async fn upload_previous_panics(
             .context("error removing panic")
             .log_err();
     }
-    Ok::<_, anyhow::Error>(())
+    Ok::<_, anyhow::Error>(most_recent_panic)
 }
 
 static LAST_CRASH_UPLOADED: &'static str = "LAST_CRASH_UPLOADED";
@@ -761,6 +767,7 @@ static LAST_CRASH_UPLOADED: &'static str = "LAST_CRASH_UPLOADED";
 /// (only if telemetry is enabled)
 async fn upload_previous_crashes(
     http: Arc<ZedHttpClient>,
+    most_recent_panic: Option<(i64, String)>,
     telemetry_settings: client::TelemetrySettings,
 ) -> Result<()> {
     if !telemetry_settings.diagnostics {
@@ -797,10 +804,17 @@ async fn upload_previous_crashes(
                 .await
                 .context("error reading crash file")?;
 
-            let request = Request::post(&crash_report_url)
+            let mut request = Request::post(&crash_report_url)
                 .redirect_policy(isahc::config::RedirectPolicy::Follow)
-                .header("Content-Type", "text/plain")
-                .body(body.into())?;
+                .header("Content-Type", "text/plain");
+
+            if let Some((panicked_on, payload)) = most_recent_panic.as_ref() {
+                request = request
+                    .header("x-zed-panicked-on", format!("{}", panicked_on))
+                    .header("x-zed-panic", payload)
+            }
+
+            let request = request.body(body.into())?;
 
             let response = http.send(request).await.context("error sending crash")?;
             if !response.status().is_success() {

docs/src/developing_zed__debugging_crashes.md

@@ -0,0 +1,27 @@
+## Crashes
+
+When an app crashes, macOS creates a `.ips` file in `~/Library/Logs/DiagnosticReports`. You can view these using the built in Console app (`cmd-space Console`) under "Crash Reports".
+
+If you have enabeld Zed's telemetry these will be uploaded to us when you restart the app. They end up in Datadog, and a [Slack channel (internal only)](https://zed-industries.slack.com/archives/C04S6T1T7TQ).
+
+These crash reports are generated by the crashing binary, and contain a wealth of information; but they are hard to read for a few reasons:
+
+- They don't contain source files and line numbers
+- The symbols are [mangled](https://doc.rust-lang.org/rustc/symbol-mangling/index.html)
+- Inlined functions are elided
+
+To get a better sense of the backtrace of a crash you can download the `.ips` file locally and run:
+
+```
+./script/symbolicate ~/path/zed-XXX-XXX.ips
+```
+
+This will download the correct debug symbols from our public [digital ocean bucket](https://zed-debug-symbols.nyc3.digitaloceanspaces.com), and run [symbolicate](https://crates.io/crates/symbolicate) for you.
+
+The output contains the source file and line number, and the demangled symbol information for every inlined frame.
+
+## Panics
+
+When the app panics at the rust level, Zed creates a file in `~/Library/Logs/Zed` with the text of the panic, and a summary of the backtrace. On boot, if you have telemetry enabled, we upload these panics so we can keep track of them.
+
+A panic is also considered a crash, and so for most panics we get both the crash report and the panic.

script/symbolicate

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -eu
+if [[ $# -eq 0 ]] || [[ "$1" == "--help" ]]; then
+  echo "Usage: $(basename $0) <path_to_ips_file>"
+  echo "This script symbolicates the provided .ips file using the appropriate dSYM file from digital ocean"
+  echo ""
+  exit 1
+fi
+
+ips_file=$1;
+
+version=$(cat $ips_file | head -n 1 | jq -r .app_version)
+bundle_id=$(cat $ips_file | head -n 1 | jq -r .bundleID)
+cpu_type=$(cat $ips_file | tail -n+2 | jq -r .cpuType)
+
+which symbolicate >/dev/null || cargo install symbolicate
+
+arch="x86_64-apple-darwin"
+if [[ "$cpu_type" == *ARM-64* ]]; then
+    arch="aarch64-apple-darwin"
+fi
+
+channel="stable"
+if [[ "$bundle_id" == *nightly* ]]; then
+    channel="nightly"
+elif [[ "$bundle_id" == *preview* ]]; then
+    channel="preview"
+fi
+
+mkdir -p target/dsyms/$channel
+
+dsym="$channel/Zed-$version-$arch.dwarf"
+if [[ ! -f target/dsyms/$dsym ]]; then
+    echo "Downloading $dsym..."
+    curl -o target/dsyms/$dsym.gz "https://zed-debug-symbols.nyc3.digitaloceanspaces.com/$channel/Zed-$version-$arch.dwarf.gz"
+    gunzip  target/dsyms/$dsym.gz
+fi
+
+symbolicate $ips_file target/dsyms/$dsym