Add telemetry events for loading extensions (#9793)

* Store extensions versions' wasm API version in the database
* Share a common struct for extension API responses between collab and
client
* Add wasm API version and schema version to extension API responses

Release Notes:

- N/A

Co-authored-by: Marshall <marshall@zed.dev>

crates/extension/Cargo.toml

@@ -22,6 +22,7 @@ async-compression.workspace = true
 async-tar.workspace = true
 async-trait.workspace = true
 cap-std.workspace = true
+client.workspace = true
 collections.workspace = true
 fs.workspace = true
 futures.workspace = true

crates/extension/src/extension_builder.rs

@@ -1,3 +1,4 @@
+use crate::wasm_host::parse_wasm_extension_version;
 use crate::ExtensionManifest;
 use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry};
 use anyhow::{anyhow, bail, Context as _, Result};
@@ -73,9 +74,11 @@ impl ExtensionBuilder {
     pub async fn compile_extension(
         &self,
         extension_dir: &Path,
-        extension_manifest: &ExtensionManifest,
+        extension_manifest: &mut ExtensionManifest,
         options: CompileExtensionOptions,
     ) -> Result<()> {
+        populate_defaults(extension_manifest, &extension_dir)?;
+
         if extension_dir.is_relative() {
             bail!(
                 "extension dir {} is not an absolute path",
@@ -85,12 +88,9 @@ impl ExtensionBuilder {
 
         fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?;
 
-        let cargo_toml_path = extension_dir.join("Cargo.toml");
-        if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)
-            || fs::metadata(&cargo_toml_path).map_or(false, |stat| stat.is_file())
-        {
+        if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) {
             log::info!("compiling Rust extension {}", extension_dir.display());
-            self.compile_rust_extension(extension_dir, options)
+            self.compile_rust_extension(extension_dir, extension_manifest, options)
                 .await
                 .context("failed to compile Rust extension")?;
         }
@@ -108,6 +108,7 @@ impl ExtensionBuilder {
     async fn compile_rust_extension(
         &self,
         extension_dir: &Path,
+        manifest: &mut ExtensionManifest,
         options: CompileExtensionOptions,
     ) -> Result<(), anyhow::Error> {
         self.install_rust_wasm_target_if_needed()?;
@@ -162,6 +163,11 @@ impl ExtensionBuilder {
             .strip_custom_sections(&component_bytes)
             .context("failed to strip debug sections from wasm component")?;
 
+        let wasm_extension_api_version =
+            parse_wasm_extension_version(&manifest.id, &component_bytes)
+                .context("compiled wasm did not contain a valid zed extension api version")?;
+        manifest.lib.version = Some(wasm_extension_api_version);
+
         fs::write(extension_dir.join("extension.wasm"), &component_bytes)
             .context("failed to write extension.wasm")?;
 
@@ -469,3 +475,86 @@ impl ExtensionBuilder {
         Ok(output)
     }
 }
+
+fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
+    // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing
+    // contents of the computed fields, since we don't care what the existing values are.
+    if manifest.schema_version == 0 {
+        manifest.languages.clear();
+        manifest.grammars.clear();
+        manifest.themes.clear();
+    }
+
+    let cargo_toml_path = extension_path.join("Cargo.toml");
+    if cargo_toml_path.exists() {
+        manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
+    }
+
+    let languages_dir = extension_path.join("languages");
+    if languages_dir.exists() {
+        for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
+            let entry = entry?;
+            let language_dir = entry.path();
+            let config_path = language_dir.join("config.toml");
+            if config_path.exists() {
+                let relative_language_dir =
+                    language_dir.strip_prefix(extension_path)?.to_path_buf();
+                if !manifest.languages.contains(&relative_language_dir) {
+                    manifest.languages.push(relative_language_dir);
+                }
+            }
+        }
+    }
+
+    let themes_dir = extension_path.join("themes");
+    if themes_dir.exists() {
+        for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
+            let entry = entry?;
+            let theme_path = entry.path();
+            if theme_path.extension() == Some("json".as_ref()) {
+                let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
+                if !manifest.themes.contains(&relative_theme_path) {
+                    manifest.themes.push(relative_theme_path);
+                }
+            }
+        }
+    }
+
+    // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in
+    // the manifest using the contents of the `grammars` directory.
+    if manifest.schema_version == 0 {
+        let grammars_dir = extension_path.join("grammars");
+        if grammars_dir.exists() {
+            for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? {
+                let entry = entry?;
+                let grammar_path = entry.path();
+                if grammar_path.extension() == Some("toml".as_ref()) {
+                    #[derive(Deserialize)]
+                    struct GrammarConfigToml {
+                        pub repository: String,
+                        pub commit: String,
+                    }
+
+                    let grammar_config = fs::read_to_string(&grammar_path)?;
+                    let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?;
+
+                    let grammar_name = grammar_path
+                        .file_stem()
+                        .and_then(|stem| stem.to_str())
+                        .ok_or_else(|| anyhow!("no grammar name"))?;
+                    if !manifest.grammars.contains_key(grammar_name) {
+                        manifest.grammars.insert(
+                            grammar_name.into(),
+                            GrammarManifestEntry {
+                                repository: grammar_config.repository,
+                                rev: grammar_config.commit,
+                            },
+                        );
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(())
+}

crates/extension/src/extension_manifest.rs

@@ -1,7 +1,14 @@
+use anyhow::{anyhow, Context, Result};
 use collections::BTreeMap;
+use fs::Fs;
 use language::LanguageServerName;
 use serde::{Deserialize, Serialize};
-use std::{path::PathBuf, sync::Arc};
+use std::{
+    ffi::OsStr,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+use util::SemanticVersion;
 
 /// This is the old version of the extension manifest, from when it was `extension.json`.
 #[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@@ -53,6 +60,7 @@ pub struct ExtensionManifest {
 #[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)]
 pub struct LibManifestEntry {
     pub kind: Option<ExtensionLibraryKind>,
+    pub version: Option<SemanticVersion>,
 }
 
 #[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
@@ -71,3 +79,68 @@ pub struct GrammarManifestEntry {
 pub struct LanguageServerManifestEntry {
     pub language: Arc<str>,
 }
+
+impl ExtensionManifest {
+    pub async fn load(fs: Arc<dyn Fs>, extension_dir: &Path) -> Result<Self> {
+        let extension_name = extension_dir
+            .file_name()
+            .and_then(OsStr::to_str)
+            .ok_or_else(|| anyhow!("invalid extension name"))?;
+
+        let mut extension_manifest_path = extension_dir.join("extension.json");
+        if fs.is_file(&extension_manifest_path).await {
+            let manifest_content = fs
+                .load(&extension_manifest_path)
+                .await
+                .with_context(|| format!("failed to load {extension_name} extension.json"))?;
+            let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
+                .with_context(|| {
+                    format!("invalid extension.json for extension {extension_name}")
+                })?;
+
+            Ok(manifest_from_old_manifest(manifest_json, extension_name))
+        } else {
+            extension_manifest_path.set_extension("toml");
+            let manifest_content = fs
+                .load(&extension_manifest_path)
+                .await
+                .with_context(|| format!("failed to load {extension_name} extension.toml"))?;
+            toml::from_str(&manifest_content)
+                .with_context(|| format!("invalid extension.json for extension {extension_name}"))
+        }
+    }
+}
+
+fn manifest_from_old_manifest(
+    manifest_json: OldExtensionManifest,
+    extension_id: &str,
+) -> ExtensionManifest {
+    ExtensionManifest {
+        id: extension_id.into(),
+        name: manifest_json.name,
+        version: manifest_json.version,
+        description: manifest_json.description,
+        repository: manifest_json.repository,
+        authors: manifest_json.authors,
+        schema_version: 0,
+        lib: Default::default(),
+        themes: {
+            let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
+            themes.sort();
+            themes.dedup();
+            themes
+        },
+        languages: {
+            let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
+            languages.sort();
+            languages.dedup();
+            languages
+        },
+        grammars: manifest_json
+            .grammars
+            .into_keys()
+            .map(|grammar_name| (grammar_name, Default::default()))
+            .collect(),
+        language_servers: Default::default(),
+    }
+}

crates/extension/src/extension_store.rs

@@ -10,6 +10,7 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit};
 use anyhow::{anyhow, bail, Context as _, Result};
 use async_compression::futures::bufread::GzipDecoder;
 use async_tar::Archive;
+use client::{telemetry::Telemetry, Client};
 use collections::{hash_map, BTreeMap, HashMap, HashSet};
 use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
 use fs::{Fs, RemoveOptions};
@@ -30,7 +31,6 @@ use node_runtime::NodeRuntime;
 use serde::{Deserialize, Serialize};
 use std::{
     cmp::Ordering,
-    ffi::OsStr,
     path::{self, Path, PathBuf},
     sync::Arc,
     time::{Duration, Instant},
@@ -75,6 +75,7 @@ pub struct ExtensionStore {
     extension_index: ExtensionIndex,
     fs: Arc<dyn Fs>,
     http_client: Arc<HttpClientWithUrl>,
+    telemetry: Option<Arc<Telemetry>>,
     reload_tx: UnboundedSender<Option<Arc<str>>>,
     reload_complete_senders: Vec<oneshot::Sender<()>>,
     installed_dir: PathBuf,
@@ -149,7 +150,7 @@ actions!(zed, [ReloadExtensions]);
 
 pub fn init(
     fs: Arc<fs::RealFs>,
-    http_client: Arc<HttpClientWithUrl>,
+    client: Arc<Client>,
     node_runtime: Arc<dyn NodeRuntime>,
     language_registry: Arc<LanguageRegistry>,
     theme_registry: Arc<ThemeRegistry>,
@@ -160,7 +161,8 @@ pub fn init(
             EXTENSIONS_DIR.clone(),
             None,
             fs,
-            http_client,
+            client.http_client().clone(),
+            Some(client.telemetry().clone()),
             node_runtime,
             language_registry,
             theme_registry,
@@ -187,6 +189,7 @@ impl ExtensionStore {
         build_dir: Option<PathBuf>,
         fs: Arc<dyn Fs>,
         http_client: Arc<HttpClientWithUrl>,
+        telemetry: Option<Arc<Telemetry>>,
         node_runtime: Arc<dyn NodeRuntime>,
         language_registry: Arc<LanguageRegistry>,
         theme_registry: Arc<ThemeRegistry>,
@@ -216,6 +219,7 @@ impl ExtensionStore {
             wasm_extensions: Vec::new(),
             fs,
             http_client,
+            telemetry,
             language_registry,
             theme_registry,
             reload_tx,
@@ -587,8 +591,8 @@ impl ExtensionStore {
         let builder = self.builder.clone();
 
         cx.spawn(move |this, mut cx| async move {
-            let extension_manifest =
-                Self::load_extension_manifest(fs.clone(), &extension_source_path).await?;
+            let mut extension_manifest =
+                ExtensionManifest::load(fs.clone(), &extension_source_path).await?;
             let extension_id = extension_manifest.id.clone();
 
             if !this.update(&mut cx, |this, cx| {
@@ -622,7 +626,7 @@ impl ExtensionStore {
                         builder
                             .compile_extension(
                                 &extension_source_path,
-                                &extension_manifest,
+                                &mut extension_manifest,
                                 CompileExtensionOptions { release: false },
                             )
                             .await
@@ -667,9 +671,13 @@ impl ExtensionStore {
 
         cx.notify();
         let compile = cx.background_executor().spawn(async move {
-            let manifest = Self::load_extension_manifest(fs, &path).await?;
+            let mut manifest = ExtensionManifest::load(fs, &path).await?;
             builder
-                .compile_extension(&path, &manifest, CompileExtensionOptions { release: true })
+                .compile_extension(
+                    &path,
+                    &mut manifest,
+                    CompileExtensionOptions { release: true },
+                )
                 .await
         });
 
@@ -759,6 +767,17 @@ impl ExtensionStore {
             extensions_to_unload.len() - reload_count
         );
 
+        if let Some(telemetry) = &self.telemetry {
+            for extension_id in &extensions_to_load {
+                if let Some(extension) = self.extension_index.extensions.get(extension_id) {
+                    telemetry.report_extension_event(
+                        extension_id.clone(),
+                        extension.manifest.version.clone(),
+                    );
+                }
+            }
+        }
+
         let themes_to_remove = old_index
             .themes
             .iter()
@@ -908,7 +927,9 @@ impl ExtensionStore {
                             cx.background_executor().clone(),
                         )
                         .await
-                        .context("failed to load wasm extension")
+                        .with_context(|| {
+                            format!("failed to load wasm extension {}", extension.manifest.id)
+                        })
                 })
                 .await;
 
@@ -989,8 +1010,7 @@ impl ExtensionStore {
         extension_dir: PathBuf,
         index: &mut ExtensionIndex,
     ) -> Result<()> {
-        let mut extension_manifest =
-            Self::load_extension_manifest(fs.clone(), &extension_dir).await?;
+        let mut extension_manifest = ExtensionManifest::load(fs.clone(), &extension_dir).await?;
         let extension_id = extension_manifest.id.clone();
 
         // TODO: distinguish dev extensions more explicitly, by the absence
@@ -1082,72 +1102,6 @@ impl ExtensionStore {
 
         Ok(())
     }
-
-    pub async fn load_extension_manifest(
-        fs: Arc<dyn Fs>,
-        extension_dir: &Path,
-    ) -> Result<ExtensionManifest> {
-        let extension_name = extension_dir
-            .file_name()
-            .and_then(OsStr::to_str)
-            .ok_or_else(|| anyhow!("invalid extension name"))?;
-
-        let mut extension_manifest_path = extension_dir.join("extension.json");
-        if fs.is_file(&extension_manifest_path).await {
-            let manifest_content = fs
-                .load(&extension_manifest_path)
-                .await
-                .with_context(|| format!("failed to load {extension_name} extension.json"))?;
-            let manifest_json = serde_json::from_str::<OldExtensionManifest>(&manifest_content)
-                .with_context(|| {
-                    format!("invalid extension.json for extension {extension_name}")
-                })?;
-
-            Ok(manifest_from_old_manifest(manifest_json, extension_name))
-        } else {
-            extension_manifest_path.set_extension("toml");
-            let manifest_content = fs
-                .load(&extension_manifest_path)
-                .await
-                .with_context(|| format!("failed to load {extension_name} extension.toml"))?;
-            toml::from_str(&manifest_content)
-                .with_context(|| format!("invalid extension.json for extension {extension_name}"))
-        }
-    }
-}
-
-fn manifest_from_old_manifest(
-    manifest_json: OldExtensionManifest,
-    extension_id: &str,
-) -> ExtensionManifest {
-    ExtensionManifest {
-        id: extension_id.into(),
-        name: manifest_json.name,
-        version: manifest_json.version,
-        description: manifest_json.description,
-        repository: manifest_json.repository,
-        authors: manifest_json.authors,
-        schema_version: 0,
-        lib: Default::default(),
-        themes: {
-            let mut themes = manifest_json.themes.into_values().collect::<Vec<_>>();
-            themes.sort();
-            themes.dedup();
-            themes
-        },
-        languages: {
-            let mut languages = manifest_json.languages.into_values().collect::<Vec<_>>();
-            languages.sort();
-            languages.dedup();
-            languages
-        },
-        grammars: manifest_json
-            .grammars
-            .into_keys()
-            .map(|grammar_name| (grammar_name, Default::default()))
-            .collect(),
-        language_servers: Default::default(),
-    }
 }
 
 fn load_plugin_queries(root_path: &Path) -> LanguageQueries {

crates/extension/src/extension_store_test.rs

@@ -262,6 +262,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
             None,
             fs.clone(),
             http_client.clone(),
+            None,
             node_runtime.clone(),
             language_registry.clone(),
             theme_registry.clone(),
@@ -381,6 +382,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
             None,
             fs.clone(),
             http_client.clone(),
+            None,
             node_runtime.clone(),
             language_registry.clone(),
             theme_registry.clone(),
@@ -538,6 +540,7 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
             Some(cache_dir),
             fs.clone(),
             http_client.clone(),
+            None,
             node_runtime,
             language_registry.clone(),
             theme_registry.clone(),

crates/extension/src/wasm_host.rs

@@ -40,7 +40,7 @@ pub struct WasmExtension {
     tx: UnboundedSender<ExtensionCall>,
     pub(crate) manifest: Arc<ExtensionManifest>,
     #[allow(unused)]
-    zed_api_version: SemanticVersion,
+    pub zed_api_version: SemanticVersion,
 }
 
 pub(crate) struct WasmState {
@@ -93,29 +93,11 @@ impl WasmHost {
     ) -> impl 'static + Future<Output = Result<WasmExtension>> {
         let this = self.clone();
         async move {
+            let zed_api_version = parse_wasm_extension_version(&manifest.id, &wasm_bytes)?;
+
             let component = Component::from_binary(&this.engine, &wasm_bytes)
                 .context("failed to compile wasm component")?;
 
-            let mut zed_api_version = None;
-            for part in wasmparser::Parser::new(0).parse_all(&wasm_bytes) {
-                if let wasmparser::Payload::CustomSection(s) = part? {
-                    if s.name() == "zed:api-version" {
-                        zed_api_version = parse_extension_version(s.data());
-                        if zed_api_version.is_none() {
-                            bail!(
-                                "extension {} has invalid zed:api-version section: {:?}",
-                                manifest.id,
-                                s.data()
-                            );
-                        }
-                    }
-                }
-            }
-
-            let Some(zed_api_version) = zed_api_version else {
-                bail!("extension {} has no zed:api-version section", manifest.id);
-            };
-
             let mut store = wasmtime::Store::new(
                 &this.engine,
                 WasmState {
@@ -196,7 +178,30 @@ impl WasmHost {
     }
 }
 
-fn parse_extension_version(data: &[u8]) -> Option<SemanticVersion> {
+pub fn parse_wasm_extension_version(
+    extension_id: &str,
+    wasm_bytes: &[u8],
+) -> Result<SemanticVersion> {
+    for part in wasmparser::Parser::new(0).parse_all(wasm_bytes) {
+        if let wasmparser::Payload::CustomSection(s) = part? {
+            if s.name() == "zed:api-version" {
+                let version = parse_wasm_extension_version_custom_section(s.data());
+                if let Some(version) = version {
+                    return Ok(version);
+                } else {
+                    bail!(
+                        "extension {} has invalid zed:api-version section: {:?}",
+                        extension_id,
+                        s.data()
+                    );
+                }
+            }
+        }
+    }
+    bail!("extension {} has no zed:api-version section", extension_id)
+}
+
+fn parse_wasm_extension_version_custom_section(data: &[u8]) -> Option<SemanticVersion> {
     if data.len() == 6 {
         Some(SemanticVersion {
             major: u16::from_be_bytes([data[0], data[1]]) as _,