Perform extension packaging in extension-cli (#9549)

Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>

Cargo.lock

@@ -3501,6 +3501,9 @@ dependencies = [
  "fs",
  "language",
  "log",
+ "rpc",
+ "serde",
+ "serde_json",
  "theme",
  "tokio",
  "toml 0.8.10",

crates/extension/src/extension_store.rs

@@ -12,7 +12,6 @@ use async_compression::futures::bufread::GzipDecoder;
 use async_tar::Archive;
 use collections::{hash_map, BTreeMap, HashMap, HashSet};
 use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
-use extension_manifest::ExtensionLibraryKind;
 use fs::{Fs, RemoveOptions};
 use futures::{
     channel::{
@@ -43,7 +42,9 @@ use util::{
 };
 use wasm_host::{WasmExtension, WasmHost};
 
-pub use extension_manifest::{ExtensionManifest, GrammarManifestEntry, OldExtensionManifest};
+pub use extension_manifest::{
+    ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, OldExtensionManifest,
+};
 
 const RELOAD_DEBOUNCE_DURATION: Duration = Duration::from_millis(200);
 const FS_WATCH_LATENCY: Duration = Duration::from_millis(100);

crates/extension_cli/Cargo.toml

@@ -8,10 +8,6 @@ license = "GPL-3.0-or-later"
 [lints]
 workspace = true
 
-[[bin]]
-name = "zed-extension"
-path = "src/main.rs"
-
 [dependencies]
 anyhow.workspace = true
 clap = { workspace = true, features = ["derive"] }
@@ -20,6 +16,9 @@ fs.workspace = true
 extension.workspace = true
 language.workspace = true
 log.workspace = true
+rpc.workspace = true
+serde.workspace = true
+serde_json.workspace = true
 theme.workspace = true
 tokio.workspace = true
 toml.workspace = true

crates/extension_cli/src/main.rs

@@ -1,16 +1,17 @@
 use std::{
     collections::HashMap,
-    fs,
+    env, fs,
     path::{Path, PathBuf},
+    process::Command,
     sync::Arc,
 };
 
-use ::fs::{Fs, RealFs};
+use ::fs::{copy_recursive, CopyOptions, Fs, RealFs};
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use clap::Parser;
 use extension::{
     extension_builder::{CompileExtensionOptions, ExtensionBuilder},
-    ExtensionStore,
+    ExtensionLibraryKind, ExtensionManifest, ExtensionStore,
 };
 use language::LanguageConfig;
 use theme::ThemeRegistry;
@@ -20,10 +21,11 @@ use tree_sitter::{Language, Query, WasmStore};
 #[command(name = "zed-extension")]
 struct Args {
     /// The path to the extension directory
-    extension_path: PathBuf,
-    /// Whether to compile with optimizations
     #[arg(long)]
-    release: bool,
+    source_dir: PathBuf,
+    /// The output directory to place the packaged extension.
+    #[arg(long)]
+    output_dir: PathBuf,
     /// The path to a directory where build dependencies are downloaded
     #[arg(long)]
     scratch_dir: PathBuf,
@@ -39,69 +41,214 @@ async fn main() -> Result<()> {
     let mut wasm_store = WasmStore::new(engine)?;
 
     let extension_path = args
-        .extension_path
+        .source_dir
         .canonicalize()
-        .context("can't canonicalize extension_path")?;
+        .context("failed to canonicalize source_dir")?;
     let scratch_dir = args
         .scratch_dir
         .canonicalize()
-        .context("can't canonicalize scratch_dir")?;
+        .context("failed to canonicalize scratch_dir")?;
+    let output_dir = if args.output_dir.is_relative() {
+        env::current_dir()?.join(&args.output_dir)
+    } else {
+        args.output_dir
+    };
+
+    let mut manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
+    populate_default_paths(&mut manifest, &extension_path)?;
 
-    let manifest = ExtensionStore::load_extension_manifest(fs.clone(), &extension_path).await?;
     let builder = ExtensionBuilder::new(scratch_dir);
     builder
         .compile_extension(
             &extension_path,
             &manifest,
-            CompileExtensionOptions {
+            CompileExtensionOptions { release: true },
-                release: args.release,
-            },
         )
-        .await?;
+        .await
+        .context("failed to compile extension")?;
 
-    let grammars = test_grammars(&extension_path, &mut wasm_store)?;
+    let grammars = test_grammars(&manifest, &extension_path, &mut wasm_store)?;
-    test_languages(&extension_path, &grammars)?;
+    test_languages(&manifest, &extension_path, &grammars)?;
-    test_themes(&extension_path, fs.clone()).await?;
+    test_themes(&manifest, &extension_path, fs.clone()).await?;
+
+    let archive_dir = output_dir.join("archive");
+    fs::remove_dir_all(&archive_dir).ok();
+    copy_extension_resources(&manifest, &extension_path, &archive_dir, fs.clone())
+        .await
+        .context("failed to copy extension resources")?;
+
+    let tar_output = Command::new("tar")
+        .current_dir(&output_dir)
+        .args(&["-czvf", "archive.tar.gz", "-C", "archive", "."])
+        .output()
+        .context("failed to run tar")?;
+    if !tar_output.status.success() {
+        bail!(
+            "failed to create archive.tar.gz: {}",
+            String::from_utf8_lossy(&tar_output.stderr)
+        );
+    }
+
+    let manifest_json = serde_json::to_string(&rpc::ExtensionApiManifest {
+        name: manifest.name,
+        version: manifest.version.to_string(),
+        description: manifest.description,
+        authors: manifest.authors,
+        repository: manifest
+            .repository
+            .ok_or_else(|| anyhow!("missing repository in extension manifest"))?,
+    })?;
+    fs::remove_dir_all(&archive_dir)?;
+    fs::write(output_dir.join("manifest.json"), manifest_json.as_bytes())?;
+
+    Ok(())
+}
+
+fn populate_default_paths(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> {
+    let cargo_toml_path = extension_path.join("Cargo.toml");
+    if cargo_toml_path.exists() {
+        manifest.lib.kind = Some(ExtensionLibraryKind::Rust);
+    }
+
+    let languages_dir = extension_path.join("languages");
+    if languages_dir.exists() {
+        for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? {
+            let entry = entry?;
+            let language_dir = entry.path();
+            let config_path = language_dir.join("config.toml");
+            if config_path.exists() {
+                let relative_language_dir =
+                    language_dir.strip_prefix(extension_path)?.to_path_buf();
+                if !manifest.languages.contains(&relative_language_dir) {
+                    manifest.languages.push(relative_language_dir);
+                }
+            }
+        }
+    }
+
+    let themes_dir = extension_path.join("themes");
+    if themes_dir.exists() {
+        for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? {
+            let entry = entry?;
+            let theme_path = entry.path();
+            if theme_path.extension() == Some("json".as_ref()) {
+                let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf();
+                if !manifest.themes.contains(&relative_theme_path) {
+                    manifest.themes.push(relative_theme_path);
+                }
+            }
+        }
+    }
+
+    Ok(())
+}
+
+async fn copy_extension_resources(
+    manifest: &ExtensionManifest,
+    extension_path: &Path,
+    output_dir: &Path,
+    fs: Arc<dyn Fs>,
+) -> Result<()> {
+    fs::create_dir_all(&output_dir).context("failed to create output dir")?;
+
+    let manifest_toml = toml::to_string(&manifest).context("failed to serialize manifest")?;
+    fs::write(output_dir.join("extension.toml"), &manifest_toml)
+        .context("failed to write extension.toml")?;
+
+    if manifest.lib.kind.is_some() {
+        fs::copy(
+            extension_path.join("extension.wasm"),
+            output_dir.join("extension.wasm"),
+        )
+        .context("failed to copy extension.wasm")?;
+    }
+
+    if !manifest.grammars.is_empty() {
+        let source_grammars_dir = extension_path.join("grammars");
+        let output_grammars_dir = output_dir.join("grammars");
+        fs::create_dir_all(&output_grammars_dir)?;
+        for grammar_name in manifest.grammars.keys() {
+            let mut grammar_filename = PathBuf::from(grammar_name.as_ref());
+            grammar_filename.set_extension("wasm");
+            fs::copy(
+                &source_grammars_dir.join(&grammar_filename),
+                &output_grammars_dir.join(&grammar_filename),
+            )
+            .with_context(|| format!("failed to copy grammar '{}'", grammar_filename.display()))?;
+        }
+    }
+
+    if !manifest.themes.is_empty() {
+        let output_themes_dir = output_dir.join("themes");
+        fs::create_dir_all(&output_themes_dir)?;
+        for theme_path in &manifest.themes {
+            fs::copy(
+                extension_path.join(theme_path),
+                output_themes_dir.join(
+                    theme_path
+                        .file_name()
+                        .ok_or_else(|| anyhow!("invalid theme path"))?,
+                ),
+            )
+            .with_context(|| format!("failed to copy theme '{}'", theme_path.display()))?;
+        }
+    }
+
+    if !manifest.languages.is_empty() {
+        let output_languages_dir = output_dir.join("languages");
+        fs::create_dir_all(&output_languages_dir)?;
+        for language_path in &manifest.languages {
+            copy_recursive(
+                fs.as_ref(),
+                &extension_path.join(language_path),
+                &output_languages_dir.join(
+                    language_path
+                        .file_name()
+                        .ok_or_else(|| anyhow!("invalid language path"))?,
+                ),
+                CopyOptions {
+                    overwrite: true,
+                    ignore_if_exists: false,
+                },
+            )
+            .await
+            .with_context(|| {
+                format!("failed to copy language dir '{}'", language_path.display())
+            })?;
+        }
+    }
 
     Ok(())
 }
 
 fn test_grammars(
+    manifest: &ExtensionManifest,
     extension_path: &Path,
     wasm_store: &mut WasmStore,
 ) -> Result<HashMap<String, Language>> {
     let mut grammars = HashMap::default();
     let grammars_dir = extension_path.join("grammars");
-    if !grammars_dir.exists() {
-        return Ok(grammars);
-    }
 
-    let entries = fs::read_dir(&grammars_dir)?;
+    for grammar_name in manifest.grammars.keys() {
-    for entry in entries {
+        let mut grammar_path = grammars_dir.join(grammar_name.as_ref());
-        let entry = entry?;
+        grammar_path.set_extension("wasm");
-        let grammar_path = entry.path();
+
-        let grammar_name = grammar_path.file_stem().unwrap().to_str().unwrap();
+        let wasm = fs::read(&grammar_path)?;
-        if grammar_path.extension() == Some("wasm".as_ref()) {
+        let language = wasm_store.load_language(grammar_name, &wasm)?;
-            let wasm = fs::read(&grammar_path)?;
+        log::info!("loaded grammar {grammar_name}");
-            let language = wasm_store.load_language(grammar_name, &wasm)?;
+        grammars.insert(grammar_name.to_string(), language);
-            log::info!("loaded grammar {grammar_name}");
-            grammars.insert(grammar_name.into(), language);
-        }
     }
 
     Ok(grammars)
 }
 
-fn test_languages(extension_path: &Path, grammars: &HashMap<String, Language>) -> Result<()> {
+fn test_languages(
-    let languages_dir = extension_path.join("languages");
+    manifest: &ExtensionManifest,
-    if !languages_dir.exists() {
+    extension_path: &Path,
-        return Ok(());
+    grammars: &HashMap<String, Language>,
-    }
+) -> Result<()> {
-
+    for relative_language_dir in &manifest.languages {
-    let entries = fs::read_dir(&languages_dir)?;
+        let language_dir = extension_path.join(relative_language_dir);
-    for entry in entries {
-        let entry = entry?;
-        let language_dir = entry.path();
         let config_path = language_dir.join("config.toml");
         let config_content = fs::read_to_string(&config_path)?;
         let config: LanguageConfig = toml::from_str(&config_content)?;
@@ -139,20 +286,15 @@ fn test_languages(extension_path: &Path, grammars: &HashMap<String, Language>) -
     Ok(())
 }
 
-async fn test_themes(extension_path: &Path, fs: Arc<dyn Fs>) -> Result<()> {
+async fn test_themes(
-    let themes_dir = extension_path.join("themes");
+    manifest: &ExtensionManifest,
-    if !themes_dir.exists() {
+    extension_path: &Path,
-        return Ok(());
+    fs: Arc<dyn Fs>,
-    }
+) -> Result<()> {
-
+    for relative_theme_path in &manifest.themes {
-    let entries = fs::read_dir(&themes_dir)?;
+        let theme_path = extension_path.join(relative_theme_path);
-    for entry in entries {
+        let theme_family = ThemeRegistry::read_user_theme(&theme_path, fs.clone()).await?;
-        let entry = entry?;
+        log::info!("loaded theme family {}", theme_family.name);
-        let theme_path = entry.path();
-        if theme_path.extension() == Some("json".as_ref()) {
-            let theme_family = ThemeRegistry::read_user_theme(&entry.path(), fs.clone()).await?;
-            log::info!("loaded theme family {}", theme_family.name);
-        }
     }
 
     Ok(())

crates/rpc/src/extension.rs

@@ -0,0 +1,10 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+pub struct ExtensionApiManifest {
+    pub name: String,
+    pub version: String,
+    pub description: Option<String>,
+    pub authors: Vec<String>,
+    pub repository: String,
+}

crates/rpc/src/rpc.rs

@@ -1,12 +1,14 @@
 pub mod auth;
 mod conn;
 mod error;
+mod extension;
 mod notification;
 mod peer;
 pub mod proto;
 
 pub use conn::Connection;
 pub use error::*;
+pub use extension::*;
 pub use notification::*;
 pub use peer::*;
 mod macros;

extensions/gleam/extension.toml

@@ -3,6 +3,7 @@ name = "Gleam"
 description = "Gleam support for Zed"
 version = "0.0.1"
 authors = ["Marshall Bowers <elliott.codes@gmail.com>"]
+repository = "https://github.com/zed-industries/zed"
 
 [language_servers.gleam]
 name = "Gleam LSP"

extensions/uiua/extension.toml

@@ -3,6 +3,7 @@ name = "Uiua"
 description = "Uiua support for Zed"
 version = "0.0.1"
 authors = ["Max Brunsfeld <max@zed.dev>"]
+repository = "https://github.com/zed-industries/zed"
 
 [language_servers.uiua]
 name = "Uiua LSP"