Introduce extension-cli binary, for packaging extensions in CI (#9523)

This will be used in the
[extensions](https://github.com/zed-industries/extensions) repository
for packaging the extensions that users submit.

Release Notes:

- N/A

---------

Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>

crates/extension/src/extension_builder.rs

@@ -6,9 +6,8 @@ use async_tar::Archive;
 use futures::io::BufReader;
 use futures::AsyncReadExt;
 use serde::Deserialize;
-use std::mem;
 use std::{
-    env, fs,
+    env, fs, mem,
     path::{Path, PathBuf},
     process::{Command, Stdio},
     sync::Arc,
@@ -72,22 +71,27 @@ impl ExtensionBuilder {
     pub async fn compile_extension(
         &self,
         extension_dir: &Path,
+        extension_manifest: &ExtensionManifest,
         options: CompileExtensionOptions,
     ) -> Result<()> {
+        if extension_dir.is_relative() {
+            bail!(
+                "extension dir {} is not an absolute path",
+                extension_dir.display()
+            );
+        }
+
         fs::create_dir_all(&self.cache_dir)?;
-        let extension_toml_path = extension_dir.join("extension.toml");
-        let extension_toml_content = fs::read_to_string(&extension_toml_path)?;
-        let extension_toml: ExtensionManifest = toml::from_str(&extension_toml_content)?;
 
         let cargo_toml_path = extension_dir.join("Cargo.toml");
-        if extension_toml.lib.kind == Some(ExtensionLibraryKind::Rust)
+        if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust)
             || fs::metadata(&cargo_toml_path)?.is_file()
         {
             self.compile_rust_extension(extension_dir, options).await?;
         }
 
-        for (grammar_name, grammar_metadata) in extension_toml.grammars {
-            self.compile_grammar(extension_dir, grammar_name, grammar_metadata)
+        for (grammar_name, grammar_metadata) in &extension_manifest.grammars {
+            self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata)
                 .await?;
         }
 
@@ -157,13 +161,13 @@ impl ExtensionBuilder {
     async fn compile_grammar(
         &self,
         extension_dir: &Path,
-        grammar_name: Arc<str>,
-        grammar_metadata: GrammarManifestEntry,
+        grammar_name: &str,
+        grammar_metadata: &GrammarManifestEntry,
     ) -> Result<()> {
         let clang_path = self.install_wasi_sdk_if_needed().await?;
 
         let mut grammar_repo_dir = extension_dir.to_path_buf();
-        grammar_repo_dir.extend(["grammars", grammar_name.as_ref()]);
+        grammar_repo_dir.extend(["grammars", grammar_name]);
 
         let mut grammar_wasm_path = grammar_repo_dir.clone();
         grammar_wasm_path.set_extension("wasm");
@@ -277,9 +281,10 @@ impl ExtensionBuilder {
                 );
             }
             bail!(
-                "failed to checkout revision {} in directory '{}'",
+                "failed to checkout revision {} in directory '{}': {}",
                 rev,
-                directory.display()
+                directory.display(),
+                String::from_utf8_lossy(&checkout_output.stderr)
             );
         }
 

crates/extension/src/extension_store.rs

@@ -1,4 +1,4 @@
-mod build_extension;
+pub mod extension_builder;
 mod extension_lsp_adapter;
 mod extension_manifest;
 mod wasm_host;
@@ -10,8 +10,8 @@ use crate::{extension_lsp_adapter::ExtensionLspAdapter, wasm_host::wit};
 use anyhow::{anyhow, bail, Context as _, Result};
 use async_compression::futures::bufread::GzipDecoder;
 use async_tar::Archive;
-use build_extension::{CompileExtensionOptions, ExtensionBuilder};
 use collections::{hash_map, BTreeMap, HashMap, HashSet};
+use extension_builder::{CompileExtensionOptions, ExtensionBuilder};
 use extension_manifest::ExtensionLibraryKind;
 use fs::{Fs, RemoveOptions};
 use futures::{
@@ -545,6 +545,7 @@ impl ExtensionStore {
                         builder
                             .compile_extension(
                                 &extension_source_path,
+                                &extension_manifest,
                                 CompileExtensionOptions { release: false },
                             )
                             .await
@@ -580,6 +581,7 @@ impl ExtensionStore {
     pub fn rebuild_dev_extension(&mut self, extension_id: Arc<str>, cx: &mut ModelContext<Self>) {
         let path = self.installed_dir.join(extension_id.as_ref());
         let builder = self.builder.clone();
+        let fs = self.fs.clone();
 
         match self.outstanding_operations.entry(extension_id.clone()) {
             hash_map::Entry::Occupied(_) => return,
@@ -588,8 +590,9 @@ impl ExtensionStore {
 
         cx.notify();
         let compile = cx.background_executor().spawn(async move {
+            let manifest = Self::load_extension_manifest(fs, &path).await?;
             builder
-                .compile_extension(&path, CompileExtensionOptions { release: true })
+                .compile_extension(&path, &manifest, CompileExtensionOptions { release: true })
                 .await
         });
 
@@ -1000,7 +1003,7 @@ impl ExtensionStore {
         Ok(())
     }
 
-    async fn load_extension_manifest(
+    pub async fn load_extension_manifest(
         fs: Arc<dyn Fs>,
         extension_dir: &Path,
     ) -> Result<ExtensionManifest> {