Add a command for building and installing a locally-developed Zed extension (#8781)

This PR adds an `zed: Install Local Extension` action, which lets you
select a path to a folder containing a Zed extension, and install that .
When you select a directory, the extension will be compiled (both the
Tree-sitter grammars and the Rust code for the extension itself) and
installed as a Zed extension, using a symlink.

### Details

A few dependencies are needed to build an extension:
* The Rust `wasm32-wasi` target. This is automatically installed if
needed via `rustup`.
* A wasi-preview1 adapter WASM module, for building WASM components with
Rust. This is automatically downloaded if needed from a `wasmtime`
GitHub release
* For building Tree-sitter parsers, a distribution of `wasi-sdk`. This
is automatically downloaded if needed from a `wasi-sdk` GitHub release.

The downloaded artifacts are cached in a support directory called
`Zed/extensions/build`.

### Tasks

UX

* [x] Show local extensions in the Extensions view
* [x] Provide a button for recompiling a linked extension
* [x] Make this action discoverable by adding a button for it on the
Extensions view
* [ ] Surface errors (don't just write them to the Zed log)

Packaging

* [ ] Create a separate executable that performs the extension
compilation. We'll switch the packaging system in our
[extensions](https://github.com/zed-industries/extensions) repo to use
this binary, so that there is one canonical definition of how to
build/package an extensions.

### Release Notes:

- N/A

---------

Co-authored-by: Marshall <marshall@zed.dev>
Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>
This commit is contained in:
Max Brunsfeld 2024-03-06 15:35:22 -08:00 committed by GitHub
parent e273198ada
commit 675ae24964
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 1662 additions and 763 deletions

View file

@ -39,6 +39,7 @@ util.workspace = true
wasmtime = { workspace = true, features = ["async"] }
wasmtime-wasi.workspace = true
wasmparser.workspace = true
wit-component.workspace = true
[dev-dependencies]
fs = { workspace = true, features = ["test-support"] }

View file

@ -0,0 +1,375 @@
use crate::ExtensionManifest;
use crate::{extension_manifest::ExtensionLibraryKind, GrammarManifestEntry};
use anyhow::{anyhow, bail, Context as _, Result};
use async_compression::futures::bufread::GzipDecoder;
use async_tar::Archive;
use futures::io::BufReader;
use futures::AsyncReadExt;
use serde::Deserialize;
use std::{
env, fs,
path::{Path, PathBuf},
process::{Command, Stdio},
sync::Arc,
};
use util::http::{AsyncBody, HttpClient};
use wit_component::ComponentEncoder;
/// Currently, we compile with Rust's `wasm32-wasi` target, which works with WASI `preview1`.
/// But the WASM component model is based on WASI `preview2`. So we need an 'adapter' WASM
/// module, which implements the `preview1` interface in terms of `preview2`.
///
/// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will
/// not need the adapter anymore.
const RUST_TARGET: &str = "wasm32-wasi";
const WASI_ADAPTER_URL: &str =
"https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm";
/// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc
/// and clang's runtime library. The `wasi-sdk` provides these binaries.
///
/// Once Clang 17 and its wasm target are available via system package managers, we won't need
/// to download this.
const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/";
const WASI_SDK_ASSET_NAME: Option<&str> = if cfg!(target_os = "macos") {
Some("wasi-sdk-21.0-macos.tar.gz")
} else if cfg!(target_os = "linux") {
Some("wasi-sdk-21.0-linux.tar.gz")
} else {
None
};
pub struct ExtensionBuilder {
cache_dir: PathBuf,
pub http: Arc<dyn HttpClient>,
}
pub struct CompileExtensionOptions {
pub release: bool,
}
#[derive(Deserialize)]
struct CargoToml {
package: CargoTomlPackage,
}
#[derive(Deserialize)]
struct CargoTomlPackage {
name: String,
}
impl ExtensionBuilder {
pub fn new(cache_dir: PathBuf, http: Arc<dyn HttpClient>) -> Self {
Self { cache_dir, http }
}
pub async fn compile_extension(
&self,
extension_dir: &Path,
options: CompileExtensionOptions,
) -> Result<()> {
fs::create_dir_all(&self.cache_dir)?;
let extension_toml_path = extension_dir.join("extension.toml");
let extension_toml_content = fs::read_to_string(&extension_toml_path)?;
let extension_toml: ExtensionManifest = toml::from_str(&extension_toml_content)?;
let cargo_toml_path = extension_dir.join("Cargo.toml");
if extension_toml.lib.kind == Some(ExtensionLibraryKind::Rust)
|| fs::metadata(&cargo_toml_path)?.is_file()
{
self.compile_rust_extension(extension_dir, options).await?;
}
for (grammar_name, grammar_metadata) in extension_toml.grammars {
self.compile_grammar(extension_dir, grammar_name, grammar_metadata)
.await?;
}
log::info!("finished compiling extension {}", extension_dir.display());
Ok(())
}
async fn compile_rust_extension(
&self,
extension_dir: &Path,
options: CompileExtensionOptions,
) -> Result<(), anyhow::Error> {
self.install_rust_wasm_target_if_needed()?;
let adapter_bytes = self.install_wasi_preview1_adapter_if_needed().await?;
let cargo_toml_content = fs::read_to_string(&extension_dir.join("Cargo.toml"))?;
let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?;
log::info!("compiling rust extension {}", extension_dir.display());
let output = Command::new("cargo")
.args(["build", "--target", RUST_TARGET])
.args(options.release.then_some("--release"))
.arg("--target-dir")
.arg(extension_dir.join("target"))
.current_dir(&extension_dir)
.output()
.context("failed to run `cargo`")?;
if !output.status.success() {
bail!(
"failed to build extension {}",
String::from_utf8_lossy(&output.stderr)
);
}
let mut wasm_path = PathBuf::from(extension_dir);
wasm_path.extend([
"target",
RUST_TARGET,
if options.release { "release" } else { "debug" },
cargo_toml.package.name.as_str(),
]);
wasm_path.set_extension("wasm");
let wasm_bytes = fs::read(&wasm_path)
.with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?;
let encoder = ComponentEncoder::default()
.module(&wasm_bytes)?
.adapter("wasi_snapshot_preview1", &adapter_bytes)
.context("failed to load adapter module")?
.validate(true);
let component_bytes = encoder
.encode()
.context("failed to encode wasm component")?;
fs::write(extension_dir.join("extension.wasm"), &component_bytes)
.context("failed to write extension.wasm")?;
Ok(())
}
async fn compile_grammar(
&self,
extension_dir: &Path,
grammar_name: Arc<str>,
grammar_metadata: GrammarManifestEntry,
) -> Result<()> {
let clang_path = self.install_wasi_sdk_if_needed().await?;
let mut grammar_repo_dir = extension_dir.to_path_buf();
grammar_repo_dir.extend(["grammars", grammar_name.as_ref()]);
let mut grammar_wasm_path = grammar_repo_dir.clone();
grammar_wasm_path.set_extension("wasm");
log::info!("checking out {grammar_name} parser");
self.checkout_repo(
&grammar_repo_dir,
&grammar_metadata.repository,
&grammar_metadata.rev,
)?;
let src_path = grammar_repo_dir.join("src");
let parser_path = src_path.join("parser.c");
let scanner_path = src_path.join("scanner.c");
log::info!("compiling {grammar_name} parser");
let clang_output = Command::new(&clang_path)
.args(["-fPIC", "-shared", "-Os"])
.arg(format!("-Wl,--export=tree_sitter_{grammar_name}"))
.arg("-o")
.arg(&grammar_wasm_path)
.arg("-I")
.arg(&src_path)
.arg(&parser_path)
.args(scanner_path.exists().then_some(scanner_path))
.output()
.context("failed to run clang")?;
if !clang_output.status.success() {
bail!(
"failed to compile {} parser with clang: {}",
grammar_name,
String::from_utf8_lossy(&clang_output.stderr),
);
}
Ok(())
}
fn checkout_repo(&self, directory: &Path, url: &str, rev: &str) -> Result<()> {
let git_dir = directory.join(".git");
if directory.exists() {
let remotes_output = Command::new("git")
.arg("--git-dir")
.arg(&git_dir)
.args(["remote", "-v"])
.output()?;
let has_remote = remotes_output.status.success()
&& String::from_utf8_lossy(&remotes_output.stdout)
.lines()
.any(|line| {
let mut parts = line.split(|c: char| c.is_whitespace());
parts.next() == Some("origin") && parts.any(|part| part == url)
});
if !has_remote {
bail!(
"grammar directory '{}' already exists, but is not a git clone of '{}'",
directory.display(),
url
);
}
} else {
fs::create_dir_all(&directory).with_context(|| {
format!("failed to create grammar directory {}", directory.display(),)
})?;
let init_output = Command::new("git")
.arg("init")
.current_dir(&directory)
.output()?;
if !init_output.status.success() {
bail!(
"failed to run `git init` in directory '{}'",
directory.display()
);
}
let remote_add_output = Command::new("git")
.arg("--git-dir")
.arg(&git_dir)
.args(["remote", "add", "origin", url])
.output()
.context("failed to execute `git remote add`")?;
if !remote_add_output.status.success() {
bail!(
"failed to add remote {url} for git repository {}",
git_dir.display()
);
}
}
let fetch_output = Command::new("git")
.arg("--git-dir")
.arg(&git_dir)
.args(["fetch", "--depth", "1", "origin", &rev])
.output()
.context("failed to execute `git fetch`")?;
if !fetch_output.status.success() {
bail!(
"failed to fetch revision {} in directory '{}'",
rev,
directory.display()
);
}
let checkout_output = Command::new("git")
.arg("--git-dir")
.arg(&git_dir)
.args(["checkout", &rev])
.current_dir(&directory)
.output()
.context("failed to execute `git checkout`")?;
if !checkout_output.status.success() {
bail!(
"failed to checkout revision {} in directory '{}'",
rev,
directory.display()
);
}
Ok(())
}
fn install_rust_wasm_target_if_needed(&self) -> Result<()> {
let rustc_output = Command::new("rustc")
.arg("--print")
.arg("sysroot")
.output()
.context("failed to run rustc")?;
if !rustc_output.status.success() {
bail!(
"failed to retrieve rust sysroot: {}",
String::from_utf8_lossy(&rustc_output.stderr)
);
}
let sysroot = PathBuf::from(String::from_utf8(rustc_output.stdout)?.trim());
if sysroot.join("lib/rustlib").join(RUST_TARGET).exists() {
return Ok(());
}
let output = Command::new("rustup")
.args(["target", "add", RUST_TARGET])
.stderr(Stdio::inherit())
.stdout(Stdio::inherit())
.output()
.context("failed to run `rustup target add`")?;
if !output.status.success() {
bail!("failed to install the `{RUST_TARGET}` target");
}
Ok(())
}
async fn install_wasi_preview1_adapter_if_needed(&self) -> Result<Vec<u8>> {
let cache_path = self.cache_dir.join("wasi_snapshot_preview1.reactor.wasm");
if let Ok(content) = fs::read(&cache_path) {
if wasmparser::Parser::is_core_wasm(&content) {
return Ok(content);
}
}
fs::remove_file(&cache_path).ok();
log::info!("downloading wasi adapter module");
let mut response = self
.http
.get(WASI_ADAPTER_URL, AsyncBody::default(), true)
.await?;
let mut content = Vec::new();
let mut body = BufReader::new(response.body_mut());
body.read_to_end(&mut content).await?;
fs::write(&cache_path, &content)
.with_context(|| format!("failed to save file {}", cache_path.display()))?;
if !wasmparser::Parser::is_core_wasm(&content) {
bail!("downloaded wasi adapter is invalid");
}
Ok(content)
}
async fn install_wasi_sdk_if_needed(&self) -> Result<PathBuf> {
let url = if let Some(asset_name) = WASI_SDK_ASSET_NAME {
format!("{WASI_SDK_URL}/{asset_name}")
} else {
bail!("wasi-sdk is not available for platform {}", env::consts::OS);
};
let wasi_sdk_dir = self.cache_dir.join("wasi-sdk");
let mut clang_path = wasi_sdk_dir.clone();
clang_path.extend(["bin", "clang-17"]);
if fs::metadata(&clang_path).map_or(false, |metadata| metadata.is_file()) {
return Ok(clang_path);
}
fs::remove_dir_all(&wasi_sdk_dir).ok();
let mut response = self.http.get(&url, AsyncBody::default(), true).await?;
let mut tar_out_dir = wasi_sdk_dir.clone();
tar_out_dir.set_extension(".output");
let body = BufReader::new(response.body_mut());
let body = GzipDecoder::new(body);
let tar = Archive::new(body);
tar.unpack(&tar_out_dir).await?;
let inner_dir = fs::read_dir(&tar_out_dir)?
.next()
.ok_or_else(|| anyhow!("no content"))?
.context("failed to read contents of extracted wasi archive directory")?
.path();
fs::rename(&inner_dir, &wasi_sdk_dir).context("failed to move extracted wasi dir")?;
fs::remove_dir_all(&tar_out_dir).ok();
Ok(clang_path)
}
}

View file

@ -0,0 +1,72 @@
use collections::BTreeMap;
use language::LanguageServerName;
use serde::{Deserialize, Serialize};
use std::{path::PathBuf, sync::Arc};
/// This is the old version of the extension manifest, from when it was `extension.json`.
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct OldExtensionManifest {
pub name: String,
pub version: Arc<str>,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub repository: Option<String>,
#[serde(default)]
pub authors: Vec<String>,
#[serde(default)]
pub themes: BTreeMap<Arc<str>, PathBuf>,
#[serde(default)]
pub languages: BTreeMap<Arc<str>, PathBuf>,
#[serde(default)]
pub grammars: BTreeMap<Arc<str>, PathBuf>,
}
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct ExtensionManifest {
pub id: Arc<str>,
pub name: String,
pub version: Arc<str>,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub repository: Option<String>,
#[serde(default)]
pub authors: Vec<String>,
#[serde(default)]
pub lib: LibManifestEntry,
#[serde(default)]
pub themes: Vec<PathBuf>,
#[serde(default)]
pub languages: Vec<PathBuf>,
#[serde(default)]
pub grammars: BTreeMap<Arc<str>, GrammarManifestEntry>,
#[serde(default)]
pub language_servers: BTreeMap<LanguageServerName, LanguageServerManifestEntry>,
}
#[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct LibManifestEntry {
pub kind: Option<ExtensionLibraryKind>,
}
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub enum ExtensionLibraryKind {
Rust,
}
#[derive(Clone, Default, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct GrammarManifestEntry {
pub repository: String,
#[serde(alias = "commit")]
pub rev: String,
}
#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)]
pub struct LanguageServerManifestEntry {
pub language: Arc<str>,
}

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,7 @@
use crate::{
ExtensionIndex, ExtensionIndexEntry, ExtensionIndexLanguageEntry, ExtensionManifest,
ExtensionStore, GrammarManifestEntry,
build_extension::{CompileExtensionOptions, ExtensionBuilder},
ExtensionIndex, ExtensionIndexEntry, ExtensionIndexLanguageEntry, ExtensionIndexThemeEntry,
ExtensionManifest, ExtensionStore, GrammarManifestEntry, RELOAD_DEBOUNCE_DURATION,
};
use async_compression::futures::bufread::GzipEncoder;
use collections::BTreeMap;
@ -21,7 +22,7 @@ use std::{
sync::Arc,
};
use theme::ThemeRegistry;
use util::http::{FakeHttpClient, Response};
use util::http::{self, FakeHttpClient, Response};
#[gpui::test]
async fn test_extension_store(cx: &mut TestAppContext) {
@ -131,45 +132,49 @@ async fn test_extension_store(cx: &mut TestAppContext) {
extensions: [
(
"zed-ruby".into(),
ExtensionManifest {
id: "zed-ruby".into(),
name: "Zed Ruby".into(),
version: "1.0.0".into(),
description: None,
authors: Vec::new(),
repository: None,
themes: Default::default(),
lib: Default::default(),
languages: vec!["languages/erb".into(), "languages/ruby".into()],
grammars: [
("embedded_template".into(), GrammarManifestEntry::default()),
("ruby".into(), GrammarManifestEntry::default()),
]
.into_iter()
.collect(),
language_servers: BTreeMap::default(),
}
.into(),
ExtensionIndexEntry {
manifest: Arc::new(ExtensionManifest {
id: "zed-ruby".into(),
name: "Zed Ruby".into(),
version: "1.0.0".into(),
description: None,
authors: Vec::new(),
repository: None,
themes: Default::default(),
lib: Default::default(),
languages: vec!["languages/erb".into(), "languages/ruby".into()],
grammars: [
("embedded_template".into(), GrammarManifestEntry::default()),
("ruby".into(), GrammarManifestEntry::default()),
]
.into_iter()
.collect(),
language_servers: BTreeMap::default(),
}),
dev: false,
},
),
(
"zed-monokai".into(),
ExtensionManifest {
id: "zed-monokai".into(),
name: "Zed Monokai".into(),
version: "2.0.0".into(),
description: None,
authors: vec![],
repository: None,
themes: vec![
"themes/monokai-pro.json".into(),
"themes/monokai.json".into(),
],
lib: Default::default(),
languages: Default::default(),
grammars: BTreeMap::default(),
language_servers: BTreeMap::default(),
}
.into(),
ExtensionIndexEntry {
manifest: Arc::new(ExtensionManifest {
id: "zed-monokai".into(),
name: "Zed Monokai".into(),
version: "2.0.0".into(),
description: None,
authors: vec![],
repository: None,
themes: vec![
"themes/monokai-pro.json".into(),
"themes/monokai.json".into(),
],
lib: Default::default(),
languages: Default::default(),
grammars: BTreeMap::default(),
language_servers: BTreeMap::default(),
}),
dev: false,
},
),
]
.into_iter()
@ -205,28 +210,28 @@ async fn test_extension_store(cx: &mut TestAppContext) {
themes: [
(
"Monokai Dark".into(),
ExtensionIndexEntry {
ExtensionIndexThemeEntry {
extension: "zed-monokai".into(),
path: "themes/monokai.json".into(),
},
),
(
"Monokai Light".into(),
ExtensionIndexEntry {
ExtensionIndexThemeEntry {
extension: "zed-monokai".into(),
path: "themes/monokai.json".into(),
},
),
(
"Monokai Pro Dark".into(),
ExtensionIndexEntry {
ExtensionIndexThemeEntry {
extension: "zed-monokai".into(),
path: "themes/monokai-pro.json".into(),
},
),
(
"Monokai Pro Light".into(),
ExtensionIndexEntry {
ExtensionIndexThemeEntry {
extension: "zed-monokai".into(),
path: "themes/monokai-pro.json".into(),
},
@ -252,7 +257,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
)
});
cx.executor().run_until_parked();
cx.executor().advance_clock(super::RELOAD_DEBOUNCE_DURATION);
store.read_with(cx, |store, _| {
let index = &store.extension_index;
assert_eq!(index.extensions, expected_index.extensions);
@ -305,32 +310,34 @@ async fn test_extension_store(cx: &mut TestAppContext) {
expected_index.extensions.insert(
"zed-gruvbox".into(),
ExtensionManifest {
id: "zed-gruvbox".into(),
name: "Zed Gruvbox".into(),
version: "1.0.0".into(),
description: None,
authors: vec![],
repository: None,
themes: vec!["themes/gruvbox.json".into()],
lib: Default::default(),
languages: Default::default(),
grammars: BTreeMap::default(),
language_servers: BTreeMap::default(),
}
.into(),
ExtensionIndexEntry {
manifest: Arc::new(ExtensionManifest {
id: "zed-gruvbox".into(),
name: "Zed Gruvbox".into(),
version: "1.0.0".into(),
description: None,
authors: vec![],
repository: None,
themes: vec!["themes/gruvbox.json".into()],
lib: Default::default(),
languages: Default::default(),
grammars: BTreeMap::default(),
language_servers: BTreeMap::default(),
}),
dev: false,
},
);
expected_index.themes.insert(
"Gruvbox".into(),
ExtensionIndexEntry {
ExtensionIndexThemeEntry {
extension: "zed-gruvbox".into(),
path: "themes/gruvbox.json".into(),
},
);
store.update(cx, |store, cx| store.reload(cx));
let _ = store.update(cx, |store, _| store.reload(None));
cx.executor().run_until_parked();
cx.executor().advance_clock(RELOAD_DEBOUNCE_DURATION);
store.read_with(cx, |store, _| {
let index = &store.extension_index;
assert_eq!(index.extensions, expected_index.extensions);
@ -400,7 +407,7 @@ async fn test_extension_store(cx: &mut TestAppContext) {
store.uninstall_extension("zed-ruby".into(), cx)
});
cx.executor().run_until_parked();
cx.executor().advance_clock(RELOAD_DEBOUNCE_DURATION);
expected_index.extensions.remove("zed-ruby");
expected_index.languages.remove("Ruby");
expected_index.languages.remove("ERB");
@ -416,17 +423,23 @@ async fn test_extension_store(cx: &mut TestAppContext) {
async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
init_test(cx);
let gleam_extension_dir = PathBuf::from_iter([
env!("CARGO_MANIFEST_DIR"),
"..",
"..",
"extensions",
"gleam",
])
.canonicalize()
.unwrap();
let root_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
.parent()
.unwrap()
.parent()
.unwrap();
let cache_dir = root_dir.join("target");
let gleam_extension_dir = root_dir.join("extensions").join("gleam");
compile_extension("zed_gleam", &gleam_extension_dir);
cx.executor().allow_parking();
ExtensionBuilder::new(cache_dir, http::client())
.compile_extension(
&gleam_extension_dir,
CompileExtensionOptions { release: false },
)
.await
.unwrap();
cx.executor().forbid_parking();
let fs = FakeFs::new(cx.executor());
fs.insert_tree("/the-extension-dir", json!({ "installed": {} }))
@ -509,7 +522,7 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
)
});
cx.executor().run_until_parked();
cx.executor().advance_clock(RELOAD_DEBOUNCE_DURATION);
let mut fake_servers = language_registry.fake_language_servers("Gleam");
@ -572,27 +585,6 @@ async fn test_extension_store_with_gleam_extension(cx: &mut TestAppContext) {
);
}
fn compile_extension(name: &str, extension_dir_path: &Path) {
let output = std::process::Command::new("cargo")
.args(["component", "build", "--target-dir"])
.arg(extension_dir_path.join("target"))
.current_dir(&extension_dir_path)
.output()
.unwrap();
assert!(
output.status.success(),
"failed to build component {}",
String::from_utf8_lossy(&output.stderr)
);
let mut wasm_path = PathBuf::from(extension_dir_path);
wasm_path.extend(["target", "wasm32-wasi", "debug", name]);
wasm_path.set_extension("wasm");
std::fs::rename(wasm_path, extension_dir_path.join("extension.wasm")).unwrap();
}
fn init_test(cx: &mut TestAppContext) {
cx.update(|cx| {
let store = SettingsStore::test(cx);