use crate::{ parse_wasm_extension_version, ExtensionLibraryKind, ExtensionManifest, GrammarManifestEntry, }; use anyhow::{anyhow, bail, Context as _, Result}; use async_compression::futures::bufread::GzipDecoder; use async_tar::Archive; use futures::io::BufReader; use futures::AsyncReadExt; use http_client::{self, AsyncBody, HttpClient}; use serde::Deserialize; use std::{ env, fs, mem, path::{Path, PathBuf}, process::{Command, Stdio}, sync::Arc, }; use wasm_encoder::{ComponentSectionId, Encode as _, RawSection, Section as _}; use wasmparser::Parser; use wit_component::ComponentEncoder; /// Currently, we compile with Rust's `wasm32-wasip1` target, which works with WASI `preview1`. /// But the WASM component model is based on WASI `preview2`. So we need an 'adapter' WASM /// module, which implements the `preview1` interface in terms of `preview2`. /// /// Once Rust 1.78 is released, there will be a `wasm32-wasip2` target available, so we will /// not need the adapter anymore. const RUST_TARGET: &str = "wasm32-wasip1"; const WASI_ADAPTER_URL: &str = "https://github.com/bytecodealliance/wasmtime/releases/download/v18.0.2/wasi_snapshot_preview1.reactor.wasm"; /// Compiling Tree-sitter parsers from C to WASM requires Clang 17, and a WASM build of libc /// and clang's runtime library. The `wasi-sdk` provides these binaries. /// /// Once Clang 17 and its wasm target are available via system package managers, we won't need /// to download this. const WASI_SDK_URL: &str = "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-21/"; const WASI_SDK_ASSET_NAME: Option<&str> = if cfg!(target_os = "macos") { Some("wasi-sdk-21.0-macos.tar.gz") } else if cfg!(target_os = "linux") { Some("wasi-sdk-21.0-linux.tar.gz") } else if cfg!(target_os = "windows") { Some("wasi-sdk-21.0.m-mingw.tar.gz") } else { None }; pub struct ExtensionBuilder { cache_dir: PathBuf, pub http: Arc, } pub struct CompileExtensionOptions { pub release: bool, } #[derive(Deserialize)] struct CargoToml { package: CargoTomlPackage, } #[derive(Deserialize)] struct CargoTomlPackage { name: String, } impl ExtensionBuilder { pub fn new(http_client: Arc, cache_dir: PathBuf) -> Self { Self { cache_dir, http: http_client, } } pub async fn compile_extension( &self, extension_dir: &Path, extension_manifest: &mut ExtensionManifest, options: CompileExtensionOptions, ) -> Result<()> { populate_defaults(extension_manifest, extension_dir)?; if extension_dir.is_relative() { bail!( "extension dir {} is not an absolute path", extension_dir.display() ); } fs::create_dir_all(&self.cache_dir).context("failed to create cache dir")?; if extension_manifest.lib.kind == Some(ExtensionLibraryKind::Rust) { log::info!("compiling Rust extension {}", extension_dir.display()); self.compile_rust_extension(extension_dir, extension_manifest, options) .await .context("failed to compile Rust extension")?; log::info!("compiled Rust extension {}", extension_dir.display()); } for (grammar_name, grammar_metadata) in &extension_manifest.grammars { log::info!( "compiling grammar {grammar_name} for extension {}", extension_dir.display() ); self.compile_grammar(extension_dir, grammar_name.as_ref(), grammar_metadata) .await .with_context(|| format!("failed to compile grammar '{grammar_name}'"))?; log::info!( "compiled grammar {grammar_name} for extension {}", extension_dir.display() ); } log::info!("finished compiling extension {}", extension_dir.display()); Ok(()) } async fn compile_rust_extension( &self, extension_dir: &Path, manifest: &mut ExtensionManifest, options: CompileExtensionOptions, ) -> Result<(), anyhow::Error> { self.install_rust_wasm_target_if_needed()?; let adapter_bytes = self.install_wasi_preview1_adapter_if_needed().await?; let cargo_toml_content = fs::read_to_string(extension_dir.join("Cargo.toml"))?; let cargo_toml: CargoToml = toml::from_str(&cargo_toml_content)?; log::info!( "compiling Rust crate for extension {}", extension_dir.display() ); let output = Command::new("cargo") .args(["build", "--target", RUST_TARGET]) .args(options.release.then_some("--release")) .arg("--target-dir") .arg(extension_dir.join("target")) .current_dir(extension_dir) .output() .context("failed to run `cargo`")?; if !output.status.success() { bail!( "failed to build extension {}", String::from_utf8_lossy(&output.stderr) ); } log::info!( "compiled Rust crate for extension {}", extension_dir.display() ); let mut wasm_path = PathBuf::from(extension_dir); wasm_path.extend([ "target", RUST_TARGET, if options.release { "release" } else { "debug" }, &cargo_toml .package .name // The wasm32-wasip1 target normalizes `-` in package names to `_` in the resulting `.wasm` file. .replace('-', "_"), ]); wasm_path.set_extension("wasm"); let wasm_bytes = fs::read(&wasm_path) .with_context(|| format!("failed to read output module `{}`", wasm_path.display()))?; let encoder = ComponentEncoder::default() .module(&wasm_bytes)? .adapter("wasi_snapshot_preview1", &adapter_bytes) .context("failed to load adapter module")? .validate(true); log::info!( "encoding wasm component for extension {}", extension_dir.display() ); let component_bytes = encoder .encode() .context("failed to encode wasm component")?; let component_bytes = self .strip_custom_sections(&component_bytes) .context("failed to strip debug sections from wasm component")?; let wasm_extension_api_version = parse_wasm_extension_version(&manifest.id, &component_bytes) .context("compiled wasm did not contain a valid zed extension api version")?; manifest.lib.version = Some(wasm_extension_api_version); let extension_file = extension_dir.join("extension.wasm"); fs::write(extension_file.clone(), &component_bytes) .context("failed to write extension.wasm")?; log::info!( "extension {} written to {}", extension_dir.display(), extension_file.display() ); Ok(()) } async fn compile_grammar( &self, extension_dir: &Path, grammar_name: &str, grammar_metadata: &GrammarManifestEntry, ) -> Result<()> { let clang_path = self.install_wasi_sdk_if_needed().await?; let mut grammar_repo_dir = extension_dir.to_path_buf(); grammar_repo_dir.extend(["grammars", grammar_name]); let mut grammar_wasm_path = grammar_repo_dir.clone(); grammar_wasm_path.set_extension("wasm"); log::info!("checking out {grammar_name} parser"); self.checkout_repo( &grammar_repo_dir, &grammar_metadata.repository, &grammar_metadata.rev, )?; let base_grammar_path = grammar_metadata .path .as_ref() .map(|path| grammar_repo_dir.join(path)) .unwrap_or(grammar_repo_dir); let src_path = base_grammar_path.join("src"); let parser_path = src_path.join("parser.c"); let scanner_path = src_path.join("scanner.c"); log::info!("compiling {grammar_name} parser"); let clang_output = Command::new(&clang_path) .args(["-fPIC", "-shared", "-Os"]) .arg(format!("-Wl,--export=tree_sitter_{grammar_name}")) .arg("-o") .arg(&grammar_wasm_path) .arg("-I") .arg(&src_path) .arg(&parser_path) .args(scanner_path.exists().then_some(scanner_path)) .output() .context("failed to run clang")?; if !clang_output.status.success() { bail!( "failed to compile {} parser with clang: {}", grammar_name, String::from_utf8_lossy(&clang_output.stderr), ); } Ok(()) } fn checkout_repo(&self, directory: &Path, url: &str, rev: &str) -> Result<()> { let git_dir = directory.join(".git"); if directory.exists() { let remotes_output = Command::new("git") .arg("--git-dir") .arg(&git_dir) .args(["remote", "-v"]) .output()?; let has_remote = remotes_output.status.success() && String::from_utf8_lossy(&remotes_output.stdout) .lines() .any(|line| { let mut parts = line.split(|c: char| c.is_whitespace()); parts.next() == Some("origin") && parts.any(|part| part == url) }); if !has_remote { bail!( "grammar directory '{}' already exists, but is not a git clone of '{}'", directory.display(), url ); } } else { fs::create_dir_all(directory).with_context(|| { format!("failed to create grammar directory {}", directory.display(),) })?; let init_output = Command::new("git") .arg("init") .current_dir(directory) .output()?; if !init_output.status.success() { bail!( "failed to run `git init` in directory '{}'", directory.display() ); } let remote_add_output = Command::new("git") .arg("--git-dir") .arg(&git_dir) .args(["remote", "add", "origin", url]) .output() .context("failed to execute `git remote add`")?; if !remote_add_output.status.success() { bail!( "failed to add remote {url} for git repository {}", git_dir.display() ); } } let fetch_output = Command::new("git") .arg("--git-dir") .arg(&git_dir) .args(["fetch", "--depth", "1", "origin", rev]) .output() .context("failed to execute `git fetch`")?; let checkout_output = Command::new("git") .arg("--git-dir") .arg(&git_dir) .args(["checkout", rev]) .current_dir(directory) .output() .context("failed to execute `git checkout`")?; if !checkout_output.status.success() { if !fetch_output.status.success() { bail!( "failed to fetch revision {} in directory '{}'", rev, directory.display() ); } bail!( "failed to checkout revision {} in directory '{}': {}", rev, directory.display(), String::from_utf8_lossy(&checkout_output.stderr) ); } Ok(()) } fn install_rust_wasm_target_if_needed(&self) -> Result<()> { let rustc_output = Command::new("rustc") .arg("--print") .arg("sysroot") .output() .context("failed to run rustc")?; if !rustc_output.status.success() { bail!( "failed to retrieve rust sysroot: {}", String::from_utf8_lossy(&rustc_output.stderr) ); } let sysroot = PathBuf::from(String::from_utf8(rustc_output.stdout)?.trim()); if sysroot.join("lib/rustlib").join(RUST_TARGET).exists() { return Ok(()); } let output = Command::new("rustup") .args(["target", "add", RUST_TARGET]) .stderr(Stdio::inherit()) .stdout(Stdio::inherit()) .output() .context("failed to run `rustup target add`")?; if !output.status.success() { bail!("failed to install the `{RUST_TARGET}` target"); } Ok(()) } async fn install_wasi_preview1_adapter_if_needed(&self) -> Result> { let cache_path = self.cache_dir.join("wasi_snapshot_preview1.reactor.wasm"); if let Ok(content) = fs::read(&cache_path) { if Parser::is_core_wasm(&content) { return Ok(content); } } fs::remove_file(&cache_path).ok(); log::info!( "downloading wasi adapter module to {}", cache_path.display() ); let mut response = self .http .get(WASI_ADAPTER_URL, AsyncBody::default(), true) .await?; let mut content = Vec::new(); let mut body = BufReader::new(response.body_mut()); body.read_to_end(&mut content).await?; fs::write(&cache_path, &content) .with_context(|| format!("failed to save file {}", cache_path.display()))?; if !Parser::is_core_wasm(&content) { bail!("downloaded wasi adapter is invalid"); } Ok(content) } async fn install_wasi_sdk_if_needed(&self) -> Result { let url = if let Some(asset_name) = WASI_SDK_ASSET_NAME { format!("{WASI_SDK_URL}/{asset_name}") } else { bail!("wasi-sdk is not available for platform {}", env::consts::OS); }; let wasi_sdk_dir = self.cache_dir.join("wasi-sdk"); let mut clang_path = wasi_sdk_dir.clone(); clang_path.extend(["bin", &format!("clang{}", env::consts::EXE_SUFFIX)]); if fs::metadata(&clang_path).map_or(false, |metadata| metadata.is_file()) { return Ok(clang_path); } let mut tar_out_dir = wasi_sdk_dir.clone(); tar_out_dir.set_extension("archive"); fs::remove_dir_all(&wasi_sdk_dir).ok(); fs::remove_dir_all(&tar_out_dir).ok(); log::info!("downloading wasi-sdk to {}", wasi_sdk_dir.display()); let mut response = self.http.get(&url, AsyncBody::default(), true).await?; let body = BufReader::new(response.body_mut()); let body = GzipDecoder::new(body); let tar = Archive::new(body); tar.unpack(&tar_out_dir) .await .context("failed to unpack wasi-sdk archive")?; let inner_dir = fs::read_dir(&tar_out_dir)? .next() .ok_or_else(|| anyhow!("no content"))? .context("failed to read contents of extracted wasi archive directory")? .path(); fs::rename(&inner_dir, &wasi_sdk_dir).context("failed to move extracted wasi dir")?; fs::remove_dir_all(&tar_out_dir).ok(); Ok(clang_path) } // This was adapted from: // https://github.com/bytecodealliance/wasm-tools/blob/1791a8f139722e9f8679a2bd3d8e423e55132b22/src/bin/wasm-tools/strip.rs fn strip_custom_sections(&self, input: &Vec) -> Result> { use wasmparser::Payload::*; let strip_custom_section = |name: &str| name.starts_with(".debug"); let mut output = Vec::new(); let mut stack = Vec::new(); for payload in Parser::new(0).parse_all(input) { let payload = payload?; let component_header = wasm_encoder::Component::HEADER; let module_header = wasm_encoder::Module::HEADER; // Track nesting depth, so that we don't mess with inner producer sections: match payload { Version { encoding, .. } => { output.extend_from_slice(match encoding { wasmparser::Encoding::Component => &component_header, wasmparser::Encoding::Module => &module_header, }); } ModuleSection { .. } | ComponentSection { .. } => { stack.push(mem::take(&mut output)); continue; } End { .. } => { let mut parent = match stack.pop() { Some(c) => c, None => break, }; if output.starts_with(&component_header) { parent.push(ComponentSectionId::Component as u8); output.encode(&mut parent); } else { parent.push(ComponentSectionId::CoreModule as u8); output.encode(&mut parent); } output = parent; } _ => {} } if let CustomSection(c) = &payload { if strip_custom_section(c.name()) { continue; } } if let Some((id, range)) = payload.as_section() { RawSection { id, data: &input[range], } .append_to(&mut output); } } Ok(output) } } fn populate_defaults(manifest: &mut ExtensionManifest, extension_path: &Path) -> Result<()> { // For legacy extensions on the v0 schema (aka, using `extension.json`), clear out any existing // contents of the computed fields, since we don't care what the existing values are. if manifest.schema_version.is_v0() { manifest.languages.clear(); manifest.grammars.clear(); manifest.themes.clear(); } let cargo_toml_path = extension_path.join("Cargo.toml"); if cargo_toml_path.exists() { manifest.lib.kind = Some(ExtensionLibraryKind::Rust); } let languages_dir = extension_path.join("languages"); if languages_dir.exists() { for entry in fs::read_dir(&languages_dir).context("failed to list languages dir")? { let entry = entry?; let language_dir = entry.path(); let config_path = language_dir.join("config.toml"); if config_path.exists() { let relative_language_dir = language_dir.strip_prefix(extension_path)?.to_path_buf(); if !manifest.languages.contains(&relative_language_dir) { manifest.languages.push(relative_language_dir); } } } } let themes_dir = extension_path.join("themes"); if themes_dir.exists() { for entry in fs::read_dir(&themes_dir).context("failed to list themes dir")? { let entry = entry?; let theme_path = entry.path(); if theme_path.extension() == Some("json".as_ref()) { let relative_theme_path = theme_path.strip_prefix(extension_path)?.to_path_buf(); if !manifest.themes.contains(&relative_theme_path) { manifest.themes.push(relative_theme_path); } } } } let snippets_json_path = extension_path.join("snippets.json"); if snippets_json_path.exists() { manifest.snippets = Some(snippets_json_path); } // For legacy extensions on the v0 schema (aka, using `extension.json`), we want to populate the grammars in // the manifest using the contents of the `grammars` directory. if manifest.schema_version.is_v0() { let grammars_dir = extension_path.join("grammars"); if grammars_dir.exists() { for entry in fs::read_dir(&grammars_dir).context("failed to list grammars dir")? { let entry = entry?; let grammar_path = entry.path(); if grammar_path.extension() == Some("toml".as_ref()) { #[derive(Deserialize)] struct GrammarConfigToml { pub repository: String, pub commit: String, #[serde(default)] pub path: Option, } let grammar_config = fs::read_to_string(&grammar_path)?; let grammar_config: GrammarConfigToml = toml::from_str(&grammar_config)?; let grammar_name = grammar_path .file_stem() .and_then(|stem| stem.to_str()) .ok_or_else(|| anyhow!("no grammar name"))?; if !manifest.grammars.contains_key(grammar_name) { manifest.grammars.insert( grammar_name.into(), GrammarManifestEntry { repository: grammar_config.repository, rev: grammar_config.commit, path: grammar_config.path, }, ); } } } } } Ok(()) }