Re-enable language plugin functionality with some fixes (#7105)

Part of https://github.com/zed-industries/zed/issues/7096

* [x] Load all queries for language plugins, not just highlight query
* [x] Auto-reload languages when changing the `plugins` directory
* [x] Bump Tree-sitter for language loading and unloading fixes
* [x] Figure out code signing

Release Notes:

- N/A

---------

Co-authored-by: Antonio <antonio@zed.dev>
Co-authored-by: Marshall Bowers <elliott.codes@gmail.com>

crates/language/src/language.rs

@@ -52,7 +52,7 @@ use std::{
 };
 use syntax_map::SyntaxSnapshot;
 use theme::{SyntaxTheme, Theme};
-use tree_sitter::{self, Query};
+use tree_sitter::{self, wasmtime, Query, WasmStore};
 use unicase::UniCase;
 use util::{http::HttpClient, paths::PathExt};
 use util::{post_inc, ResultExt, TryFutureExt as _, UnwrapFuture};
@@ -96,7 +96,9 @@ impl LspBinaryStatusSender {
 
 thread_local! {
     static PARSER: RefCell<Parser> = {
-        RefCell::new(Parser::new())
+        let mut parser = Parser::new();
+        parser.set_wasm_store(WasmStore::new(WASM_ENGINE.clone()).unwrap()).unwrap();
+        RefCell::new(parser)
     };
 }
 
@@ -104,6 +106,7 @@ lazy_static! {
     pub(crate) static ref NEXT_GRAMMAR_ID: AtomicUsize = Default::default();
     /// A shared grammar for plain text, exposed for reuse by downstream crates.
     #[doc(hidden)]
+    pub static ref WASM_ENGINE: wasmtime::Engine = wasmtime::Engine::default();
     pub static ref PLAIN_TEXT: Arc<Language> = Arc::new(Language::new(
         LanguageConfig {
             name: "Plain Text".into(),
@@ -706,8 +709,8 @@ enum AvailableGrammar {
         get_queries: fn(&str) -> LanguageQueries,
     },
     Wasm {
-        _grammar_name: Arc<str>,
-        _path: Arc<Path>,
+        path: Arc<Path>,
+        get_queries: fn(&Path) -> LanguageQueries,
     },
 }
 
@@ -773,9 +776,6 @@ impl LanguageRegistry {
     }
 
     /// Clear out all of the loaded languages and reload them from scratch.
-    ///
-    /// This is useful in development, when queries have changed.
-    #[cfg(debug_assertions)]
     pub fn reload(&self) {
         self.state.write().reload();
     }
@@ -802,15 +802,17 @@ impl LanguageRegistry {
         });
     }
 
-    pub fn register_wasm(&self, path: Arc<Path>, grammar_name: Arc<str>, config: LanguageConfig) {
+    pub fn register_wasm(
+        &self,
+        path: Arc<Path>,
+        config: LanguageConfig,
+        get_queries: fn(&Path) -> LanguageQueries,
+    ) {
         let state = &mut *self.state.write();
         state.available_languages.push(AvailableLanguage {
             id: post_inc(&mut state.next_available_language_id),
             config,
-            grammar: AvailableGrammar::Wasm {
-                _grammar_name: grammar_name,
-                _path: path,
-            },
+            grammar: AvailableGrammar::Wasm { path, get_queries },
             lsp_adapters: Vec::new(),
             loaded: false,
         });
@@ -944,8 +946,23 @@ impl LanguageRegistry {
                                             asset_dir,
                                             get_queries,
                                         } => (grammar, (get_queries)(asset_dir)),
-                                        AvailableGrammar::Wasm { .. } => {
-                                            Err(anyhow!("not supported"))?
+                                        AvailableGrammar::Wasm { path, get_queries } => {
+                                            let grammar_name =
+                                                &language.config.grammar_name.as_ref().ok_or_else(
+                                                    || anyhow!("missing grammar name"),
+                                                )?;
+                                            let mut wasm_path = path.join(grammar_name.as_ref());
+                                            wasm_path.set_extension("wasm");
+                                            let wasm_bytes = std::fs::read(&wasm_path)?;
+                                            let grammar = PARSER.with(|parser| {
+                                                let mut parser = parser.borrow_mut();
+                                                let mut store = parser.take_wasm_store().unwrap();
+                                                let grammar =
+                                                    store.load_language(&grammar_name, &wasm_bytes);
+                                                parser.set_wasm_store(store).unwrap();
+                                                grammar
+                                            })?;
+                                            (grammar, get_queries(path.as_ref()))
                                         }
                                     };
                                     Language::new(language.config, Some(grammar))
@@ -1172,7 +1189,6 @@ impl LanguageRegistryState {
         *self.subscription.0.borrow_mut() = ();
     }
 
-    #[cfg(debug_assertions)]
     fn reload(&mut self) {
         self.languages.clear();
         self.version += 1;

crates/zed/resources/zed.entitlements

@@ -6,6 +6,10 @@
 	<true/>
 	<key>com.apple.security.cs.allow-jit</key>
 	<true/>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+	<!-- <key>com.apple.security.cs.disable-library-validation</key>
+	<true/> -->
 	<key>com.apple.security.device.audio-input</key>
 	<true/>
 	<key>com.apple.security.device.camera</key>
@@ -18,7 +22,5 @@
 	<true/>
 	<key>com.apple.security.personal-information.photos-library</key>
 	<true/>
-	<!-- <key>com.apple.security.cs.disable-library-validation</key>
-	<true/> -->
 </dict>
 </plist>

crates/zed/src/languages.rs

@@ -4,8 +4,8 @@ pub use language::*;
 use node_runtime::NodeRuntime;
 use rust_embed::RustEmbed;
 use settings::Settings;
-use std::{borrow::Cow, str, sync::Arc};
-use util::{asset_str, paths::PLUGINS_DIR};
+use std::{borrow::Cow, fs, path::Path, str, sync::Arc};
+use util::{asset_str, paths::PLUGINS_DIR, ResultExt};
 
 use self::{deno::DenoSettings, elixir::ElixirSettings};
 
@@ -303,10 +303,11 @@ pub fn init(
                 let path = child.path();
                 let config_path = path.join("config.toml");
                 if let Ok(config) = std::fs::read(&config_path) {
-                    let config: LanguageConfig = ::toml::from_slice(&config).unwrap();
-                    if let Some(grammar_name) = config.grammar_name.clone() {
-                        languages.register_wasm(path.into(), grammar_name, config);
-                    }
+                    languages.register_wasm(
+                        path.into(),
+                        ::toml::from_slice(&config).unwrap(),
+                        load_plugin_queries,
+                    );
                 }
             }
         }
@@ -338,27 +339,62 @@ fn load_config(name: &str) -> LanguageConfig {
     .unwrap()
 }
 
-fn load_queries(name: &str) -> LanguageQueries {
-    LanguageQueries {
-        highlights: load_query(name, "/highlights"),
-        brackets: load_query(name, "/brackets"),
-        indents: load_query(name, "/indents"),
-        outline: load_query(name, "/outline"),
-        embedding: load_query(name, "/embedding"),
-        injections: load_query(name, "/injections"),
-        overrides: load_query(name, "/overrides"),
-    }
-}
+const QUERY_FILENAME_PREFIXES: &[(
+    &str,
+    fn(&mut LanguageQueries) -> &mut Option<Cow<'static, str>>,
+)] = &[
+    ("highlights", |q| &mut q.highlights),
+    ("brackets", |q| &mut q.brackets),
+    ("outline", |q| &mut q.outline),
+    ("indents", |q| &mut q.indents),
+    ("embedding", |q| &mut q.embedding),
+    ("injections", |q| &mut q.injections),
+    ("overrides", |q| &mut q.overrides),
+];
 
-fn load_query(name: &str, filename_prefix: &str) -> Option<Cow<'static, str>> {
-    let mut result = None;
+fn load_queries(name: &str) -> LanguageQueries {
+    let mut result = LanguageQueries::default();
     for path in LanguageDir::iter() {
-        if let Some(remainder) = path.strip_prefix(name) {
-            if remainder.starts_with(filename_prefix) {
-                let contents = asset_str::<LanguageDir>(path.as_ref());
-                match &mut result {
-                    None => result = Some(contents),
-                    Some(r) => r.to_mut().push_str(contents.as_ref()),
+        if let Some(remainder) = path.strip_prefix(name).and_then(|p| p.strip_prefix('/')) {
+            if !remainder.ends_with(".scm") {
+                continue;
+            }
+            for (name, query) in QUERY_FILENAME_PREFIXES {
+                if remainder.starts_with(name) {
+                    let contents = asset_str::<LanguageDir>(path.as_ref());
+                    match query(&mut result) {
+                        None => *query(&mut result) = Some(contents),
+                        Some(r) => r.to_mut().push_str(contents.as_ref()),
+                    }
+                }
+            }
+        }
+    }
+    result
+}
+
+fn load_plugin_queries(root_path: &Path) -> LanguageQueries {
+    let mut result = LanguageQueries::default();
+    if let Some(entries) = fs::read_dir(root_path).log_err() {
+        for entry in entries {
+            let Some(entry) = entry.log_err() else {
+                continue;
+            };
+            let path = entry.path();
+            if let Some(remainder) = path.strip_prefix(root_path).ok().and_then(|p| p.to_str()) {
+                if !remainder.ends_with(".scm") {
+                    continue;
+                }
+                for (name, query) in QUERY_FILENAME_PREFIXES {
+                    if remainder.starts_with(name) {
+                        if let Some(contents) = fs::read_to_string(&path).log_err() {
+                            match query(&mut result) {
+                                None => *query(&mut result) = Some(contents.into()),
+                                Some(r) => r.to_mut().push_str(contents.as_ref()),
+                            }
+                        }
+                        break;
+                    }
                 }
             }
         }

crates/zed/src/main.rs

@@ -39,12 +39,13 @@ use std::{
         Arc,
     },
     thread,
+    time::Duration,
 };
 use theme::{ActiveTheme, ThemeRegistry, ThemeSettings};
 use util::{
     async_maybe,
     http::{self, HttpClient, ZedHttpClient},
-    paths::{self, CRASHES_DIR, CRASHES_RETIRED_DIR},
+    paths::{self, CRASHES_DIR, CRASHES_RETIRED_DIR, PLUGINS_DIR},
     ResultExt,
 };
 use uuid::Uuid;
@@ -894,26 +895,28 @@ fn load_embedded_fonts(cx: &AppContext) {
         .unwrap();
 }
 
-#[cfg(debug_assertions)]
-async fn watch_languages(fs: Arc<dyn fs::Fs>, languages: Arc<LanguageRegistry>) -> Option<()> {
-    use std::time::Duration;
+async fn watch_languages(fs: Arc<dyn fs::Fs>, languages: Arc<LanguageRegistry>) {
+    let reload_debounce = Duration::from_millis(250);
 
-    let mut events = fs
-        .watch(
-            "crates/zed/src/languages".as_ref(),
-            Duration::from_millis(100),
+    let mut events = fs.watch(PLUGINS_DIR.as_ref(), reload_debounce).await;
+
+    #[cfg(debug_assertions)]
+    {
+        events = futures::stream::select(
+            events,
+            fs.watch("crates/zed/src/languages".as_ref(), reload_debounce)
+                .await,
         )
-        .await;
+        .boxed();
+    }
+
     while (events.next().await).is_some() {
         languages.reload();
     }
-    Some(())
 }
 
 #[cfg(debug_assertions)]
 fn watch_file_types(fs: Arc<dyn fs::Fs>, cx: &mut AppContext) {
-    use std::time::Duration;
-
     cx.spawn(|cx| async move {
         let mut events = fs
             .watch(
@@ -933,10 +936,5 @@ fn watch_file_types(fs: Arc<dyn fs::Fs>, cx: &mut AppContext) {
     .detach()
 }
 
-#[cfg(not(debug_assertions))]
-async fn watch_languages(_: Arc<dyn fs::Fs>, _: Arc<LanguageRegistry>) -> Option<()> {
-    None
-}
-
 #[cfg(not(debug_assertions))]
 fn watch_file_types(_fs: Arc<dyn fs::Fs>, _cx: &mut AppContext) {}