Start switching JSON LSP adapter to plugin (take 2)

.gitignore

@@ -1,6 +1,7 @@
 **/target
 /zed.xcworkspace
 .DS_Store
+/plugins/bin
 /script/node_modules
 /styles/node_modules
 /crates/collab/.env.toml

Cargo.lock

@@ -3629,6 +3629,18 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "plugin_runtime"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "bincode",
+ "map-macro",
+ "mlua",
+ "serde",
+ "wasmtime",
+]
+
 [[package]]
 name = "png"
 version = "0.16.8"
@@ -4284,18 +4296,6 @@ dependencies = [
  "zeroize",
 ]
 
-[[package]]
-name = "runner"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "bincode",
- "map-macro",
- "mlua",
- "serde",
- "wasmtime",
-]
-
 [[package]]
 name = "rust-embed"
 version = "6.4.0"
@@ -6786,6 +6786,8 @@ dependencies = [
  "num_cpus",
  "outline",
  "parking_lot 0.11.2",
+ "plugin",
+ "plugin_runtime",
  "postage",
  "project",
  "project_panel",

crates/language/src/language.rs

@@ -88,8 +88,8 @@ pub trait LspAdapter: 'static + Send + Sync {
         None
     }
 
-    fn server_args(&self) -> &[&str] {
+    fn server_args(&self) -> Vec<String> {
-        &[]
+        Vec::new()
     }
 
     fn initialization_options(&self) -> Option<Value> {
@@ -366,7 +366,7 @@ impl LanguageRegistry {
             let server = lsp::LanguageServer::new(
                 server_id,
                 &server_binary_path,
-                server_args,
+                &server_args,
                 &root_path,
                 cx,
             )?;

crates/lsp/src/lsp.rs

@@ -101,10 +101,10 @@ struct Error {
 }
 
 impl LanguageServer {
-    pub fn new(
+    pub fn new<T: AsRef<std::ffi::OsStr>>(
         server_id: usize,
         binary_path: &Path,
-        args: &[&str],
+        args: &[T],
         root_path: &Path,
         cx: AsyncAppContext,
     ) -> Result<Self> {

crates/plugin/src/lib.rs

@@ -1,3 +1,6 @@
+pub use bincode;
+pub use serde;
+
 #[repr(C)]
 pub struct __Buffer {
     pub ptr: *const u8,

crates/plugin_macros/src/lib.rs

@@ -30,9 +30,9 @@ pub fn bind(args: TokenStream, function: TokenStream) -> TokenStream {
             let data = unsafe { buffer.to_vec() };
 
             // operation
-            let argument = ::bincode::deserialize(&data).unwrap();
+            let argument = ::plugin::bincode::deserialize(&data).unwrap();
             let result = #inner_fn_name(argument);
-            let new_data: Result<Vec<u8>, _> = ::bincode::serialize(&result);
+            let new_data: Result<Vec<u8>, _> = ::plugin::bincode::serialize(&result);
             let new_data = new_data.unwrap();
 
             // teardown

crates/plugin_runtime/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "runner"
+name = "plugin_runtime"
 version = "0.1.0"
 edition = "2021"
 

crates/plugin_runtime/build.rs

@@ -1,4 +1,3 @@
-use std::env;
 use std::process::Command;
 
 fn main() {

crates/plugin_runtime/src/lua.rs

@@ -17,23 +17,23 @@ impl Runtime for Lua {
         return Ok(lua);
     }
 
-    fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T> {
+    // fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T> {
-        let val: Value = self.globals().get(handle.inner())?;
+    //     let val: Value = self.globals().get(handle.inner())?;
-        Ok(self.from_value(val)?)
+    //     Ok(self.from_value(val)?)
-    }
+    // }
 
-    fn call<A: Serialize, R: DeserializeOwned>(&mut self, handle: &Handle, arg: A) -> Result<R> {
+    fn call<A: Serialize, R: DeserializeOwned>(&mut self, handle: &str, arg: A) -> Result<R> {
-        let fun: Function = self.globals().get(handle.inner())?;
+        let fun: Function = self.globals().get(handle.to_string())?;
         let arg: Value = self.to_value(&arg)?;
         let result = fun.call(arg)?;
         Ok(self.from_value(result)?)
     }
 
-    fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
+    // fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
-        self.globals()
+    //     self.globals()
-            .contains_key(name.as_ref().to_string())
+    //         .contains_key(name.as_ref().to_string())
-            .unwrap_or(false)
+    //         .unwrap_or(false)
-    }
+    // }
 }
 
 pub struct LuaPlugin {

crates/plugin_runtime/src/main.rs

@@ -1,6 +1,6 @@
 use mlua::Lua;
 
-use runner::*;
+use plugin_runtime::*;
 
 pub fn main() -> anyhow::Result<()> {
     let plugin = WasmPlugin {
@@ -12,7 +12,10 @@ pub fn main() -> anyhow::Result<()> {
     };
 
     let mut sum = Wasm::init(plugin)?;
-    let strings = "I hope you have a nice day".split(" ").iter().collect();
+    let strings = "I hope you have a nice day"
+        .split(" ")
+        .map(|x| x.to_string())
+        .collect();
     let result = sum.sum_lengths(strings);
 
     dbg!(result);
@@ -45,8 +48,7 @@ trait SumLengths {
 
 impl<T: Runtime> SumLengths for T {
     fn sum_lengths(&mut self, strings: Vec<String>) -> usize {
-        let handle = self.handle_for("sum_lengths").unwrap();
+        let result = self.call("sum_lengths", strings).ok().unwrap();
-        let result = self.call(&handle, strings).ok().unwrap();
         return result;
     }
 }

crates/plugin_runtime/src/runtime.rs

@@ -2,26 +2,26 @@
 
 use serde::{de::DeserializeOwned, Serialize};
 
-/// Represents a handle to a constant or function in the Runtime.
+// /// Represents a handle to a constant or function in the Runtime.
-/// Should be constructed by calling [`Runtime::handle_for`].
+// /// Should be constructed by calling [`Runtime::handle_for`].
-#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+// #[derive(Debug, Clone, Hash, PartialEq, Eq)]
-pub struct Handle(String);
+// pub struct Handle(String);
 
-impl Handle {
+// impl Handle {
-    pub fn inner(&self) -> &str {
+//     pub fn inner(&self) -> &str {
-        &self.0
+//         &self.0
-    }
+//     }
-}
+// }
 
-/// Represents an interface that can be implemented by a plugin.
+// /// Represents an interface that can be implemented by a plugin.
-pub trait Interface
+// pub trait Interface
-where
+// where
-    Self: Sized,
+//     Self: Sized,
-{
+// {
-    /// Create an interface from a given runtime.
+//     /// Create an interface from a given runtime.
-    /// All handles to be used by the interface should be registered and stored in `Self`.
+//     /// All handles to be used by the interface should be registered and stored in `Self`.
-    fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self>;
+//     fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self>;
-}
+// }
 
 pub trait Runtime
 where
@@ -39,39 +39,39 @@ where
     /// Note that if you have any configuration,
     fn init(plugin: Self::Plugin) -> Result<Self, Self::Error>;
 
-    /// Returns a top-level constant from the module.
+    // /// Returns a top-level constant from the module.
-    /// This can be used to extract configuration information from the module, for example.
+    // /// This can be used to extract configuration information from the module, for example.
-    /// Before calling this function, get a handle into the runtime using [`handle_for`].
+    // /// Before calling this function, get a handle into the runtime using [`handle_for`].
-    fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error>;
+    // fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error>;
 
     /// Call a function defined in the module.
     fn call<A: Serialize, R: DeserializeOwned>(
         &mut self,
-        handle: &Handle,
+        handle: &str,
         arg: A,
     ) -> Result<R, Self::Error>;
 
-    /// Registers a handle with the runtime.
+    // /// Registers a handle with the runtime.
-    /// This is a mutable item if needed, but generally
+    // /// This is a mutable item if needed, but generally
-    /// this should be an immutable operation.
+    // /// this should be an immutable operation.
-    /// Returns whether the handle exists/was successfully registered.
+    // /// Returns whether the handle exists/was successfully registered.
-    fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool;
+    // fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool;
 
-    /// Returns the handle for a given name if the handle is defined.
+    // /// Returns the handle for a given name if the handle is defined.
-    /// Will only return an error if there was an error while trying to register the handle.
+    // /// Will only return an error if there was an error while trying to register the handle.
-    /// This function uses [`register_handle`], no need to implement this one.
+    // /// This function uses [`register_handle`], no need to implement this one.
-    fn handle_for<T: AsRef<str>>(&mut self, name: T) -> Option<Handle> {
+    // fn handle_for<T: AsRef<str>>(&mut self, name: T) -> Option<Handle> {
-        if self.register_handle(&name) {
+    //     if self.register_handle(&name) {
-            Some(Handle(name.as_ref().to_string()))
+    //         Some(Handle(name.as_ref().to_string()))
-        } else {
+    //     } else {
-            None
+    //         None
-        }
+    //     }
-    }
+    // }
 
-    /// Creates the given interface from the current module.
+    // /// Creates the given interface from the current module.
-    /// Returns [`Error`] if the provided plugin does not match the expected interface.
+    // /// Returns [`Error`] if the provided plugin does not match the expected interface.
-    /// Essentially wraps the [`Interface`] trait.
+    // /// Essentially wraps the [`Interface`] trait.
-    fn as_interface<T: Interface>(&mut self) -> Option<T> {
+    // fn as_interface<T: Interface>(&mut self) -> Option<T> {
-        Interface::from_runtime(self)
+    //     Interface::from_runtime(self)
-    }
+    // }
 }

crates/plugin_runtime/src/wasm.rs

@@ -62,14 +62,14 @@ impl<S> Runtime for Wasm<S> {
         })
     }
 
-    fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error> {
+    // fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error> {
-        let export = self
+    //     let export = self
-            .instance
+    //         .instance
-            .get_export(&mut self.store, handle.inner())
+    //         .get_export(&mut self.store, handle.inner())
-            .ok_or_else(|| anyhow!("Could not get export"))?;
+    //         .ok_or_else(|| anyhow!("Could not get export"))?;
 
-        todo!()
+    //     todo!()
-    }
+    // }
 
     // So this call function is kinda a dance, I figured it'd be a good idea to document it.
     // the high level is we take a serde type, serialize it to a byte array,
@@ -117,7 +117,7 @@ impl<S> Runtime for Wasm<S> {
     // TODO: dont' use as for conversions
     fn call<A: Serialize, R: DeserializeOwned>(
         &mut self,
-        handle: &Handle,
+        handle: &str,
         arg: A,
     ) -> Result<R, Self::Error> {
         // serialize the argument using bincode
@@ -136,7 +136,7 @@ impl<S> Runtime for Wasm<S> {
 
         // get the webassembly function we want to actually call
         // TODO: precompute handle
-        let fun_name = format!("__{}", handle.inner());
+        let fun_name = format!("__{}", handle);
         let fun = self
             .instance
             .get_typed_func::<(i32, i32), i32, _>(&mut self.store, &fun_name)?;
@@ -170,9 +170,9 @@ impl<S> Runtime for Wasm<S> {
         return Ok(result);
     }
 
-    fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
+    // fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
-        self.instance
+    //     self.instance
-            .get_export(&mut self.store, name.as_ref())
+    //         .get_export(&mut self.store, name.as_ref())
-            .is_some()
+    //         .is_some()
-    }
+    // }
 }

crates/zed/Cargo.toml

@@ -39,6 +39,8 @@ journal = { path = "../journal" }
 language = { path = "../language" }
 lsp = { path = "../lsp" }
 outline = { path = "../outline" }
+plugin = { path = "../plugin" }
+plugin_runtime = { path = "../plugin_runtime" }
 project = { path = "../project" }
 project_panel = { path = "../project_panel" }
 project_symbols = { path = "../project_symbols" }

crates/zed/src/languages.rs

@@ -6,10 +6,11 @@ use std::{borrow::Cow, str, sync::Arc};
 mod c;
 mod go;
 mod installation;
-mod json;
 mod python;
+mod language_plugin;
 mod rust;
 mod typescript;
+// mod json;
 
 #[derive(RustEmbed)]
 #[folder = "src/languages"]
@@ -37,7 +38,7 @@ pub fn build_language_registry(login_shell_env_loaded: Task<()>) -> LanguageRegi
         (
             "json",
             tree_sitter_json::language(),
-            Some(Arc::new(json::JsonLspAdapter)),
+            Some(Arc::new(language_plugin::new_json())),
         ),
         (
             "markdown",

crates/zed/src/languages/json.rs

@@ -24,8 +24,8 @@ impl LspAdapter for JsonLspAdapter {
         LanguageServerName("vscode-json-languageserver".into())
     }
 
-    fn server_args(&self) -> &[&str] {
+    fn server_args(&self) -> Vec<String> {
-        &["--stdio"]
+        vec!["--stdio".into()]
     }
 
     fn fetch_latest_server_version(

crates/zed/src/languages/language_plugin.rs

@@ -0,0 +1,116 @@
+use super::installation::{npm_install_packages, npm_package_latest_version};
+use anyhow::{anyhow, Context, Result};
+use client::http::HttpClient;
+use futures::{future::BoxFuture, FutureExt, StreamExt};
+use language::{LanguageServerName, LspAdapter};
+use parking_lot::{Mutex, RwLock};
+use plugin_runtime::{Runtime, Wasm, WasmPlugin};
+use serde_json::json;
+use smol::fs;
+use std::{any::Any, path::PathBuf, sync::Arc};
+use util::{ResultExt, TryFutureExt};
+
+pub fn new_json() {}
+
+pub struct LanguagePluginLspAdapter {
+    runtime: Mutex<Wasm<()>>,
+}
+
+impl LanguagePluginLspAdapter {
+    pub fn new(plugin: WasmPlugin<()>) -> Self {
+        Self {
+            runtime: Mutex::new(Wasm::init(plugin).unwrap()),
+        }
+    }
+}
+
+struct Versions {
+    language_version: String,
+    server_version: String,
+}
+
+impl LspAdapter for LanguagePluginLspAdapter {
+    fn name(&self) -> LanguageServerName {
+        let name: String = self.runtime.lock().call("name", ()).unwrap();
+        LanguageServerName(name.into())
+    }
+
+    fn server_args<'a>(&'a self) -> Vec<String> {
+        self.runtime.lock().call("args", ()).unwrap()
+    }
+
+    fn fetch_latest_server_version(
+        &self,
+        _: Arc<dyn HttpClient>,
+    ) -> BoxFuture<'static, Result<Box<dyn 'static + Send + Any>>> {
+        let versions: Result<(String, String)> =
+            self.runtime.lock().call("fetch_latest_server_version", ());
+
+        async move {
+            if let Ok((language_version, server_version)) = versions {
+                Ok(Box::new(Versions {
+                    language_version,
+                    server_version,
+                }) as Box<_>)
+            } else {
+                panic!()
+            }
+        }
+        .boxed()
+    }
+
+    fn fetch_server_binary(
+        &self,
+        versions: Box<dyn 'static + Send + Any>,
+        _: Arc<dyn HttpClient>,
+        container_dir: PathBuf,
+    ) -> BoxFuture<'static, Result<PathBuf>> {
+        // TODO: async runtime
+        let result = self
+            .runtime
+            .lock()
+            .call("fetch_server_binary", container_dir);
+        async move { result }.boxed()
+    }
+
+    fn cached_server_binary(&self, container_dir: PathBuf) -> BoxFuture<'static, Option<PathBuf>> {
+        let result = self
+            .runtime
+            .lock()
+            .call("cached_server_binary", container_dir);
+        async move { result }.log_err().boxed()
+    }
+
+    fn process_diagnostics(&self, _: &mut lsp::PublishDiagnosticsParams) {}
+
+    fn label_for_completion(
+        &self,
+        item: &lsp::CompletionItem,
+        language: &language::Language,
+    ) -> Option<language::CodeLabel> {
+        use lsp::CompletionItemKind as Kind;
+        let len = item.label.len();
+        let grammar = language.grammar()?;
+        let kind = format!("{:?}", item.kind?);
+        let name: String = self
+            .runtime
+            .lock()
+            .call("label_for_completion", kind)
+            .ok()?;
+        let highlight_id = grammar.highlight_id_for_name(&name)?;
+        Some(language::CodeLabel {
+            text: item.label.clone(),
+            runs: vec![(0..len, highlight_id)],
+            filter_range: 0..len,
+        })
+    }
+
+    fn initialization_options(&self) -> Option<serde_json::Value> {
+        let result = self
+            .runtime
+            .lock()
+            .call("initialization_options", ())
+            .unwrap();
+        Some(result)
+    }
+}

crates/zed/src/languages/typescript.rs

@@ -28,8 +28,11 @@ impl LspAdapter for TypeScriptLspAdapter {
         LanguageServerName("typescript-language-server".into())
     }
 
-    fn server_args(&self) -> &[&str] {
+    fn server_args(&self) -> Vec<String> {
-        &["--stdio", "--tsserver-path", "node_modules/typescript/lib"]
+        ["--stdio", "--tsserver-path", "node_modules/typescript/lib"]
+            .into_iter()
+            .map(str::to_string)
+            .collect()
     }
 
     fn fetch_latest_server_version(

plugins/Cargo.lock

@@ -0,0 +1,80 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "json_language"
+version = "0.1.0"
+dependencies = [
+ "plugin",
+]
+
+[[package]]
+name = "plugin"
+version = "0.1.0"
+dependencies = [
+ "bincode",
+ "plugin_macros",
+ "serde",
+]
+
+[[package]]
+name = "plugin_macros"
+version = "0.1.0"
+dependencies = [
+ "bincode",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "syn",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.39"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+
+[[package]]
+name = "syn"
+version = "1.0.96"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"

plugins/Cargo.toml

@@ -0,0 +1,2 @@
+[workspace]
+members = ["./json_language"]

plugins/json_language/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "json_language"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+plugin = { path = "../../crates/plugin" }
+
+[lib]
+crate-type = ["cdylib"]

plugins/json_language/src/lib.rs

@@ -0,0 +1,6 @@
+use plugin::prelude::*;
+
+#[bind]
+pub fn add(a: (f64, f64)) -> f64 {
+    a.0 + a.1
+}

script/build-plugins

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -e
+
+echo "Clearing cached plugins..."
+cargo clean --manifest-path plugins/Cargo.toml
+
+echo "Building Wasm plugins..."
+cargo build --release --target wasm32-unknown-unknown --manifest-path plugins/Cargo.toml
+
+echo
+echo "Extracting binaries..."
+rm -rf plugins/bin
+mkdir plugins/bin
+
+for f in plugins/target/wasm32-unknown-unknown/release/*.wasm
+do
+    name=$(basename $f)
+    cp $f plugins/bin/$name
+    echo "- Extracted plugin $name"
+done
+
+echo
+echo "Creating .wat versions (for human inspection)..."
+
+for f in plugins/bin/*.wasm
+do
+    name=$(basename $f)
+    base=$(echo $name | sed "s/\..*//")
+    wasm2wat $f --output plugins/bin/$base.wat
+    echo "- Converted $base.wasm -> $base.wat"
+done
+
+echo
+echo "Optimizing plugins using wasm-opt..."
+
+for f in plugins/bin/*.wasm
+do
+    name=$(basename $f)
+    wasm-opt -Oz $f --output $f
+    echo "- Optimized $name"
+done
+
+echo
+echo "Done!"