Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Work on macro binding generation, some cleanup needed, rename runner to plugin

Browse source
This commit is contained in:
Isaac Clayton 2022-06-03 10:33:11 +02:00
parent dda6dcb3b8
commit f6a9558c5c
22 changed files with 330 additions and 59 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/plugin_runtime/Cargo.toml Normal file
View file

@ -0,0 +1,12 @@
[package]
name = "runner"
version = "0.1.0"
edition = "2021"
[dependencies]
mlua = { version = "0.8.0-beta.5", features = ["lua54", "vendored", "serialize"] }
serde = "1.0"
map-macro = "0.2"
wasmtime = "0.37.0"
anyhow = { version = "1.0", features = ["std"] }
bincode = "1.3"

6
crates/plugin_runtime/README.md Normal file
View file

@ -0,0 +1,6 @@
# Zed's Plugin Runner
This crate contains a fairly generic interface through which plugins may be added to extend the editor. Currently the intention of this plugin runtime is language server definitions.
Anything that implements the `Runtime` trait may be used as a plugin. Plugin interfaces are declared by implementing the `Interface` trait.
Wasm plugins can be run through `wasmtime`. We plan to add wasi support eventually. We also plan to add macros to generate bindings between Rust plugins compiled to Wasm and the host runtime.

24
crates/plugin_runtime/build.rs Normal file
View file

@ -0,0 +1,24 @@
use std::env;
use std::process::Command;
fn main() {
let cwd = std::env::current_dir().unwrap();
let plugin_workspace = cwd.join("plugin").join("Cargo.toml");
Command::new("cargo")
.args(&["clean", "--manifest-path"])
.arg(&plugin_workspace)
.status()
.unwrap();
Command::new("cargo")
.args(&[
"build",
"--release",
"--target",
"wasm32-unknown-unknown",
"--manifest-path",
])
.arg(&plugin_workspace)
.status()
.unwrap();
println!("cargo:warning=recompiling plugins")
}

0
crates/plugin_runtime/heck.txt Normal file
View file

180
crates/plugin_runtime/plugin/Cargo.lock generated Normal file
View file

@ -0,0 +1,180 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "bincode"
version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
dependencies = [
"serde",
]
[[package]]
name = "bumpalo"
version = "3.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3"
[[package]]
name = "cargo_test"
version = "0.1.0"
dependencies = [
"bincode",
"rust_plugin_macros",
"serde",
"wasm-bindgen",
]
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "itoa"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
[[package]]
name = "lazy_static"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "log"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
dependencies = [
"cfg-if",
]
[[package]]
name = "proc-macro2"
version = "1.0.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rust_plugin_macros"
version = "0.1.0"
dependencies = [
"bincode",
"proc-macro2",
"quote",
"serde",
"syn",
]
[[package]]
name = "ryu"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
[[package]]
name = "serde"
version = "1.0.137"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
[[package]]
name = "serde_json"
version = "1.0.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "1.0.96"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "unicode-ident"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
[[package]]
name = "wasm-bindgen"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
dependencies = [
"cfg-if",
"serde",
"serde_json",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"

2
crates/plugin_runtime/plugin/Cargo.toml Normal file
View file

@ -0,0 +1,2 @@
[workspace]
members = ["cargo_test"]

16
crates/plugin_runtime/plugin/cargo_test.lua Normal file
View file

@ -0,0 +1,16 @@
query = [[(
(attribute_item
(meta_item
(identifier) @test)) @attribute
.
(function_item
name: (identifier) @name) @funciton
)]]
function run_test(name)
local command = 'cargo test -- ' .. name
local openPop = assert(io.popen(command, 'r'))
local output = openPop:read('*all')
openPop:close()
return output
end

7
crates/plugin_runtime/plugin/cargo_test/Cargo.lock generated Normal file
View file

@ -0,0 +1,7 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 3
[[package]]
name = "cargo_test"
version = "0.1.0"

9
crates/plugin_runtime/plugin/cargo_test/Cargo.toml Normal file
View file

@ -0,0 +1,9 @@
[package]
name = "cargo_test"
version = "0.1.0"
edition = "2021"
[dependencies]
serde = "1.0"
bincode = "1.3"
plugin = { path = "../../../plugin" }

34
crates/plugin_runtime/plugin/cargo_test/src/main.rs Normal file
View file

@ -0,0 +1,34 @@
use core::slice;
use plugin::prelude::*;
#[no_mangle]
pub extern "C" fn banana(ptr: *const u8, len: usize) -> *const Buffer {
// setup
let buffer = Buffer { ptr, len };
let data = unsafe { buffer.to_vec() };
// operation
// let reversed: Vec<u8> = data.into_iter().rev().collect();
let number: f64 = bincode::deserialize(&data).unwrap();
let new_number = number * 2.0;
let new_data = bincode::serialize(&new_number).unwrap();
// teardown
let new_buffer = unsafe { Buffer::from_vec(new_data) };
return new_buffer.leak_to_heap();
}
pub fn banana2(number: f64) -> f64 {
number * 2.0
}
#[bind]
pub fn sum_lengths(strings: Vec<String>) -> usize {
let mut total = 0;
for string in strings {
total += string.len();
}
return total;
}
pub fn main() -> () {
()
}

13
crates/plugin_runtime/src/lib.rs Normal file
View file

@ -0,0 +1,13 @@
use mlua::{Function, Lua, LuaSerdeExt, Value};
use serde::{de::DeserializeOwned, Serialize};
pub use map_macro::{map, set};
pub mod runtime;
pub use runtime::*;
pub mod lua;
pub use lua::*;
pub mod wasm;
pub use wasm::*;

62
crates/plugin_runtime/src/lua.rs Normal file
View file

@ -0,0 +1,62 @@
use mlua::Result;
use crate::*;
impl Runtime for Lua {
type Plugin = LuaPlugin;
type Error = mlua::Error;
fn init(module: Self::Plugin) -> Result<Self> {
let lua = Lua::new();
// for action in module.actions {
// action(&mut lua).ok()?;
// }
lua.load(&module.source).exec()?;
return Ok(lua);
}
fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T> {
let val: Value = self.globals().get(handle.inner())?;
Ok(self.from_value(val)?)
}
fn call<A: Serialize, R: DeserializeOwned>(&mut self, handle: &Handle, arg: A) -> Result<R> {
let fun: Function = self.globals().get(handle.inner())?;
let arg: Value = self.to_value(&arg)?;
let result = fun.call(arg)?;
Ok(self.from_value(result)?)
}
fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
self.globals()
.contains_key(name.as_ref().to_string())
.unwrap_or(false)
}
}
pub struct LuaPlugin {
// name: String,
source: String,
// actions: Vec<Box<dyn FnOnce(&mut Lua) -> Result<(), ()>>>,
}
impl LuaPlugin {
pub fn new(
// name: String,
source: String,
) -> LuaPlugin {
LuaPlugin {
// name,
source,
// actions: Vec::new(),
}
}
// pub fn setup(mut self, action: fn(&mut Lua) -> Result<(), ()>) -> LuaPlugin {
// let action = Box::new(action);
// self.actions.push(action);
// self
// }
}

89
crates/plugin_runtime/src/main.rs Normal file
View file

@ -0,0 +1,89 @@
use mlua::Lua;
use runner::*;
// pub fn main() -> Result<(), mlua::Error> {
// let source = include_str!("../plugin/cargo_test.lua").to_string();
// let module = LuaPlugin::new(source);
// let mut lua: Lua = Runtime::init(module)?;
// let runner: TestRunner = lua.as_interface::<TestRunner>().unwrap();
// println!("extracted interface: {:#?}", &runner);
// let contents = runner.run_test(&mut lua, "it_works".into());
// println!("test results:{}", contents.unwrap());
// Ok(())
// }
// pub fn main() -> mlua::Result<()> {
// let module = LuaPlugin::new(include_str!("../plugin/cargo_test.lua").to_string());
// let mut lua: Lua = Runtime::init(module)?;
// let runner = lua.as_interface::<TestRunner>().unwrap();
// let test_results = runner.run_test(&mut lua, "it_works".into());
// Ok(())
// }
pub fn main() -> anyhow::Result<()> {
let plugin = WasmPlugin {
source_bytes: include_bytes!(
"../plugin/target/wasm32-unknown-unknown/release/cargo_test.wasm"
)
.to_vec(),
store_data: (),
};
let mut wasm: Wasm<()> = Runtime::init(plugin)?;
let banana = wasm.as_interface::<Banana>().unwrap();
let result = banana.banana(&mut wasm, 420.69);
dbg!("{}", result);
Ok(())
}
struct Banana {
banana: Handle,
}
impl Interface for Banana {
fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self> {
let banana = runtime.handle_for("banana")?;
Some(Banana { banana })
}
}
impl Banana {
fn banana<T: Runtime>(&self, runtime: &mut T, number: f64) -> Option<f64> {
runtime.call(&self.banana, number).ok()
}
}
#[allow(dead_code)]
#[derive(Debug)]
struct TestRunner {
pub query: String,
run_test: Handle,
}
impl Interface for TestRunner {
fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self> {
let run_test = runtime.handle_for("run_test")?;
let query = runtime.handle_for("query")?;
let query: String = runtime.constant(&query).ok()?;
Some(TestRunner { query, run_test })
}
}
impl TestRunner {
pub fn run_test<T: Runtime>(&self, runtime: &mut T, test_name: String) -> Option<String> {
runtime.call(&self.run_test, test_name).ok()
}
}
#[test]
pub fn it_works() {
panic!("huh, that was surprising...");
}

77
crates/plugin_runtime/src/runtime.rs Normal file
View file

@ -0,0 +1,77 @@
// use std::Error;
use serde::{de::DeserializeOwned, Serialize};
/// Represents a handle to a constant or function in the Runtime.
/// Should be constructed by calling [`Runtime::handle_for`].
#[derive(Debug, Clone, Hash, PartialEq, Eq)]
pub struct Handle(String);
impl Handle {
pub fn inner(&self) -> &str {
&self.0
}
}
/// Represents an interface that can be implemented by a plugin.
pub trait Interface
where
Self: Sized,
{
/// Create an interface from a given runtime.
/// All handles to be used by the interface should be registered and stored in `Self`.
fn from_runtime<T: Runtime>(runtime: &mut T) -> Option<Self>;
}
pub trait Runtime
where
Self: Sized,
{
/// Represents a plugin to be loaded by the runtime,
/// e.g. some source code + anything else needed to set up.
type Plugin;
/// The error type for this module.
/// Ideally should implement the [`std::err::Error`] trait.
type Error;
/// Initializes a plugin, returning a [`Runtime`] that can be queried.
/// Note that if you have any configuration,
fn init(plugin: Self::Plugin) -> Result<Self, Self::Error>;
/// Returns a top-level constant from the module.
/// This can be used to extract configuration information from the module, for example.
/// Before calling this function, get a handle into the runtime using [`handle_for`].
fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error>;
/// Call a function defined in the module.
fn call<A: Serialize, R: DeserializeOwned>(
&mut self,
handle: &Handle,
arg: A,
) -> Result<R, Self::Error>;
/// Registers a handle with the runtime.
/// This is a mutable item if needed, but generally
/// this should be an immutable operation.
/// Returns whether the handle exists/was successfully registered.
fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool;
/// Returns the handle for a given name if the handle is defined.
/// Will only return an error if there was an error while trying to register the handle.
/// This function uses [`register_handle`], no need to implement this one.
fn handle_for<T: AsRef<str>>(&mut self, name: T) -> Option<Handle> {
if self.register_handle(&name) {
Some(Handle(name.as_ref().to_string()))
} else {
None
}
}
/// Creates the given interface from the current module.
/// Returns [`Error`] if the provided plugin does not match the expected interface.
/// Essentially wraps the [`Interface`] trait.
fn as_interface<T: Interface>(&mut self) -> Option<T> {
Interface::from_runtime(self)
}
}

182
crates/plugin_runtime/src/wasm.rs Normal file
View file

@ -0,0 +1,182 @@
use std::collections::HashMap;
use anyhow::anyhow;
use wasmtime::{Engine, Func, Instance, Memory, MemoryType, Module, Store, TypedFunc};
use crate::*;
pub struct Wasm<T> {
engine: Engine,
module: Module,
store: Store<T>,
instance: Instance,
alloc_buffer: TypedFunc<i32, i32>,
// free_buffer: TypedFunc<(i32, i32), ()>,
}
pub struct WasmPlugin<T> {
pub source_bytes: Vec<u8>,
pub store_data: T,
}
impl<T> Wasm<T> {
pub fn dump_memory(data: &[u8]) {
for (i, byte) in data.iter().enumerate() {
if i % 32 == 0 {
println!();
}
if i % 4 == 0 {
print!("|");
}
if *byte == 0 {
print!("__")
} else {
print!("{:02x}", byte);
}
}
println!();
}
}
impl<S> Runtime for Wasm<S> {
type Plugin = WasmPlugin<S>;
type Error = anyhow::Error;
fn init(plugin: WasmPlugin<S>) -> Result<Self, Self::Error> {
let engine = Engine::default();
let module = Module::new(&engine, plugin.source_bytes)?;
let mut store: Store<S> = Store::new(&engine, plugin.store_data);
let instance = Instance::new(&mut store, &module, &[])?;
let alloc_buffer = instance.get_typed_func(&mut store, "__alloc_buffer")?;
// let free_buffer = instance.get_typed_func(&mut store, "__free_buffer")?;
Ok(Wasm {
engine,
module,
store,
instance,
alloc_buffer,
// free_buffer,
})
}
fn constant<T: DeserializeOwned>(&mut self, handle: &Handle) -> Result<T, Self::Error> {
let export = self
.instance
.get_export(&mut self.store, handle.inner())
.ok_or_else(|| anyhow!("Could not get export"))?;
todo!()
}
// So this call function is kinda a dance, I figured it'd be a good idea to document it.
// the high level is we take a serde type, serialize it to a byte array,
// (we're doing this using bincode for now)
// then toss that byte array into webassembly.
// webassembly grabs that byte array, does some magic,
// and serializes the result into yet another byte array.
// we then grab *that* result byte array and deserialize it into a result.
//
// phew...
//
// now the problem is, webassambly doesn't support buffers.
// only really like i32s, that's it (yeah, it's sad. Not even unsigned!)
// (ok, I'm exaggerating a bit).
//
// the Wasm function that this calls must have a very specific signature:
//
// fn(pointer to byte array: i32, length of byte array: i32)
// -> pointer to (
// pointer to byte_array: i32,
// length of byte array: i32,
// ): i32
//
// This pair `(pointer to byte array, length of byte array)` is called a `Buffer`
// and can be found in the cargo_test plugin.
//
// so on the wasm side, we grab the two parameters to the function,
// stuff them into a `Buffer`,
// and then pray to the `unsafe` Rust gods above that a valid byte array pops out.
//
// On the flip side, when returning from a wasm function,
// we convert whatever serialized result we get into byte array,
// which we stuff into a Buffer and allocate on the heap,
// which pointer to we then return.
// Note the double indirection!
//
// So when returning from a function, we actually leak memory *twice*:
//
// 1) once when we leak the byte array
// 2) again when we leak the allocated `Buffer`
//
// This isn't a problem because Wasm stops executing after the function returns,
// so the heap is still valid for our inspection when we want to pull things out.
// TODO: dont' use as for conversions
fn call<A: Serialize, R: DeserializeOwned>(
&mut self,
handle: &Handle,
arg: A,
) -> Result<R, Self::Error> {
// serialize the argument using bincode
let arg = bincode::serialize(&arg)?;
let arg_buffer_len = arg.len();
// allocate a buffer and write the argument to that buffer
let arg_buffer_ptr = self
.alloc_buffer
.call(&mut self.store, arg_buffer_len as i32)?;
let plugin_memory = self
.instance
.get_memory(&mut self.store, "memory")
.ok_or_else(|| anyhow!("Could not grab slice of plugin memory"))?;
plugin_memory.write(&mut self.store, arg_buffer_ptr as usize, &arg)?;
// get the webassembly function we want to actually call
let fun = self
.instance
.get_typed_func::<(i32, i32), i32, _>(&mut self.store, handle.inner())?;
// call the function, passing in the buffer and its length
// this should return a pointer to a (ptr, lentgh) pair
let arg_buffer = (arg_buffer_ptr, arg_buffer_len as i32);
let result_buffer = fun.call(&mut self.store, arg_buffer)?;
dbg!(result_buffer);
// panic!();
// dbg!()
// create a buffer to read the (ptr, length) pair into
// this is a total of 4 + 4 = 8 bytes.
let buffer = &mut [0; 8];
plugin_memory.read(&mut self.store, result_buffer as usize, buffer)?;
// use these bytes (wasm stores things little-endian)
// to get a pointer to the buffer and its length
let b = buffer;
let result_buffer_ptr = u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
let result_buffer_len = u32::from_le_bytes([b[4], b[5], b[6], b[7]]) as usize;
let result_buffer_end = result_buffer_ptr + result_buffer_len;
dbg!(result_buffer_ptr);
dbg!(result_buffer_len);
// read the buffer at this point into a byte array
// deserialize the byte array into the provided serde type
let result = &plugin_memory.data(&mut self.store)[result_buffer_ptr..result_buffer_end];
let result = bincode::deserialize(result)?;
// // deallocate the argument buffer
// self.free_buffer.call(&mut self.store, arg_buffer);
return Ok(result);
}
fn register_handle<T: AsRef<str>>(&mut self, name: T) -> bool {
self.instance
.get_export(&mut self.store, name.as_ref())
.is_some()
}
}