Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Load languages lazily in the background

Browse source
This commit is contained in:
Max Brunsfeld 2023-01-25 17:42:49 -08:00
parent 81ed961659
commit ddf4e1a316
12 changed files with 311 additions and 210 deletions
Download patch file Download diff file Expand all files Collapse all files

20
crates/language/src/buffer_tests.rs
View file

@ -51,7 +51,7 @@ fn test_line_endings(cx: &mut gpui::MutableAppContext) {
#[gpui::test]
fn test_select_language() {
let registry = LanguageRegistry::test();
let registry = Arc::new(LanguageRegistry::test());
registry.add(Arc::new(Language::new(
LanguageConfig {
name: "Rust".into(),
@ -71,27 +71,33 @@ fn test_select_language() {
// matching file extension
assert_eq!(
registry.select_language("zed/lib.rs").map(|l| l.name()),
registry.language_for_path("zed/lib.rs").map(|l| l.name()),
Some("Rust".into())
);
assert_eq!(
registry.select_language("zed/lib.mk").map(|l| l.name()),
registry.language_for_path("zed/lib.mk").map(|l| l.name()),
Some("Make".into())
);
// matching filename
assert_eq!(
registry.select_language("zed/Makefile").map(|l| l.name()),
registry.language_for_path("zed/Makefile").map(|l| l.name()),
Some("Make".into())
);
// matching suffix that is not the full file extension or filename
assert_eq!(registry.select_language("zed/cars").map(|l| l.name()), None);
assert_eq!(
registry.select_language("zed/a.cars").map(|l| l.name()),
registry.language_for_path("zed/cars").map(|l| l.name()),
None
);
assert_eq!(
registry.language_for_path("zed/a.cars").map(|l| l.name()),
None
);
assert_eq!(
registry.language_for_path("zed/sumk").map(|l| l.name()),
None
);
assert_eq!(registry.select_language("zed/sumk").map(|l| l.name()), None);
}
#[gpui::test]

256
crates/language/src/language.rs
View file

@ -16,7 +16,7 @@ use futures::{
future::{BoxFuture, Shared},
FutureExt, TryFutureExt,
};
use gpui::{MutableAppContext, Task};
use gpui::{executor::Background, MutableAppContext, Task};
use highlight_map::HighlightMap;
use lazy_static::lazy_static;
use parking_lot::{Mutex, RwLock};
@ -26,6 +26,7 @@ use serde::{de, Deserialize, Deserializer};
use serde_json::Value;
use std::{
any::Any,
borrow::Cow,
cell::RefCell,
fmt::Debug,
hash::Hash,
@ -89,8 +90,7 @@ pub struct CachedLspAdapter {
}
impl CachedLspAdapter {
pub async fn new<T: LspAdapter>(adapter: T) -> Arc<Self> {
let adapter = Box::new(adapter);
pub async fn new(adapter: Box<dyn LspAdapter>) -> Arc<Self> {
let name = adapter.name().await;
let server_args = adapter.server_args().await;
let initialization_options = adapter.initialization_options().await;
@ -248,6 +248,16 @@ pub struct LanguageConfig {
pub overrides: HashMap<String, LanguageConfigOverride>,
}
#[derive(Debug, Default)]
pub struct LanguageQueries {
pub highlights: Option<Cow<'static, str>>,
pub brackets: Option<Cow<'static, str>>,
pub indents: Option<Cow<'static, str>>,
pub outline: Option<Cow<'static, str>>,
pub injections: Option<Cow<'static, str>>,
pub overrides: Option<Cow<'static, str>>,
}
#[derive(Clone)]
pub struct LanguageScope {
language: Arc<Language>,
@ -407,8 +417,17 @@ pub enum LanguageServerBinaryStatus {
Failed { error: String },
}
struct AvailableLanguage {
path: &'static str,
config: LanguageConfig,
grammar: tree_sitter::Language,
lsp_adapter: Option<Box<dyn LspAdapter>>,
get_queries: fn(&str) -> LanguageQueries,
}
pub struct LanguageRegistry {
languages: RwLock<Vec<Arc<Language>>>,
available_languages: RwLock<Vec<AvailableLanguage>>,
language_server_download_dir: Option<Arc<Path>>,
lsp_binary_statuses_tx: async_broadcast::Sender<(Arc<Language>, LanguageServerBinaryStatus)>,
lsp_binary_statuses_rx: async_broadcast::Receiver<(Arc<Language>, LanguageServerBinaryStatus)>,
@ -422,6 +441,7 @@ pub struct LanguageRegistry {
>,
subscription: RwLock<(watch::Sender<()>, watch::Receiver<()>)>,
theme: RwLock<Option<Arc<Theme>>>,
executor: Option<Arc<Background>>,
version: AtomicUsize,
}
@ -431,6 +451,7 @@ impl LanguageRegistry {
Self {
language_server_download_dir: None,
languages: Default::default(),
available_languages: Default::default(),
lsp_binary_statuses_tx,
lsp_binary_statuses_rx,
login_shell_env_loaded: login_shell_env_loaded.shared(),
@ -438,6 +459,7 @@ impl LanguageRegistry {
subscription: RwLock::new(watch::channel()),
theme: Default::default(),
version: Default::default(),
executor: None,
}
}
@ -446,6 +468,44 @@ impl LanguageRegistry {
Self::new(Task::ready(()))
}
pub fn set_executor(&mut self, executor: Arc<Background>) {
self.executor = Some(executor);
}
pub fn register(
&self,
path: &'static str,
config: LanguageConfig,
grammar: tree_sitter::Language,
lsp_adapter: Option<Box<dyn LspAdapter>>,
get_queries: fn(&str) -> LanguageQueries,
) {
self.available_languages.write().push(AvailableLanguage {
path,
config,
grammar,
lsp_adapter,
get_queries,
});
}
pub fn language_names(&self) -> Vec<String> {
let mut result = self
.available_languages
.read()
.iter()
.map(|l| l.config.name.to_string())
.chain(
self.languages
.read()
.iter()
.map(|l| l.config.name.to_string()),
)
.collect::<Vec<_>>();
result.sort_unstable();
result
}
pub fn add(&self, language: Arc<Language>) {
if let Some(theme) = self.theme.read().clone() {
language.set_theme(&theme.editor.syntax);
@ -474,58 +534,79 @@ impl LanguageRegistry {
self.language_server_download_dir = Some(path.into());
}
pub fn language_for_name(&self, name: &str) -> Option<Arc<Language>> {
pub fn language_for_name(self: &Arc<Self>, name: &str) -> Option<Arc<Language>> {
let name = UniCase::new(name);
self.languages
.read()
.iter()
.find(|language| UniCase::new(language.name()) == name)
.cloned()
self.get_or_load_language(|config| UniCase::new(config.name.as_ref()) == name)
}
pub fn language_for_extension(&self, extension: &str) -> Option<Arc<Language>> {
let extension = UniCase::new(extension);
self.languages
.read()
.iter()
.find(|language| {
language
.config
pub fn language_for_name_or_extension(self: &Arc<Self>, string: &str) -> Option<Arc<Language>> {
let string = UniCase::new(string);
self.get_or_load_language(|config| {
UniCase::new(config.name.as_ref()) == string
|| config
.path_suffixes
.iter()
.any(|suffix| UniCase::new(suffix) == extension)
})
.cloned()
.any(|suffix| UniCase::new(suffix) == string)
})
}
pub fn to_vec(&self) -> Vec<Arc<Language>> {
self.languages.read().iter().cloned().collect()
}
pub fn language_names(&self) -> Vec<String> {
self.languages
.read()
.iter()
.map(|language| language.name().to_string())
.collect()
}
pub fn select_language(&self, path: impl AsRef<Path>) -> Option<Arc<Language>> {
pub fn language_for_path(self: &Arc<Self>, path: impl AsRef<Path>) -> Option<Arc<Language>> {
let path = path.as_ref();
let filename = path.file_name().and_then(|name| name.to_str());
let extension = path.extension().and_then(|name| name.to_str());
let path_suffixes = [extension, filename];
self.languages
self.get_or_load_language(|config| {
config
.path_suffixes
.iter()
.any(|suffix| path_suffixes.contains(&Some(suffix.as_str())))
})
}
fn get_or_load_language(
self: &Arc<Self>,
callback: impl Fn(&LanguageConfig) -> bool,
) -> Option<Arc<Language>> {
if let Some(language) = self
.languages
.read()
.iter()
.find(|language| {
language
.config
.path_suffixes
.iter()
.any(|suffix| path_suffixes.contains(&Some(suffix.as_str())))
})
.cloned()
.find(|language| callback(&language.config))
{
return Some(language.clone());
}
if let Some(executor) = self.executor.clone() {
let mut available_languages = self.available_languages.write();
if let Some(ix) = available_languages.iter().position(|l| callback(&l.config)) {
let language = available_languages.remove(ix);
drop(available_languages);
let name = language.config.name.clone();
let this = self.clone();
executor
.spawn(async move {
let queries = (language.get_queries)(&language.path);
let language = Language::new(language.config, Some(language.grammar))
.with_lsp_adapter(language.lsp_adapter)
.await;
match language.with_queries(queries) {
Ok(language) => this.add(Arc::new(language)),
Err(err) => {
log::error!("failed to load language {}: {}", name, err);
return;
}
};
})
.detach();
}
}
None
}
pub fn to_vec(&self) -> Vec<Arc<Language>> {
self.languages.read().iter().cloned().collect()
}
pub fn start_language_server(
@ -729,12 +810,70 @@ impl Language {
self.grammar.as_ref().map(|g| g.id)
}
pub fn with_queries(mut self, queries: LanguageQueries) -> Result<Self> {
if let Some(query) = queries.highlights {
self = self
.with_highlights_query(query.as_ref())
.expect("failed to evaluate highlights query");
}
if let Some(query) = queries.brackets {
self = self
.with_brackets_query(query.as_ref())
.expect("failed to load brackets query");
}
if let Some(query) = queries.indents {
self = self
.with_indents_query(query.as_ref())
.expect("failed to load indents query");
}
if let Some(query) = queries.outline {
self = self
.with_outline_query(query.as_ref())
.expect("failed to load outline query");
}
if let Some(query) = queries.injections {
self = self
.with_injection_query(query.as_ref())
.expect("failed to load injection query");
}
if let Some(query) = queries.overrides {
self = self
.with_override_query(query.as_ref())
.expect("failed to load override query");
}
Ok(self)
}
pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
grammar.highlights_query = Some(Query::new(grammar.ts_language, source)?);
Ok(self)
}
pub fn with_outline_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
let query = Query::new(grammar.ts_language, source)?;
let mut item_capture_ix = None;
let mut name_capture_ix = None;
let mut context_capture_ix = None;
get_capture_indices(
&query,
&mut [
("item", &mut item_capture_ix),
("name", &mut name_capture_ix),
("context", &mut context_capture_ix),
],
);
if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
grammar.outline_config = Some(OutlineConfig {
query,
item_capture_ix,
name_capture_ix,
context_capture_ix,
});
}
Ok(self)
}
pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
let query = Query::new(grammar.ts_language, source)?;
@ -785,31 +924,6 @@ impl Language {
Ok(self)
}
pub fn with_outline_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
let query = Query::new(grammar.ts_language, source)?;
let mut item_capture_ix = None;
let mut name_capture_ix = None;
let mut context_capture_ix = None;
get_capture_indices(
&query,
&mut [
("item", &mut item_capture_ix),
("name", &mut name_capture_ix),
("context", &mut context_capture_ix),
],
);
if let Some((item_capture_ix, name_capture_ix)) = item_capture_ix.zip(name_capture_ix) {
grammar.outline_config = Some(OutlineConfig {
query,
item_capture_ix,
name_capture_ix,
context_capture_ix,
});
}
Ok(self)
}
pub fn with_injection_query(mut self, source: &str) -> Result<Self> {
let grammar = self.grammar_mut();
let query = Query::new(grammar.ts_language, source)?;
@ -882,8 +996,10 @@ impl Language {
Arc::get_mut(self.grammar.as_mut().unwrap()).unwrap()
}
pub fn with_lsp_adapter(mut self, lsp_adapter: Arc<CachedLspAdapter>) -> Self {
self.adapter = Some(lsp_adapter);
pub async fn with_lsp_adapter(mut self, lsp_adapter: Option<Box<dyn LspAdapter>>) -> Self {
if let Some(adapter) = lsp_adapter {
self.adapter = Some(CachedLspAdapter::new(adapter).await);
}
self
}
@ -894,7 +1010,7 @@ impl Language {
) -> mpsc::UnboundedReceiver<lsp::FakeLanguageServer> {
let (servers_tx, servers_rx) = mpsc::unbounded();
self.fake_adapter = Some((servers_tx, fake_lsp_adapter.clone()));
let adapter = CachedLspAdapter::new(fake_lsp_adapter).await;
let adapter = CachedLspAdapter::new(Box::new(fake_lsp_adapter)).await;
self.adapter = Some(adapter);
servers_rx
}

26
crates/language/src/syntax_map.rs
View file

@ -381,7 +381,12 @@ impl SyntaxSnapshot {
cursor.next(text);
while let Some(layer) = cursor.item() {
let SyntaxLayerContent::Pending { language_name } = &layer.content else { unreachable!() };
if language_for_injection(language_name, &registry).is_some() {
if {
let language_registry = &registry;
language_registry.language_for_name_or_extension(language_name)
}
.is_some()
{
resolved_injection_ranges.push(layer.range.to_offset(text));
}
@ -1066,7 +1071,7 @@ fn get_injections(
config: &InjectionConfig,
text: &BufferSnapshot,
node: Node,
language_registry: &LanguageRegistry,
language_registry: &Arc<LanguageRegistry>,
depth: usize,
changed_ranges: &[Range<usize>],
combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
@ -1078,7 +1083,8 @@ fn get_injections(
combined_injection_ranges.clear();
for pattern in &config.patterns {
if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
if let Some(language) = language_for_injection(language_name, language_registry) {
if let Some(language) = language_registry.language_for_name_or_extension(language_name)
{
combined_injection_ranges.insert(language, Vec::new());
}
}
@ -1123,7 +1129,10 @@ fn get_injections(
};
if let Some(language_name) = language_name {
let language = language_for_injection(&language_name, language_registry);
let language = {
let language_name: &str = &language_name;
language_registry.language_for_name_or_extension(language_name)
};
let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);
if let Some(language) = language {
if combined {
@ -1171,15 +1180,6 @@ fn get_injections(
}
}
fn language_for_injection(
language_name: &str,
language_registry: &LanguageRegistry,
) -> Option<Arc<Language>> {
language_registry
.language_for_name(language_name)
.or_else(|| language_registry.language_for_extension(language_name))
}
fn splice_included_ranges(
mut ranges: Vec<tree_sitter::Range>,
changed_ranges: &[Range<usize>],