Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Introduce a new Grammar struct and allow it to be optional

Browse source 
Co-Authored-By: Nathan Sobo <nathan@zed.dev>
This commit is contained in:
Antonio Scandurra 2021-11-29 17:38:59 +01:00
parent b9edde7b26
commit 2c17ae9aa6
11 changed files with 91 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Cargo.lock generated
View file

@ -5132,16 +5132,6 @@ dependencies = [
"regex", "regex",
] ]
[[package]]
name = "tree-sitter-markdown"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7c1cfe0396b0e500cc99067bcd2d48720eb08a077e2690194f0c3a3d105f9a0"
dependencies = [
"cc",
"tree-sitter",
]
[[package]] [[package]]
name = "tree-sitter-rust" name = "tree-sitter-rust"
version = "0.19.0" version = "0.19.0"
@ -5731,7 +5721,6 @@ dependencies = [
"tiny_http", "tiny_http",
"toml", "toml",
"tree-sitter", "tree-sitter",
"tree-sitter-markdown",
"tree-sitter-rust", "tree-sitter-rust",
"unindent", "unindent",
"url", "url",

4
crates/editor/src/display_map.rs
View file

@ -778,7 +778,7 @@ mod tests {
path_suffixes: vec![".test".to_string()], path_suffixes: vec![".test".to_string()],
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
) )
.with_highlights_query( .with_highlights_query(
r#" r#"
@ -865,7 +865,7 @@ mod tests {
path_suffixes: vec![".test".to_string()], path_suffixes: vec![".test".to_string()],
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
) )
.with_highlights_query( .with_highlights_query(
r#" r#"

8
crates/editor/src/lib.rs
View file

@ -5295,7 +5295,7 @@ mod tests {
let settings = cx.read(EditorSettings::test); let settings = cx.read(EditorSettings::test);
let language = Some(Arc::new(Language::new( let language = Some(Arc::new(Language::new(
LanguageConfig::default(), LanguageConfig::default(),
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let text = r#" let text = r#"
@ -5452,7 +5452,7 @@ mod tests {
], ],
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let text = r#" let text = r#"
@ -5551,7 +5551,7 @@ mod tests {
line_comment: Some("// ".to_string()), line_comment: Some("// ".to_string()),
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let text = " let text = "
@ -5649,7 +5649,7 @@ mod tests {
], ],
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let text = concat!( let text = concat!(

64
crates/language/src/language.rs
View file

@ -1,12 +1,12 @@
use crate::HighlightMap; use crate::HighlightMap;
use anyhow::Result; use anyhow::{anyhow, Result};
use gpui::{executor::Background, AppContext}; use gpui::{executor::Background, AppContext};
use lsp::LanguageServer; use lsp::LanguageServer;
use parking_lot::Mutex; use parking_lot::Mutex;
use serde::Deserialize; use serde::Deserialize;
use std::{collections::HashSet, path::Path, str, sync::Arc}; use std::{collections::HashSet, path::Path, str, sync::Arc};
use theme::SyntaxTheme; use theme::SyntaxTheme;
use tree_sitter::{Language as Grammar, Query}; use tree_sitter::{self, Query};
pub use tree_sitter::{Parser, Tree}; pub use tree_sitter::{Parser, Tree};
#[derive(Default, Deserialize)] #[derive(Default, Deserialize)]
@ -37,7 +37,11 @@ pub struct BracketPair {
pub struct Language { pub struct Language {
pub(crate) config: LanguageConfig, pub(crate) config: LanguageConfig,
pub(crate) grammar: Grammar, pub(crate) grammar: Option<Arc<Grammar>>,
}
pub struct Grammar {
pub(crate) ts_language: tree_sitter::Language,
pub(crate) highlights_query: Query, pub(crate) highlights_query: Query,
pub(crate) brackets_query: Query, pub(crate) brackets_query: Query,
pub(crate) indents_query: Query, pub(crate) indents_query: Query,
@ -86,29 +90,48 @@ impl LanguageRegistry {
} }
impl Language { impl Language {
pub fn new(config: LanguageConfig, grammar: Grammar) -> Self { pub fn new(config: LanguageConfig, ts_language: Option<tree_sitter::Language>) -> Self {
Self { Self {
config, config,
brackets_query: Query::new(grammar, "").unwrap(), grammar: ts_language.map(|ts_language| {
highlights_query: Query::new(grammar, "").unwrap(), Arc::new(Grammar {
indents_query: Query::new(grammar, "").unwrap(), brackets_query: Query::new(ts_language, "").unwrap(),
grammar, highlights_query: Query::new(ts_language, "").unwrap(),
indents_query: Query::new(ts_language, "").unwrap(),
ts_language,
highlight_map: Default::default(), highlight_map: Default::default(),
})
}),
} }
} }
pub fn with_highlights_query(mut self, source: &str) -> Result<Self> { pub fn with_highlights_query(mut self, source: &str) -> Result<Self> {
self.highlights_query = Query::new(self.grammar, source)?; let grammar = self
.grammar
.as_mut()
.and_then(Arc::get_mut)
.ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
grammar.highlights_query = Query::new(grammar.ts_language, source)?;
Ok(self) Ok(self)
} }
pub fn with_brackets_query(mut self, source: &str) -> Result<Self> { pub fn with_brackets_query(mut self, source: &str) -> Result<Self> {
self.brackets_query = Query::new(self.grammar, source)?; let grammar = self
.grammar
.as_mut()
.and_then(Arc::get_mut)
.ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
grammar.brackets_query = Query::new(grammar.ts_language, source)?;
Ok(self) Ok(self)
} }
pub fn with_indents_query(mut self, source: &str) -> Result<Self> { pub fn with_indents_query(mut self, source: &str) -> Result<Self> {
self.indents_query = Query::new(self.grammar, source)?; let grammar = self
.grammar
.as_mut()
.and_then(Arc::get_mut)
.ok_or_else(|| anyhow!("grammar does not exist or is already being used"))?;
grammar.indents_query = Query::new(grammar.ts_language, source)?;
Ok(self) Ok(self)
} }
@ -156,13 +179,17 @@ impl Language {
&self.config.brackets &self.config.brackets
} }
pub fn highlight_map(&self) -> HighlightMap { pub fn set_theme(&self, theme: &SyntaxTheme) {
self.highlight_map.lock().clone() if let Some(grammar) = self.grammar.as_ref() {
*grammar.highlight_map.lock() =
HighlightMap::new(grammar.highlights_query.capture_names(), theme);
}
}
} }
pub fn set_theme(&self, theme: &SyntaxTheme) { impl Grammar {
*self.highlight_map.lock() = pub fn highlight_map(&self) -> HighlightMap {
HighlightMap::new(self.highlights_query.capture_names(), theme); self.highlight_map.lock().clone()
} }
} }
@ -189,7 +216,6 @@ mod tests {
#[test] #[test]
fn test_select_language() { fn test_select_language() {
let grammar = tree_sitter_rust::language();
let registry = LanguageRegistry { let registry = LanguageRegistry {
languages: vec![ languages: vec![
Arc::new(Language::new( Arc::new(Language::new(
@ -198,7 +224,7 @@ mod tests {
path_suffixes: vec!["rs".to_string()], path_suffixes: vec!["rs".to_string()],
..Default::default() ..Default::default()
}, },
grammar, Some(tree_sitter_rust::language()),
)), )),
Arc::new(Language::new( Arc::new(Language::new(
LanguageConfig { LanguageConfig {
@ -206,7 +232,7 @@ mod tests {
path_suffixes: vec!["Makefile".to_string(), "mk".to_string()], path_suffixes: vec!["Makefile".to_string(), "mk".to_string()],
..Default::default() ..Default::default()
}, },
grammar, Some(tree_sitter_rust::language()),
)), )),
], ],
}; };

55
crates/language/src/lib.rs
View file

@ -6,7 +6,9 @@ mod tests;
pub use self::{ pub use self::{
highlight_map::{HighlightId, HighlightMap}, highlight_map::{HighlightId, HighlightMap},
language::{BracketPair, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig}, language::{
BracketPair, Grammar, Language, LanguageConfig, LanguageRegistry, LanguageServerConfig,
},
}; };
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
pub use buffer::{Buffer as TextBuffer, Operation as _, *}; pub use buffer::{Buffer as TextBuffer, Operation as _, *};
@ -594,13 +596,13 @@ impl Buffer {
return false; return false;
} }
if let Some(language) = self.language.clone() { if let Some(grammar) = self.grammar().cloned() {
let old_tree = self.syntax_tree(); let old_tree = self.syntax_tree();
let text = self.as_rope().clone(); let text = self.as_rope().clone();
let parsed_version = self.version(); let parsed_version = self.version();
let parse_task = cx.background().spawn({ let parse_task = cx.background().spawn({
let language = language.clone(); let grammar = grammar.clone();
async move { Self::parse_text(&text, old_tree, &language) } async move { Self::parse_text(&text, old_tree, &grammar) }
}); });
match cx match cx
@ -616,11 +618,10 @@ impl Buffer {
cx.spawn(move |this, mut cx| async move { cx.spawn(move |this, mut cx| async move {
let new_tree = parse_task.await; let new_tree = parse_task.await;
this.update(&mut cx, move |this, cx| { this.update(&mut cx, move |this, cx| {
let language_changed = let grammar_changed = this
this.language.as_ref().map_or(true, |curr_language| { .grammar()
!Arc::ptr_eq(curr_language, &language) .map_or(true, |curr_grammar| !Arc::ptr_eq(&grammar, curr_grammar));
}); let parse_again = this.version.gt(&parsed_version) || grammar_changed;
let parse_again = this.version.gt(&parsed_version) || language_changed;
this.parsing_in_background = false; this.parsing_in_background = false;
this.did_finish_parsing(new_tree, parsed_version, cx); this.did_finish_parsing(new_tree, parsed_version, cx);
@ -636,11 +637,11 @@ impl Buffer {
false false
} }
fn parse_text(text: &Rope, old_tree: Option<Tree>, language: &Language) -> Tree { fn parse_text(text: &Rope, old_tree: Option<Tree>, grammar: &Grammar) -> Tree {
PARSER.with(|parser| { PARSER.with(|parser| {
let mut parser = parser.borrow_mut(); let mut parser = parser.borrow_mut();
parser parser
.set_language(language.grammar) .set_language(grammar.ts_language)
.expect("incompatible grammar"); .expect("incompatible grammar");
let mut chunks = text.chunks_in_range(0..text.len()); let mut chunks = text.chunks_in_range(0..text.len());
let tree = parser let tree = parser
@ -1069,15 +1070,15 @@ impl Buffer {
&self, &self,
range: Range<T>, range: Range<T>,
) -> Option<(Range<usize>, Range<usize>)> { ) -> Option<(Range<usize>, Range<usize>)> {
let (lang, tree) = self.language.as_ref().zip(self.syntax_tree())?; let (grammar, tree) = self.grammar().zip(self.syntax_tree())?;
let open_capture_ix = lang.brackets_query.capture_index_for_name("open")?; let open_capture_ix = grammar.brackets_query.capture_index_for_name("open")?;
let close_capture_ix = lang.brackets_query.capture_index_for_name("close")?; let close_capture_ix = grammar.brackets_query.capture_index_for_name("close")?;
// Find bracket pairs that *inclusively* contain the given range. // Find bracket pairs that *inclusively* contain the given range.
let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1; let range = range.start.to_offset(self).saturating_sub(1)..range.end.to_offset(self) + 1;
let mut cursor = QueryCursorHandle::new(); let mut cursor = QueryCursorHandle::new();
let matches = cursor.set_byte_range(range).matches( let matches = cursor.set_byte_range(range).matches(
&lang.brackets_query, &grammar.brackets_query,
tree.root_node(), tree.root_node(),
TextProvider(self.as_rope()), TextProvider(self.as_rope()),
); );
@ -1342,6 +1343,10 @@ impl Buffer {
cx.notify(); cx.notify();
} }
fn grammar(&self) -> Option<&Arc<Grammar>> {
self.language.as_ref().and_then(|l| l.grammar.as_ref())
}
pub fn add_selection_set<T: ToOffset>( pub fn add_selection_set<T: ToOffset>(
&mut self, &mut self,
selections: &[Selection<T>], selections: &[Selection<T>],
@ -1550,19 +1555,19 @@ impl Snapshot {
row_range: Range<u32>, row_range: Range<u32>,
) -> Option<impl Iterator<Item = IndentSuggestion> + 'a> { ) -> Option<impl Iterator<Item = IndentSuggestion> + 'a> {
let mut query_cursor = QueryCursorHandle::new(); let mut query_cursor = QueryCursorHandle::new();
if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) { if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) {
let prev_non_blank_row = self.prev_non_blank_row(row_range.start); let prev_non_blank_row = self.prev_non_blank_row(row_range.start);
// Get the "indentation ranges" that intersect this row range. // Get the "indentation ranges" that intersect this row range.
let indent_capture_ix = language.indents_query.capture_index_for_name("indent"); let indent_capture_ix = grammar.indents_query.capture_index_for_name("indent");
let end_capture_ix = language.indents_query.capture_index_for_name("end"); let end_capture_ix = grammar.indents_query.capture_index_for_name("end");
query_cursor.set_point_range( query_cursor.set_point_range(
Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point() Point::new(prev_non_blank_row.unwrap_or(row_range.start), 0).to_ts_point()
..Point::new(row_range.end, 0).to_ts_point(), ..Point::new(row_range.end, 0).to_ts_point(),
); );
let mut indentation_ranges = Vec::<(Range<Point>, &'static str)>::new(); let mut indentation_ranges = Vec::<(Range<Point>, &'static str)>::new();
for mat in query_cursor.matches( for mat in query_cursor.matches(
&language.indents_query, &grammar.indents_query,
tree.root_node(), tree.root_node(),
TextProvider(self.as_rope()), TextProvider(self.as_rope()),
) { ) {
@ -1682,7 +1687,7 @@ impl Snapshot {
diagnostic_endpoints diagnostic_endpoints
.sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start)); .sort_unstable_by_key(|endpoint| (endpoint.offset, !endpoint.is_start));
if let Some((language, tree)) = self.language.as_ref().zip(self.tree.as_ref()) { if let Some((grammar, tree)) = self.grammar().zip(self.tree.as_ref()) {
let mut query_cursor = QueryCursorHandle::new(); let mut query_cursor = QueryCursorHandle::new();
// TODO - add a Tree-sitter API to remove the need for this. // TODO - add a Tree-sitter API to remove the need for this.
@ -1690,7 +1695,7 @@ impl Snapshot {
std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut()) std::mem::transmute::<_, &'static mut QueryCursor>(query_cursor.deref_mut())
}; };
let captures = cursor.set_byte_range(range.clone()).captures( let captures = cursor.set_byte_range(range.clone()).captures(
&language.highlights_query, &grammar.highlights_query,
tree.root_node(), tree.root_node(),
TextProvider(self.text.as_rope()), TextProvider(self.text.as_rope()),
); );
@ -1698,7 +1703,7 @@ impl Snapshot {
captures, captures,
next_capture: None, next_capture: None,
stack: Default::default(), stack: Default::default(),
highlight_map: language.highlight_map(), highlight_map: grammar.highlight_map(),
_query_cursor: query_cursor, _query_cursor: query_cursor,
theme, theme,
}) })
@ -1719,6 +1724,12 @@ impl Snapshot {
highlights, highlights,
} }
} }
fn grammar(&self) -> Option<&Arc<Grammar>> {
self.language
.as_ref()
.and_then(|language| language.grammar.as_ref())
}
} }
impl Clone for Snapshot { impl Clone for Snapshot {

2
crates/language/src/tests.rs
View file

@ -970,7 +970,7 @@ fn rust_lang() -> Language {
language_server: None, language_server: None,
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
) )
.with_indents_query( .with_indents_query(
r#" r#"

2
crates/project/src/worktree.rs
View file

@ -3671,7 +3671,7 @@ mod tests {
language_server: Some(language_server_config), language_server: Some(language_server_config),
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let dir = temp_tree(json!({ let dir = temp_tree(json!({

2
crates/server/src/rpc.rs
View file

@ -1616,7 +1616,7 @@ mod tests {
language_server: Some(language_server_config), language_server: Some(language_server_config),
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
let lang_registry = Arc::new(lang_registry); let lang_registry = Arc::new(lang_registry);

1
crates/zed/Cargo.toml
View file

@ -85,7 +85,6 @@ time = "0.3"
tiny_http = "0.8" tiny_http = "0.8"
toml = "0.5" toml = "0.5"
tree-sitter = "0.19.5" tree-sitter = "0.19.5"
tree-sitter-markdown = "0.7"
tree-sitter-rust = "0.19.0" tree-sitter-rust = "0.19.0"
url = "2.2" url = "2.2"

5
crates/zed/src/language.rs
View file

@ -17,7 +17,7 @@ pub fn build_language_registry() -> LanguageRegistry {
fn rust() -> Language { fn rust() -> Language {
let grammar = tree_sitter_rust::language(); let grammar = tree_sitter_rust::language();
let config = toml::from_slice(&LanguageDir::get("rust/config.toml").unwrap().data).unwrap(); let config = toml::from_slice(&LanguageDir::get("rust/config.toml").unwrap().data).unwrap();
Language::new(config, grammar) Language::new(config, Some(grammar))
.with_highlights_query(load_query("rust/highlights.scm").as_ref()) .with_highlights_query(load_query("rust/highlights.scm").as_ref())
.unwrap() .unwrap()
.with_brackets_query(load_query("rust/brackets.scm").as_ref()) .with_brackets_query(load_query("rust/brackets.scm").as_ref())
@ -27,9 +27,8 @@ fn rust() -> Language {
} }
fn markdown() -> Language { fn markdown() -> Language {
let grammar = tree_sitter_markdown::language();
let config = toml::from_slice(&LanguageDir::get("markdown/config.toml").unwrap().data).unwrap(); let config = toml::from_slice(&LanguageDir::get("markdown/config.toml").unwrap().data).unwrap();
Language::new(config, grammar) Language::new(config, None)
} }
fn load_query(path: &str) -> Cow<'static, str> { fn load_query(path: &str) -> Cow<'static, str> {

2
crates/zed/src/test.rs
View file

@ -30,7 +30,7 @@ pub fn test_app_state(cx: &mut MutableAppContext) -> Arc<AppState> {
path_suffixes: vec!["rs".to_string()], path_suffixes: vec!["rs".to_string()],
..Default::default() ..Default::default()
}, },
tree_sitter_rust::language(), Some(tree_sitter_rust::language()),
))); )));
Arc::new(AppState { Arc::new(AppState {
settings_tx: Arc::new(Mutex::new(settings_tx)), settings_tx: Arc::new(Mutex::new(settings_tx)),