Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Get combined injections basically working

Browse source 
Co-authored-by: Nathan Sobo <nathan@zed.dev>
Co-authored-by: Mikayla Maki <mikayla@zed.dev>
This commit is contained in:
Max Brunsfeld 2022-11-07 16:58:12 -08:00
parent 5efe2ed6d3
commit c838a7d973
3 changed files with 344 additions and 190 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Cargo.lock generated
View file

@ -6383,7 +6383,7 @@ dependencies = [
[[package]] [[package]]
name = "tree-sitter" name = "tree-sitter"
version = "0.20.9" version = "0.20.9"
source = "git+https://github.com/tree-sitter/tree-sitter?rev=f0177f216e3f76a5f68e792b6f9e45fd32383eb6#f0177f216e3f76a5f68e792b6f9e45fd32383eb6" source = "git+https://github.com/tree-sitter/tree-sitter?rev=da6e24de1751aef6a944adfcefb192b751c56f76#da6e24de1751aef6a944adfcefb192b751c56f76"
dependencies = [ dependencies = [
"cc", "cc",
"regex", "regex",

2
Cargo.toml
View file

@ -65,7 +65,7 @@ serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] }
rand = { version = "0.8" } rand = { version = "0.8" }
[patch.crates-io] [patch.crates-io]
tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "f0177f216e3f76a5f68e792b6f9e45fd32383eb6" } tree-sitter = { git = "https://github.com/tree-sitter/tree-sitter", rev = "da6e24de1751aef6a944adfcefb192b751c56f76" }
async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" } async-task = { git = "https://github.com/zed-industries/async-task", rev = "341b57d6de98cdfd7b418567b8de2022ca993a6e" }
# TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457 # TODO - Remove when a version is released with this PR: https://github.com/servo/core-foundation-rs/pull/457

530
crates/language/src/syntax_map.rs
View file

@ -126,15 +126,15 @@ struct SyntaxLayerPositionBeforeChange {
change: DepthAndMaxPosition, change: DepthAndMaxPosition,
} }
struct ReparseStep { struct ParseStep {
depth: usize, depth: usize,
language: Arc<Language>, language: Arc<Language>,
range: Range<Anchor>, range: Range<Anchor>,
included_ranges: Vec<tree_sitter::Range>, included_ranges: Vec<tree_sitter::Range>,
mode: ReparseMode, mode: ParseMode,
} }
enum ReparseMode { enum ParseMode {
Single, Single,
Combined { Combined {
parent_layer_range: Range<usize>, parent_layer_range: Range<usize>,
@ -333,7 +333,7 @@ impl SyntaxSnapshot {
from_version: &clock::Global, from_version: &clock::Global,
text: &BufferSnapshot, text: &BufferSnapshot,
registry: Option<Arc<LanguageRegistry>>, registry: Option<Arc<LanguageRegistry>>,
language: Arc<Language>, root_language: Arc<Language>,
) { ) {
let edits = text.edits_since::<usize>(from_version).collect::<Vec<_>>(); let edits = text.edits_since::<usize>(from_version).collect::<Vec<_>>();
let max_depth = self.layers.summary().max_depth; let max_depth = self.layers.summary().max_depth;
@ -344,9 +344,9 @@ impl SyntaxSnapshot {
let mut changed_regions = ChangeRegionSet::default(); let mut changed_regions = ChangeRegionSet::default();
let mut queue = BinaryHeap::new(); let mut queue = BinaryHeap::new();
let mut combined_injection_ranges = HashMap::default(); let mut combined_injection_ranges = HashMap::default();
queue.push(ReparseStep { queue.push(ParseStep {
depth: 0, depth: 0,
language: language.clone(), language: root_language.clone(),
included_ranges: vec![tree_sitter::Range { included_ranges: vec![tree_sitter::Range {
start_byte: 0, start_byte: 0,
end_byte: text.len(), end_byte: text.len(),
@ -354,7 +354,7 @@ impl SyntaxSnapshot {
end_point: text.max_point().to_ts_point(), end_point: text.max_point().to_ts_point(),
}], }],
range: Anchor::MIN..Anchor::MAX, range: Anchor::MIN..Anchor::MAX,
mode: ReparseMode::Single, mode: ParseMode::Single,
}); });
loop { loop {
@ -394,7 +394,7 @@ impl SyntaxSnapshot {
while target.cmp(&cursor.end(text), text).is_gt() { while target.cmp(&cursor.end(text), text).is_gt() {
let Some(layer) = cursor.item() else { break }; let Some(layer) = cursor.item() else { break };
if changed_regions.intersects(&layer, text) { if changed_regions.intersects(&layer, text) && !layer.combined {
changed_regions.insert( changed_regions.insert(
ChangedRegion { ChangedRegion {
depth: layer.depth + 1, depth: layer.depth + 1,
@ -430,18 +430,17 @@ impl SyntaxSnapshot {
} }
} }
let mut combined = false; let combined = matches!(step.mode, ParseMode::Combined { .. });
let mut included_ranges = step.included_ranges; let mut included_ranges = step.included_ranges;
let tree; let tree;
let changed_ranges; let changed_ranges;
if let Some(old_layer) = old_layer { if let Some(old_layer) = old_layer {
if let ReparseMode::Combined { if let ParseMode::Combined {
parent_layer_changed_ranges, parent_layer_changed_ranges,
.. ..
} = step.mode } = step.mode
{ {
combined = true;
included_ranges = splice_included_ranges( included_ranges = splice_included_ranges(
old_layer.tree.included_ranges(), old_layer.tree.included_ranges(),
&parent_layer_changed_ranges, &parent_layer_changed_ranges,
@ -484,7 +483,7 @@ impl SyntaxSnapshot {
depth: step.depth, depth: step.depth,
range: step.range, range: step.range,
tree: tree.clone(), tree: tree.clone(),
language: language.clone(), language: step.language.clone(),
combined, combined,
}, },
&text, &text,
@ -974,13 +973,21 @@ fn get_injections(
depth: usize, depth: usize,
changed_ranges: &[Range<usize>], changed_ranges: &[Range<usize>],
combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>, combined_injection_ranges: &mut HashMap<Arc<Language>, Vec<tree_sitter::Range>>,
queue: &mut BinaryHeap<ReparseStep>, queue: &mut BinaryHeap<ParseStep>,
) -> bool { ) -> bool {
let mut result = false; let mut result = false;
let mut query_cursor = QueryCursorHandle::new(); let mut query_cursor = QueryCursorHandle::new();
let mut prev_match = None; let mut prev_match = None;
combined_injection_ranges.clear(); combined_injection_ranges.clear();
for pattern in &config.patterns {
if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined) {
if let Some(language) = language_registry.get_language(language_name) {
combined_injection_ranges.insert(language, Vec::new());
}
}
}
for query_range in changed_ranges { for query_range in changed_ranges {
query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end); query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end);
for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) { for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
@ -1020,16 +1027,16 @@ fn get_injections(
..text.anchor_after(content_range.end); ..text.anchor_after(content_range.end);
if combined { if combined {
combined_injection_ranges combined_injection_ranges
.entry(language.clone()) .get_mut(&language.clone())
.or_default() .unwrap()
.extend(content_ranges); .extend(content_ranges);
} else { } else {
queue.push(ReparseStep { queue.push(ParseStep {
depth, depth,
language, language,
included_ranges: content_ranges, included_ranges: content_ranges,
range, range,
mode: ReparseMode::Single, mode: ParseMode::Single,
}); });
} }
} }
@ -1040,12 +1047,12 @@ fn get_injections(
for (language, mut included_ranges) in combined_injection_ranges.drain() { for (language, mut included_ranges) in combined_injection_ranges.drain() {
included_ranges.sort_unstable(); included_ranges.sort_unstable();
let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte()); let range = text.anchor_before(node.start_byte())..text.anchor_after(node.end_byte());
queue.push(ReparseStep { queue.push(ParseStep {
depth, depth,
language, language,
range, range,
included_ranges, included_ranges,
mode: ReparseMode::Combined { mode: ParseMode::Combined {
parent_layer_range: node.start_byte()..node.end_byte(), parent_layer_range: node.start_byte()..node.end_byte(),
parent_layer_changed_ranges: changed_ranges.to_vec(), parent_layer_changed_ranges: changed_ranges.to_vec(),
}, },
@ -1081,7 +1088,8 @@ fn splice_included_ranges(
}; };
let end_ix = ranges_ix let end_ix = ranges_ix
+ match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) { + match ranges[ranges_ix..].binary_search_by_key(&changed.end, |r| r.start_byte) {
Ok(ix) | Err(ix) => ix, Ok(ix) => ix + 1,
Err(ix) => ix,
}; };
if end_ix > start_ix { if end_ix > start_ix {
ranges.splice(start_ix..end_ix, []); ranges.splice(start_ix..end_ix, []);
@ -1113,21 +1121,21 @@ impl std::ops::Deref for SyntaxMap {
} }
} }
impl PartialEq for ReparseStep { impl PartialEq for ParseStep {
fn eq(&self, _: &Self) -> bool { fn eq(&self, _: &Self) -> bool {
false false
} }
} }
impl Eq for ReparseStep {} impl Eq for ParseStep {}
impl PartialOrd for ReparseStep { impl PartialOrd for ParseStep {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> { fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(&other)) Some(self.cmp(&other))
} }
} }
impl Ord for ReparseStep { impl Ord for ParseStep {
fn cmp(&self, other: &Self) -> Ordering { fn cmp(&self, other: &Self) -> Ordering {
let range_a = self.range(); let range_a = self.range();
let range_b = other.range(); let range_b = other.range();
@ -1138,9 +1146,9 @@ impl Ord for ReparseStep {
} }
} }
impl ReparseStep { impl ParseStep {
fn range(&self) -> Range<usize> { fn range(&self) -> Range<usize> {
if let ReparseMode::Combined { if let ParseMode::Combined {
parent_layer_range, .. parent_layer_range, ..
} = &self.mode } = &self.mode
{ {
@ -1415,6 +1423,9 @@ mod tests {
] ]
); );
let new_ranges = splice_included_ranges(ranges.clone(), &[30..50], &[ts_range(25..55)]);
assert_eq!(new_ranges, &[ts_range(25..55), ts_range(80..90)]);
fn ts_range(range: Range<usize>) -> tree_sitter::Range { fn ts_range(range: Range<usize>) -> tree_sitter::Range {
tree_sitter::Range { tree_sitter::Range {
start_byte: range.start, start_byte: range.start,
@ -1530,21 +1541,24 @@ mod tests {
#[gpui::test] #[gpui::test]
fn test_typing_multiple_new_injections() { fn test_typing_multiple_new_injections() {
let (buffer, syntax_map) = test_edit_sequence(&[ let (buffer, syntax_map) = test_edit_sequence(
"fn a() { dbg }", "Rust",
"fn a() { dbg«!» }", &[
"fn a() { dbg!«()» }", "fn a() { dbg }",
"fn a() { dbg!(«b») }", "fn a() { dbg«!» }",
"fn a() { dbg!(b«.») }", "fn a() { dbg!«()» }",
"fn a() { dbg!(b.«c») }", "fn a() { dbg!(«b») }",
"fn a() { dbg!(b.c«()») }", "fn a() { dbg!(b«.») }",
"fn a() { dbg!(b.c(«vec»)) }", "fn a() { dbg!(b.«c») }",
"fn a() { dbg!(b.c(vec«!»)) }", "fn a() { dbg!(b.c«()») }",
"fn a() { dbg!(b.c(vec!«[]»)) }", "fn a() { dbg!(b.c(«vec»)) }",
"fn a() { dbg!(b.c(vec![«d»])) }", "fn a() { dbg!(b.c(vec«!»)) }",
"fn a() { dbg!(b.c(vec![d«.»])) }", "fn a() { dbg!(b.c(vec!«[]»)) }",
"fn a() { dbg!(b.c(vec![d.«e»])) }", "fn a() { dbg!(b.c(vec![«d»])) }",
]); "fn a() { dbg!(b.c(vec![d«.»])) }",
"fn a() { dbg!(b.c(vec![d.«e»])) }",
],
);
assert_capture_ranges( assert_capture_ranges(
&syntax_map, &syntax_map,
@ -1556,29 +1570,32 @@ mod tests {
#[gpui::test] #[gpui::test]
fn test_pasting_new_injection_line_between_others() { fn test_pasting_new_injection_line_between_others() {
let (buffer, syntax_map) = test_edit_sequence(&[ let (buffer, syntax_map) = test_edit_sequence(
" "Rust",
fn a() { &[
b!(B {}); "
c!(C {}); fn a() {
d!(D {}); b!(B {});
e!(E {}); c!(C {});
f!(F {}); d!(D {});
g!(G {}); e!(E {});
} f!(F {});
", g!(G {});
" }
fn a() { ",
b!(B {}); "
c!(C {}); fn a() {
d!(D {}); b!(B {});
« h!(H {}); c!(C {});
» e!(E {}); d!(D {});
f!(F {}); « h!(H {});
g!(G {}); » e!(E {});
} f!(F {});
", g!(G {});
]); }
",
],
);
assert_capture_ranges( assert_capture_ranges(
&syntax_map, &syntax_map,
@ -1600,28 +1617,31 @@ mod tests {
#[gpui::test] #[gpui::test]
fn test_joining_injections_with_child_injections() { fn test_joining_injections_with_child_injections() {
let (buffer, syntax_map) = test_edit_sequence(&[ let (buffer, syntax_map) = test_edit_sequence(
" "Rust",
fn a() { &[
b!( "
c![one.two.three], fn a() {
d![four.five.six], b!(
); c![one.two.three],
e!( d![four.five.six],
f![seven.eight], );
); e!(
} f![seven.eight],
", );
" }
fn a() { ",
b!( "
c![one.two.three], fn a() {
d![four.five.six], b!(
ˇ f![seven.eight], c![one.two.three],
); d![four.five.six],
} ˇ f![seven.eight],
", );
]); }
",
],
);
assert_capture_ranges( assert_capture_ranges(
&syntax_map, &syntax_map,
@ -1641,128 +1661,193 @@ mod tests {
#[gpui::test] #[gpui::test]
fn test_editing_edges_of_injection() { fn test_editing_edges_of_injection() {
test_edit_sequence(&[ test_edit_sequence(
" "Rust",
fn a() { &[
b!(c!()) "
} fn a() {
b!(c!())
}
",
"
fn a() {
«d»!(c!())
}
",
"
fn a() {
«e»d!(c!())
}
",
"
fn a() {
ed!«[»c!()«]»
}
", ",
" ],
fn a() { );
«d»!(c!())
}
",
"
fn a() {
«e»d!(c!())
}
",
"
fn a() {
ed!«[»c!()«]»
}
",
]);
} }
#[gpui::test] #[gpui::test]
fn test_edits_preceding_and_intersecting_injection() { fn test_edits_preceding_and_intersecting_injection() {
test_edit_sequence(&[ test_edit_sequence(
// "Rust",
"const aaaaaaaaaaaa: B = c!(d(e.f));", &[
"const aˇa: B = c!(d(eˇ));", //
]); "const aaaaaaaaaaaa: B = c!(d(e.f));",
"const aˇa: B = c!(d(eˇ));",
],
);
} }
#[gpui::test] #[gpui::test]
fn test_non_local_changes_create_injections() { fn test_non_local_changes_create_injections() {
test_edit_sequence(&[ test_edit_sequence(
" "Rust",
// a! { &[
static B: C = d; "
// } // a! {
", static B: C = d;
" // }
ˇa! { ",
static B: C = d; "
ˇ} ˇa! {
", static B: C = d;
]); ˇ}
",
],
);
} }
#[gpui::test] #[gpui::test]
fn test_creating_many_injections_in_one_edit() { fn test_creating_many_injections_in_one_edit() {
test_edit_sequence(&[ test_edit_sequence(
" "Rust",
fn a() { &[
one(Two::three(3)); "
four(Five::six(6)); fn a() {
seven(Eight::nine(9)); one(Two::three(3));
} four(Five::six(6));
", seven(Eight::nine(9));
" }
fn a() { ",
one«!»(Two::three(3)); "
four«!»(Five::six(6)); fn a() {
seven«!»(Eight::nine(9)); one«!»(Two::three(3));
} four«!»(Five::six(6));
", seven«!»(Eight::nine(9));
" }
fn a() { ",
one!(Two::three«!»(3)); "
four!(Five::six«!»(6)); fn a() {
seven!(Eight::nine«!»(9)); one!(Two::three«!»(3));
} four!(Five::six«!»(6));
", seven!(Eight::nine«!»(9));
]); }
",
],
);
} }
#[gpui::test] #[gpui::test]
fn test_editing_across_injection_boundary() { fn test_editing_across_injection_boundary() {
test_edit_sequence(&[ test_edit_sequence(
" "Rust",
fn one() { &[
two(); "
three!( fn one() {
three.four, two();
five.six, three!(
); three.four,
} five.six,
", );
" }
fn one() { ",
two(); "
th«irty_five![» fn one() {
three.four, two();
five.six, th«irty_five![»
« seven.eight, three.four,
];» five.six,
} « seven.eight,
", ];»
]); }
",
],
);
} }
#[gpui::test] #[gpui::test]
fn test_removing_injection_by_replacing_across_boundary() { fn test_removing_injection_by_replacing_across_boundary() {
test_edit_sequence(&[ test_edit_sequence(
"Rust",
&[
"
fn one() {
two!(
three.four,
);
}
",
"
fn one() {
t«en
.eleven(
twelve,
»
three.four,
);
}
",
],
);
}
#[gpui::test]
fn test_combined_injections() {
let (buffer, syntax_map) = test_edit_sequence(
"ERB",
&[
"
<body>
<% if @one %>
<div class=one>
<% else %>
<div class=two>
<% end %>
</div>
</body>
",
"
<body>
<% if @one %>
<div class=one>
ˇ else ˇ
<div class=two>
<% end %>
</div>
</body>
",
"
<body>
<% if @one «;» end %>
</div>
</body>
",
],
);
assert_capture_ranges(
&syntax_map,
&buffer,
&["tag", "ivar"],
" "
fn one() { <«body»>
two!( <% if «@one» ; end %>
three.four, </«div»>
); </«body»>
}
", ",
" );
fn one() {
t«en
.eleven(
twelve,
»
three.four,
);
}
",
]);
} }
#[gpui::test(iterations = 100)] #[gpui::test(iterations = 100)]
@ -1952,10 +2037,13 @@ mod tests {
} }
} }
fn test_edit_sequence(steps: &[&str]) -> (Buffer, SyntaxMap) { fn test_edit_sequence(language_name: &str, steps: &[&str]) -> (Buffer, SyntaxMap) {
let registry = Arc::new(LanguageRegistry::test()); let registry = Arc::new(LanguageRegistry::test());
let language = Arc::new(rust_lang()); registry.add(Arc::new(rust_lang()));
registry.add(language.clone()); registry.add(Arc::new(ruby_lang()));
registry.add(Arc::new(html_lang()));
registry.add(Arc::new(erb_lang()));
let language = registry.get_language(language_name).unwrap();
let mut buffer = Buffer::new(0, 0, Default::default()); let mut buffer = Buffer::new(0, 0, Default::default());
let mut mutated_syntax_map = SyntaxMap::new(); let mut mutated_syntax_map = SyntaxMap::new();
@ -2001,6 +2089,72 @@ mod tests {
(buffer, mutated_syntax_map) (buffer, mutated_syntax_map)
} }
fn html_lang() -> Language {
Language::new(
LanguageConfig {
name: "HTML".into(),
path_suffixes: vec!["html".to_string()],
..Default::default()
},
Some(tree_sitter_html::language()),
)
.with_highlights_query(
r#"
(tag_name) @tag
(erroneous_end_tag_name) @tag
(attribute_name) @property
"#,
)
.unwrap()
}
fn ruby_lang() -> Language {
Language::new(
LanguageConfig {
name: "Ruby".into(),
path_suffixes: vec!["rb".to_string()],
..Default::default()
},
Some(tree_sitter_ruby::language()),
)
.with_highlights_query(
r#"
["if" "do" "else" "end"] @keyword
(instance_variable) @ivar
"#,
)
.unwrap()
}
fn erb_lang() -> Language {
Language::new(
LanguageConfig {
name: "ERB".into(),
path_suffixes: vec!["erb".to_string()],
..Default::default()
},
Some(tree_sitter_embedded_template::language()),
)
.with_highlights_query(
r#"
["<%" "%>"] @keyword
"#,
)
.unwrap()
.with_injection_query(
r#"
((code) @content
(#set! "language" "ruby")
(#set! "combined"))
((content) @content
(#set! "language" "html")
(#set! "combined"))
"#,
)
.unwrap()
}
fn rust_lang() -> Language { fn rust_lang() -> Language {
Language::new( Language::new(
LanguageConfig { LanguageConfig {