Update tree-sitter to 0.24 (#24492)

I didn't update it to 0.25 because its Wasm support seems to be
partially broken due to
https://github.com/tree-sitter/tree-sitter/pull/3938: it didn't
introduce a check that the Wasm module's ABI is new enough to include
supertype info while parsing it, and so in the case where it isn't it
ends up interpreting random bytes as the number of supertypes, causing
out-of-bounds memory accesses.

Closes #24489

Release Notes:

- Fixed a rare crash during syntax highlighting

crates/editor/src/editor_tests.rs

@@ -9711,7 +9711,7 @@ async fn test_toggle_block_comment(cx: &mut gpui::TestAppContext) {
         &r#"
             <!-- ˇ<script> -->
                 // ˇvar x = new Y();
-            // ˇ</script>
+            <!-- ˇ</script> -->
         "#
         .unindent(),
     );

crates/extension_host/src/wasm_host/wit/since_v0_0_1.rs

@@ -84,7 +84,7 @@ impl HostWorktree for WasmState {
         latest::HostWorktree::which(self, delegate, binary_name).await
     }
 
-    fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
         Ok(())
     }
 }

crates/extension_host/src/wasm_host/wit/since_v0_0_4.rs

@@ -92,7 +92,7 @@ impl HostWorktree for WasmState {
         latest::HostWorktree::which(self, delegate, binary_name).await
     }
 
-    fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
         // We only ever hand out borrows of worktrees.
         Ok(())
     }

crates/extension_host/src/wasm_host/wit/since_v0_0_6.rs

@@ -147,7 +147,7 @@ impl HostWorktree for WasmState {
         latest::HostWorktree::which(self, delegate, binary_name).await
     }
 
-    fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
         // We only ever hand out borrows of worktrees.
         Ok(())
     }

crates/extension_host/src/wasm_host/wit/since_v0_1_0.rs

@@ -240,7 +240,7 @@ impl HostKeyValueStore for WasmState {
         kv_store.insert(key, value).await.to_wasmtime_result()
     }
 
-    fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
         // We only ever hand out borrows of key-value stores.
         Ok(())
     }
@@ -282,7 +282,7 @@ impl HostWorktree for WasmState {
         latest::HostWorktree::which(self, delegate, binary_name).await
     }
 
-    fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
         // We only ever hand out borrows of worktrees.
         Ok(())
     }
@@ -350,7 +350,7 @@ impl http_client::HostHttpResponseStream for WasmState {
         .to_wasmtime_result()
     }
 
-    fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
+    async fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
         Ok(())
     }
 }

crates/extension_host/src/wasm_host/wit/since_v0_2_0.rs

@@ -259,7 +259,7 @@ impl HostKeyValueStore for WasmState {
         kv_store.insert(key, value).await.to_wasmtime_result()
     }
 
-    fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<ExtensionKeyValueStore>) -> Result<()> {
         // We only ever hand out borrows of key-value stores.
         Ok(())
     }
@@ -275,7 +275,7 @@ impl HostProject for WasmState {
         Ok(project.worktree_ids())
     }
 
-    fn drop(&mut self, _project: Resource<Project>) -> Result<()> {
+    async fn drop(&mut self, _project: Resource<Project>) -> Result<()> {
         // We only ever hand out borrows of projects.
         Ok(())
     }
@@ -325,7 +325,7 @@ impl HostWorktree for WasmState {
         Ok(delegate.which(binary_name).await)
     }
 
-    fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
+    async fn drop(&mut self, _worktree: Resource<Worktree>) -> Result<()> {
         // We only ever hand out borrows of worktrees.
         Ok(())
     }
@@ -393,7 +393,7 @@ impl http_client::HostHttpResponseStream for WasmState {
         .to_wasmtime_result()
     }
 
-    fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
+    async fn drop(&mut self, _resource: Resource<ExtensionHttpResponseStream>) -> Result<()> {
         Ok(())
     }
 }

crates/language/Cargo.toml

@@ -53,6 +53,7 @@ settings.workspace = true
 similar.workspace = true
 smallvec.workspace = true
 smol.workspace = true
+streaming-iterator.workspace = true
 strsim.workspace = true
 sum_tree.workspace = true
 task.workspace = true

crates/language/src/syntax_map.rs

@@ -14,6 +14,7 @@ use std::{
     ops::{Deref, DerefMut, Range},
     sync::Arc,
 };
+use streaming_iterator::StreamingIterator;
 use sum_tree::{Bias, SeekTarget, SumTree};
 use text::{Anchor, BufferSnapshot, OffsetRangeExt, Point, Rope, ToOffset, ToPoint};
 use tree_sitter::{Node, Query, QueryCapture, QueryCaptures, QueryCursor, QueryMatches, Tree};
@@ -1143,7 +1144,7 @@ impl<'a> SyntaxMapMatches<'a> {
 
 impl<'a> SyntaxMapCapturesLayer<'a> {
     fn advance(&mut self) {
-        self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[ix]);
+        self.next_capture = self.captures.next().map(|(mat, ix)| mat.captures[*ix]);
     }
 
     fn sort_key(&self) -> (usize, Reverse<usize>, usize) {
@@ -1280,7 +1281,8 @@ fn get_injections(
 
     for query_range in changed_ranges {
         query_cursor.set_byte_range(query_range.start.saturating_sub(1)..query_range.end + 1);
-        for mat in query_cursor.matches(&config.query, node, TextProvider(text.as_rope())) {
+        let mut matches = query_cursor.matches(&config.query, node, TextProvider(text.as_rope()));
+        while let Some(mat) = matches.next() {
             let content_ranges = mat
                 .nodes_for_capture_index(config.content_capture_ix)
                 .map(|node| node.range())
@@ -1554,7 +1556,8 @@ impl<'a> SyntaxLayer<'a> {
         query_cursor.set_byte_range(offset.saturating_sub(1)..offset.saturating_add(1));
 
         let mut smallest_match: Option<(u32, Range<usize>)> = None;
-        for mat in query_cursor.matches(&config.query, self.node(), text) {
+        let mut matches = query_cursor.matches(&config.query, self.node(), text);
+        while let Some(mat) = matches.next() {
             for capture in mat.captures {
                 let Some(override_entry) = config.values.get(&capture.index) else {
                     continue;

crates/migrator/Cargo.toml

@@ -15,6 +15,7 @@ doctest = false
 [dependencies]
 collections.workspace = true
 convert_case.workspace = true
+streaming-iterator.workspace = true
 tree-sitter-json.workspace = true
 tree-sitter.workspace = true
 

crates/migrator/src/migrator.rs

@@ -1,6 +1,7 @@
 use collections::HashMap;
 use convert_case::{Case, Casing};
 use std::{cmp::Reverse, ops::Range, sync::LazyLock};
+use streaming_iterator::StreamingIterator;
 use tree_sitter::{Query, QueryMatch};
 
 fn migrate(text: &str, patterns: MigrationPatterns, query: &Query) -> Option<String> {
@@ -11,10 +12,10 @@ fn migrate(text: &str, patterns: MigrationPatterns, query: &Query) -> Option<Str
     let syntax_tree = parser.parse(&text, None).unwrap();
 
     let mut cursor = tree_sitter::QueryCursor::new();
-    let matches = cursor.matches(query, syntax_tree.root_node(), text.as_bytes());
+    let mut matches = cursor.matches(query, syntax_tree.root_node(), text.as_bytes());
 
     let mut edits = vec![];
-    for mat in matches {
+    while let Some(mat) = matches.next() {
         if let Some((_, callback)) = patterns.get(mat.pattern_index) {
             edits.extend(callback(&text, &mat, query));
         }

crates/semantic_index/Cargo.toml

@@ -42,6 +42,7 @@ serde_json.workspace = true
 settings.workspace = true
 sha2.workspace = true
 smol.workspace = true
+streaming-iterator.workspace = true
 theme.workspace = true
 tree-sitter.workspace = true
 ui.workspace = true

crates/semantic_index/src/chunking.rs

@@ -7,6 +7,7 @@ use std::{
     path::Path,
     sync::Arc,
 };
+use streaming_iterator::StreamingIterator;
 use tree_sitter::QueryCapture;
 use util::ResultExt as _;
 
@@ -88,7 +89,7 @@ fn syntactic_ranges(
     let mut ranges = with_query_cursor(|cursor| {
         cursor
             .matches(&outline.query, tree.root_node(), text.as_bytes())
-            .filter_map(|mat| {
+            .filter_map_deref(|mat| {
                 mat.captures
                     .iter()
                     .find_map(|QueryCapture { node, index }| {

crates/settings/Cargo.toml

@@ -32,6 +32,7 @@ serde_derive.workspace = true
 serde_json.workspace = true
 serde_json_lenient.workspace = true
 smallvec.workspace = true
+streaming-iterator.workspace = true
 tree-sitter-json.workspace = true
 tree-sitter.workspace = true
 util.workspace = true

crates/settings/src/settings_store.rs

@@ -17,6 +17,7 @@ use std::{
     str::{self, FromStr},
     sync::{Arc, LazyLock},
 };
+use streaming_iterator::StreamingIterator;
 use tree_sitter::Query;
 use util::RangeExt;
 
@@ -1262,8 +1263,8 @@ fn replace_value_in_json_text(
     let mut last_value_range = 0..0;
     let mut first_key_start = None;
     let mut existing_value_range = 0..text.len();
-    let matches = cursor.matches(&PAIR_QUERY, syntax_tree.root_node(), text.as_bytes());
-    for mat in matches {
+    let mut matches = cursor.matches(&PAIR_QUERY, syntax_tree.root_node(), text.as_bytes());
+    while let Some(mat) = matches.next() {
         if mat.captures.len() != 2 {
             continue;
         }