editor: Improve code completion filtering to provide fewer and more accurate suggestions (#32928)

Closes #32756

- Uses `filter_text` from LSP source to filter items in completion list.
This fixes noisy lists like on typing `await` in Rust, it would suggest
`await.or`, `await.and`, etc., which are bad suggestions. Fallbacks to
label.
- Add `penalize_length` flag to fuzzy matcher, which was the default
behavior across. Now, this flag is set to `false` just for code
completion fuzzy matching. This fixes the case where if the query is
`unreac` and the completion items are `unreachable` and
`unreachable!()`, the item with a shorter length would have a larger
score than the other one, which is not right in the case of
auto-complete context. Now these two items will have the same fuzzy
score, and LSP `sort_text` will take over in finalizing its ranking.
- Updated test to be more utility based rather than example based. This
will help to iterate/verify logic faster on what's going on.

Before/After:

await: 
<img width="600" alt="before-await"
src="https://github.com/user-attachments/assets/384138dd-a90d-4942-a430-6ae15df37268"
/>
<img width="600" alt="after-await"
src="https://github.com/user-attachments/assets/d05a10fa-bae5-49bd-9fe7-9933ff215f29"
/>

iter:
<img width="600" alt="before-iter"
src="https://github.com/user-attachments/assets/6e57ffe9-007d-4b17-9cc2-d48fc0176c8e"
/>
<img width="600" alt="after-iter"
src="https://github.com/user-attachments/assets/a8577a9f-dcc8-4fd6-9ba0-b7590584ec31"
/>

opt:
<img width="600" alt="opt-before"
src="https://github.com/user-attachments/assets/d45b6c52-c9ee-4bf3-8552-d5e3fdbecbff"
/>
<img width="600" alt="opt-after"
src="https://github.com/user-attachments/assets/daac11a8-9699-48f8-b441-19fe9803848d"
/>

Release Notes:

- Improved code completion filtering to provide fewer and more accurate
suggestions.

crates/fuzzy/src/matcher.rs

@@ -17,6 +17,7 @@ pub struct Matcher<'a> {
     lowercase_query: &'a [char],
     query_char_bag: CharBag,
     smart_case: bool,
+    penalize_length: bool,
     min_score: f64,
     match_positions: Vec<usize>,
     last_positions: Vec<usize>,
@@ -35,6 +36,7 @@ impl<'a> Matcher<'a> {
         lowercase_query: &'a [char],
         query_char_bag: CharBag,
         smart_case: bool,
+        penalize_length: bool,
     ) -> Self {
         Self {
             query,
@@ -46,6 +48,7 @@ impl<'a> Matcher<'a> {
             score_matrix: Vec::new(),
             best_position_matrix: Vec::new(),
             smart_case,
+            penalize_length,
         }
     }
 
@@ -294,7 +297,7 @@ impl<'a> Matcher<'a> {
                 let mut multiplier = char_score;
 
                 // Scale the score based on how deep within the path we found the match.
-                if query_idx == 0 {
+                if self.penalize_length && query_idx == 0 {
                     multiplier /= ((prefix.len() + path.len()) - last_slash) as f64;
                 }
 
@@ -355,18 +358,18 @@ mod tests {
     #[test]
     fn test_get_last_positions() {
         let mut query: &[char] = &['d', 'c'];
-        let mut matcher = Matcher::new(query, query, query.into(), false);
+        let mut matcher = Matcher::new(query, query, query.into(), false, true);
         let result = matcher.find_last_positions(&['a', 'b', 'c'], &['b', 'd', 'e', 'f']);
         assert!(!result);
 
         query = &['c', 'd'];
-        let mut matcher = Matcher::new(query, query, query.into(), false);
+        let mut matcher = Matcher::new(query, query, query.into(), false, true);
         let result = matcher.find_last_positions(&['a', 'b', 'c'], &['b', 'd', 'e', 'f']);
         assert!(result);
         assert_eq!(matcher.last_positions, vec![2, 4]);
 
         query = &['z', '/', 'z', 'f'];
-        let mut matcher = Matcher::new(query, query, query.into(), false);
+        let mut matcher = Matcher::new(query, query, query.into(), false, true);
         let result = matcher.find_last_positions(&['z', 'e', 'd', '/'], &['z', 'e', 'd', '/', 'f']);
         assert!(result);
         assert_eq!(matcher.last_positions, vec![0, 3, 4, 8]);
@@ -613,7 +616,7 @@ mod tests {
             });
         }
 
-        let mut matcher = Matcher::new(&query, &lowercase_query, query_chars, smart_case);
+        let mut matcher = Matcher::new(&query, &lowercase_query, query_chars, smart_case, true);
 
         let cancel_flag = AtomicBool::new(false);
         let mut results = Vec::new();

crates/fuzzy/src/paths.rs

@@ -95,7 +95,7 @@ pub fn match_fixed_path_set(
     let query = query.chars().collect::<Vec<_>>();
     let query_char_bag = CharBag::from(&lowercase_query[..]);
 
-    let mut matcher = Matcher::new(&query, &lowercase_query, query_char_bag, smart_case);
+    let mut matcher = Matcher::new(&query, &lowercase_query, query_char_bag, smart_case, true);
 
     let mut results = Vec::new();
     matcher.match_candidates(
@@ -153,7 +153,7 @@ pub async fn match_path_sets<'a, Set: PathMatchCandidateSet<'a>>(
                     let segment_start = segment_idx * segment_size;
                     let segment_end = segment_start + segment_size;
                     let mut matcher =
-                        Matcher::new(query, lowercase_query, query_char_bag, smart_case);
+                        Matcher::new(query, lowercase_query, query_char_bag, smart_case, true);
 
                     let mut tree_start = 0;
                     for candidate_set in candidate_sets {

crates/fuzzy/src/strings.rs

@@ -117,6 +117,7 @@ pub async fn match_strings<T>(
     candidates: &[T],
     query: &str,
     smart_case: bool,
+    penalize_length: bool,
     max_results: usize,
     cancel_flag: &AtomicBool,
     executor: BackgroundExecutor,
@@ -160,8 +161,13 @@ where
                 scope.spawn(async move {
                     let segment_start = cmp::min(segment_idx * segment_size, candidates.len());
                     let segment_end = cmp::min(segment_start + segment_size, candidates.len());
-                    let mut matcher =
-                        Matcher::new(query, lowercase_query, query_char_bag, smart_case);
+                    let mut matcher = Matcher::new(
+                        query,
+                        lowercase_query,
+                        query_char_bag,
+                        smart_case,
+                        penalize_length,
+                    );
 
                     matcher.match_candidates(
                         &[],