editor: Use quantize score for code completions sort + Add code completions tests (#29182)

Closes #27994, #29050, #27352, #27616

This PR implements new logic for code completions, which improve cases
where local variables, etc LSP based hints are not shown on top of code
completion menu. The new logic is explained in comment of code.

This new sort is similar to VSCode's completions sort where order of
sort is like:

Fuzzy > Snippet > LSP sort_key > LSP sort_text 

whenever two items have same value, it proceeds to use next one as tie
breaker. Changing fuzzy score from float to int based makes it possible
for two items two have same fuzzy int score, making them get sorted by
next criteria.

Release Notes:

- Improved code completions to prioritize LSP hints, such as local
variables, so they appear at the top of the list.

crates/editor/src/code_context_menus.rs

@@ -27,8 +27,8 @@ use util::ResultExt;
 
 use crate::hover_popover::{hover_markdown_style, open_markdown_url};
 use crate::{
-    CodeActionProvider, CompletionId, CompletionProvider, DisplayRow, Editor, EditorStyle,
-    ResolvedTasks,
+    CodeActionProvider, CompletionId, CompletionItemKind, CompletionProvider, DisplayRow, Editor,
+    EditorStyle, ResolvedTasks,
     actions::{ConfirmCodeAction, ConfirmCompletion},
     split_words, styled_runs_for_code_label,
 };
@@ -657,6 +657,63 @@ impl CompletionsMenu {
         )
     }
 
+    pub fn sort_matches(matches: &mut Vec<SortableMatch<'_>>, query: Option<&str>) {
+        #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+        enum MatchTier<'a> {
+            WordStartMatch {
+                sort_score_int: Reverse<i32>,
+                sort_snippet: Reverse<i32>,
+                sort_text: Option<&'a str>,
+                sort_key: (usize, &'a str),
+            },
+            OtherMatch {
+                sort_score: Reverse<OrderedFloat<f64>>,
+            },
+        }
+
+        // Our goal here is to intelligently sort completion suggestions. We want to
+        // balance the raw fuzzy match score with hints from the language server
+        //
+        // We first primary sort using fuzzy score by putting matches into two buckets
+        // strong one and weak one. Among these buckets matches are then compared by
+        // various criteria like snippet, LSP hints, kind, label text etc.
+        //
+        const FUZZY_THRESHOLD: f64 = 0.1317;
+
+        let query_start_lower = query
+            .and_then(|q| q.chars().next())
+            .and_then(|c| c.to_lowercase().next());
+
+        matches.sort_unstable_by_key(|mat| {
+            let score = mat.string_match.score;
+
+            let is_other_match = query_start_lower
+                .map(|query_char| {
+                    !split_words(&mat.string_match.string).any(|word| {
+                        word.chars()
+                            .next()
+                            .and_then(|c| c.to_lowercase().next())
+                            .map_or(false, |word_char| word_char == query_char)
+                    })
+                })
+                .unwrap_or(false);
+
+            if is_other_match {
+                let sort_score = Reverse(OrderedFloat(score));
+                MatchTier::OtherMatch { sort_score }
+            } else {
+                let sort_score_int = Reverse(if score >= FUZZY_THRESHOLD { 1 } else { 0 });
+                let sort_snippet = Reverse(if mat.is_snippet { 1 } else { 0 });
+                MatchTier::WordStartMatch {
+                    sort_score_int,
+                    sort_snippet,
+                    sort_text: mat.sort_text,
+                    sort_key: mat.sort_key,
+                }
+            }
+        });
+    }
+
     pub async fn filter(&mut self, query: Option<&str>, executor: BackgroundExecutor) {
         let mut matches = if let Some(query) = query {
             fuzzy::match_strings(
@@ -681,85 +738,45 @@ impl CompletionsMenu {
                 .collect()
         };
 
-        let mut additional_matches = Vec::new();
-        // Deprioritize all candidates where the query's start does not match the start of any word in the candidate
-        if let Some(query) = query {
-            if let Some(query_start) = query.chars().next() {
-                let (primary, secondary) = matches.into_iter().partition(|string_match| {
-                    split_words(&string_match.string).any(|word| {
-                        // Check that the first codepoint of the word as lowercase matches the first
-                        // codepoint of the query as lowercase
-                        word.chars()
-                            .flat_map(|codepoint| codepoint.to_lowercase())
-                            .zip(query_start.to_lowercase())
-                            .all(|(word_cp, query_cp)| word_cp == query_cp)
-                    })
-                });
-                matches = primary;
-                additional_matches = secondary;
-            }
-        }
-
-        let completions = self.completions.borrow_mut();
         if self.sort_completions {
-            matches.sort_unstable_by_key(|mat| {
-                // We do want to strike a balance here between what the language server tells us
-                // to sort by (the sort_text) and what are "obvious" good matches (i.e. when you type
-                // `Creat` and there is a local variable called `CreateComponent`).
-                // So what we do is: we bucket all matches into two buckets
-                // - Strong matches
-                // - Weak matches
-                // Strong matches are the ones with a high fuzzy-matcher score (the "obvious" matches)
-                // and the Weak matches are the rest.
-                //
-                // For the strong matches, we sort by our fuzzy-finder score first and for the weak
-                // matches, we prefer language-server sort_text first.
-                //
-                // The thinking behind that: we want to show strong matches first in order of relevance(fuzzy score).
-                // Rest of the matches(weak) can be sorted as language-server expects.
+            let completions = self.completions.borrow();
 
-                #[derive(PartialEq, Eq, PartialOrd, Ord)]
-                enum MatchScore<'a> {
-                    Strong {
-                        score: Reverse<OrderedFloat<f64>>,
-                        sort_text: Option<&'a str>,
-                        sort_key: (usize, &'a str),
-                    },
-                    Weak {
-                        sort_text: Option<&'a str>,
-                        score: Reverse<OrderedFloat<f64>>,
-                        sort_key: (usize, &'a str),
-                    },
-                }
+            let mut sortable_items: Vec<SortableMatch<'_>> = matches
+                .into_iter()
+                .map(|string_match| {
+                    let completion = &completions[string_match.candidate_id];
 
-                let completion = &completions[mat.candidate_id];
-                let sort_key = completion.sort_key();
-                let sort_text =
-                    if let CompletionSource::Lsp { lsp_completion, .. } = &completion.source {
-                        lsp_completion.sort_text.as_deref()
-                    } else {
-                        None
-                    };
-                let score = Reverse(OrderedFloat(mat.score));
+                    let is_snippet = matches!(
+                        &completion.source,
+                        CompletionSource::Lsp { lsp_completion, .. }
+                        if lsp_completion.kind == Some(CompletionItemKind::SNIPPET)
+                    );
 
-                if mat.score >= 0.2 {
-                    MatchScore::Strong {
-                        score,
+                    let sort_text =
+                        if let CompletionSource::Lsp { lsp_completion, .. } = &completion.source {
+                            lsp_completion.sort_text.as_deref()
+                        } else {
+                            None
+                        };
+
+                    let sort_key = completion.sort_key();
+
+                    SortableMatch {
+                        string_match,
+                        is_snippet,
                         sort_text,
                         sort_key,
                     }
-                } else {
-                    MatchScore::Weak {
-                        sort_text,
-                        score,
-                        sort_key,
-                    }
-                }
-            });
+                })
+                .collect();
+
+            Self::sort_matches(&mut sortable_items, query);
+
+            matches = sortable_items
+                .into_iter()
+                .map(|sortable| sortable.string_match)
+                .collect();
         }
-        drop(completions);
-
-        matches.extend(additional_matches);
 
         *self.entries.borrow_mut() = matches;
         self.selected_item = 0;
@@ -768,6 +785,14 @@ impl CompletionsMenu {
     }
 }
 
+#[derive(Debug)]
+pub struct SortableMatch<'a> {
+    pub string_match: StringMatch,
+    pub is_snippet: bool,
+    pub sort_text: Option<&'a str>,
+    pub sort_key: (usize, &'a str),
+}
+
 #[derive(Clone)]
 pub struct AvailableCodeAction {
     pub excerpt_id: ExcerptId,

crates/editor/src/editor.rs

@@ -39,6 +39,8 @@ pub mod scroll;
 mod selections_collection;
 pub mod tasks;
 
+#[cfg(test)]
+mod code_completion_tests;
 #[cfg(test)]
 mod editor_tests;
 #[cfg(test)]

crates/editor/src/editor_tests.rs

@@ -10704,7 +10704,7 @@ async fn test_completion(cx: &mut TestAppContext) {
             .confirm_completion(&ConfirmCompletion::default(), window, cx)
             .unwrap()
     });
-    cx.assert_editor_state("editor.closeˇ");
+    cx.assert_editor_state("editor.clobberˇ");
     handle_resolve_completion_request(&mut cx, None).await;
     apply_additional_edits.await.unwrap();
 }
@@ -11266,76 +11266,6 @@ async fn test_completion_page_up_down_keys(cx: &mut TestAppContext) {
     });
 }
 
-#[gpui::test]
-async fn test_completion_sort(cx: &mut TestAppContext) {
-    init_test(cx, |_| {});
-    let mut cx = EditorLspTestContext::new_rust(
-        lsp::ServerCapabilities {
-            completion_provider: Some(lsp::CompletionOptions {
-                trigger_characters: Some(vec![".".to_string()]),
-                ..Default::default()
-            }),
-            ..Default::default()
-        },
-        cx,
-    )
-    .await;
-    cx.lsp
-        .set_request_handler::<lsp::request::Completion, _, _>(move |_, _| async move {
-            Ok(Some(lsp::CompletionResponse::Array(vec![
-                lsp::CompletionItem {
-                    label: "Range".into(),
-                    sort_text: Some("a".into()),
-                    ..Default::default()
-                },
-                lsp::CompletionItem {
-                    label: "r".into(),
-                    sort_text: Some("b".into()),
-                    ..Default::default()
-                },
-                lsp::CompletionItem {
-                    label: "ret".into(),
-                    sort_text: Some("c".into()),
-                    ..Default::default()
-                },
-                lsp::CompletionItem {
-                    label: "return".into(),
-                    sort_text: Some("d".into()),
-                    ..Default::default()
-                },
-                lsp::CompletionItem {
-                    label: "slice".into(),
-                    sort_text: Some("d".into()),
-                    ..Default::default()
-                },
-            ])))
-        });
-    cx.set_state("rˇ");
-    cx.executor().run_until_parked();
-    cx.update_editor(|editor, window, cx| {
-        editor.show_completions(
-            &ShowCompletions {
-                trigger: Some("r".into()),
-            },
-            window,
-            cx,
-        );
-    });
-    cx.executor().run_until_parked();
-
-    cx.update_editor(|editor, _, _| {
-        if let Some(CodeContextMenu::Completions(menu)) = editor.context_menu.borrow_mut().as_ref()
-        {
-            assert_eq!(
-                completion_menu_entries(&menu),
-                &["r", "ret", "Range", "return"]
-            );
-        } else {
-            panic!("expected completion menu to be open");
-        }
-    });
-}
-
 #[gpui::test]
 async fn test_as_is_completions(cx: &mut TestAppContext) {
     init_test(cx, |_| {});
@@ -14061,7 +13991,7 @@ async fn test_completions_in_languages_with_extra_word_characters(cx: &mut TestA
         {
             assert_eq!(
                 completion_menu_entries(&menu),
-                &["bg-red", "bg-blue", "bg-yellow"]
+                &["bg-blue", "bg-red", "bg-yellow"]
             );
         } else {
             panic!("expected completion menu to be open");

crates/project/src/project.rs

@@ -5006,7 +5006,7 @@ impl Completion {
     /// A key that can be used to sort completions when displaying
     /// them to the user.
     pub fn sort_key(&self) -> (usize, &str) {
-        const DEFAULT_KIND_KEY: usize = 2;
+        const DEFAULT_KIND_KEY: usize = 3;
         let kind_key = self
             .source
             // `lsp::CompletionListItemDefaults` has no `kind` field
@@ -5015,6 +5015,7 @@ impl Completion {
             .and_then(|lsp_completion_kind| match lsp_completion_kind {
                 lsp::CompletionItemKind::KEYWORD => Some(0),
                 lsp::CompletionItemKind::VARIABLE => Some(1),
+                lsp::CompletionItemKind::CONSTANT => Some(2),
                 _ => None,
             })
             .unwrap_or(DEFAULT_KIND_KEY);