Support word-based completions (#26410)

Closes https://github.com/zed-industries/zed/issues/4957


https://github.com/user-attachments/assets/ff491378-376d-48ec-b552-6cc80f74200b

Adds `"completions"` language settings section, to configure LSP and
word completions per language.
Word-based completions may be turned on never, always (returned along
with the LSP ones), and as a fallback if no LSP completion items were
returned.

Future work:

* words are matched with the same fuzzy matching code that the rest of
the completions are

This might worsen the completion menu's usability even more, and will
require work on better completion sorting.

* completion entries currently have no icons or other ways to indicate
those are coming from LSP or from word search, or from something else

* we may work with language scopes more intelligently, group words by
them and distinguish during completions

Release Notes:

- Supported word-based completions

---------

Co-authored-by: Max Brunsfeld <max@zed.dev>

crates/language/src/buffer.rs

@@ -4145,6 +4145,63 @@ impl BufferSnapshot {
             None
         }
     }
+
+    pub fn words_in_range(
+        &self,
+        query: Option<&str>,
+        range: Range<usize>,
+    ) -> HashMap<String, Range<Anchor>> {
+        if query.map_or(false, |query| query.is_empty()) {
+            return HashMap::default();
+        }
+
+        let classifier = CharClassifier::new(self.language.clone().map(|language| LanguageScope {
+            language,
+            override_id: None,
+        }));
+
+        let mut query_ix = 0;
+        let query = query.map(|query| query.chars().collect::<Vec<_>>());
+        let query_len = query.as_ref().map_or(0, |query| query.len());
+
+        let mut words = HashMap::default();
+        let mut current_word_start_ix = None;
+        let mut chunk_ix = range.start;
+        for chunk in self.chunks(range, false) {
+            for (i, c) in chunk.text.char_indices() {
+                let ix = chunk_ix + i;
+                if classifier.is_word(c) {
+                    if current_word_start_ix.is_none() {
+                        current_word_start_ix = Some(ix);
+                    }
+
+                    if let Some(query) = &query {
+                        if query_ix < query_len {
+                            let query_c = query.get(query_ix).expect(
+                                "query_ix is a vec of chars, which we access only if before the end",
+                            );
+                            if c.to_lowercase().eq(query_c.to_lowercase()) {
+                                query_ix += 1;
+                            }
+                        }
+                    }
+                    continue;
+                } else if let Some(word_start) = current_word_start_ix.take() {
+                    if query_ix == query_len {
+                        let word_range = self.anchor_before(word_start)..self.anchor_after(ix);
+                        words.insert(
+                            self.text_for_range(word_start..ix).collect::<String>(),
+                            word_range,
+                        );
+                    }
+                }
+                query_ix = 0;
+            }
+            chunk_ix += chunk.text.len();
+        }
+
+        words
+    }
 }
 
 fn indent_size_for_line(text: &text::BufferSnapshot, row: u32) -> IndentSize {

crates/language/src/buffer_tests.rs

@@ -13,6 +13,7 @@ use proto::deserialize_operation;
 use rand::prelude::*;
 use regex::RegexBuilder;
 use settings::SettingsStore;
+use std::collections::BTreeSet;
 use std::{
     env,
     ops::Range,
@@ -3140,6 +3141,93 @@ fn test_trailing_whitespace_ranges(mut rng: StdRng) {
     );
 }
 
+#[gpui::test]
+fn test_words_in_range(cx: &mut gpui::App) {
+    init_settings(cx, |_| {});
+
+    let contents = r#"let word=öäpple.bar你 Öäpple word2-öÄpPlE-Pizza-word ÖÄPPLE word"#;
+
+    let buffer = cx.new(|cx| {
+        let buffer = Buffer::local(contents, cx).with_language(Arc::new(rust_lang()), cx);
+        assert_eq!(buffer.text(), contents);
+        buffer.check_invariants();
+        buffer
+    });
+
+    buffer.update(cx, |buffer, _| {
+        let snapshot = buffer.snapshot();
+        assert_eq!(
+            BTreeSet::from_iter(["Pizza".to_string()]),
+            snapshot
+                .words_in_range(Some("piz"), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::from_iter([
+                "öäpple".to_string(),
+                "Öäpple".to_string(),
+                "öÄpPlE".to_string(),
+                "ÖÄPPLE".to_string(),
+            ]),
+            snapshot
+                .words_in_range(Some("öp"), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::from_iter([
+                "öÄpPlE".to_string(),
+                "Öäpple".to_string(),
+                "ÖÄPPLE".to_string(),
+                "öäpple".to_string(),
+            ]),
+            snapshot
+                .words_in_range(Some("öÄ"), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::default(),
+            snapshot
+                .words_in_range(Some("öÄ好"), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::from_iter(["bar你".to_string(),]),
+            snapshot
+                .words_in_range(Some("你"), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::default(),
+            snapshot
+                .words_in_range(Some(""), 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+        assert_eq!(
+            BTreeSet::from_iter([
+                "bar你".to_string(),
+                "öÄpPlE".to_string(),
+                "Öäpple".to_string(),
+                "ÖÄPPLE".to_string(),
+                "öäpple".to_string(),
+                "let".to_string(),
+                "Pizza".to_string(),
+                "word".to_string(),
+                "word2".to_string(),
+            ]),
+            snapshot
+                .words_in_range(None, 0..snapshot.len())
+                .into_keys()
+                .collect::<BTreeSet<_>>()
+        );
+    });
+}
+
 fn ruby_lang() -> Language {
     Language::new(
         LanguageConfig {

crates/language/src/language_settings.rs

@@ -79,10 +79,10 @@ pub struct LanguageSettings {
     /// The column at which to soft-wrap lines, for buffers where soft-wrap
     /// is enabled.
     pub preferred_line_length: u32,
-    // Whether to show wrap guides (vertical rulers) in the editor.
-    // Setting this to true will show a guide at the 'preferred_line_length' value
-    // if softwrap is set to 'preferred_line_length', and will show any
-    // additional guides as specified by the 'wrap_guides' setting.
+    /// Whether to show wrap guides (vertical rulers) in the editor.
+    /// Setting this to true will show a guide at the 'preferred_line_length' value
+    /// if softwrap is set to 'preferred_line_length', and will show any
+    /// additional guides as specified by the 'wrap_guides' setting.
     pub show_wrap_guides: bool,
     /// Character counts at which to show wrap guides (vertical rulers) in the editor.
     pub wrap_guides: Vec<usize>,
@@ -137,7 +137,7 @@ pub struct LanguageSettings {
     pub use_on_type_format: bool,
     /// Whether indentation of pasted content should be adjusted based on the context.
     pub auto_indent_on_paste: bool,
-    // Controls how the editor handles the autoclosed characters.
+    /// Controls how the editor handles the autoclosed characters.
     pub always_treat_brackets_as_autoclosed: bool,
     /// Which code actions to run on save
     pub code_actions_on_format: HashMap<String, bool>,
@@ -151,6 +151,8 @@ pub struct LanguageSettings {
     /// Whether to display inline and alongside documentation for items in the
     /// completions menu.
     pub show_completion_documentation: bool,
+    /// Completion settings for this language.
+    pub completions: CompletionSettings,
 }
 
 impl LanguageSettings {
@@ -306,6 +308,50 @@ pub struct AllLanguageSettingsContent {
     pub file_types: HashMap<Arc<str>, Vec<String>>,
 }
 
+/// Controls how completions are processed for this language.
+#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub struct CompletionSettings {
+    /// Controls how words are completed.
+    /// For large documents, not all words may be fetched for completion.
+    ///
+    /// Default: `fallback`
+    #[serde(default = "default_words_completion_mode")]
+    pub words: WordsCompletionMode,
+    /// Whether to fetch LSP completions or not.
+    ///
+    /// Default: true
+    #[serde(default = "default_true")]
+    pub lsp: bool,
+    /// When fetching LSP completions, determines how long to wait for a response of a particular server.
+    /// When set to 0, waits indefinitely.
+    ///
+    /// Default: 500
+    #[serde(default = "lsp_fetch_timeout_ms")]
+    pub lsp_fetch_timeout_ms: u64,
+}
+
+/// Controls how document's words are completed.
+#[derive(Copy, Clone, Debug, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
+#[serde(rename_all = "snake_case")]
+pub enum WordsCompletionMode {
+    /// Always fetch document's words for completions.
+    Enabled,
+    /// Only if LSP response errors/times out/is empty,
+    /// use document's words to show completions.
+    Fallback,
+    /// Never fetch or complete document's words for completions.
+    Disabled,
+}
+
+fn default_words_completion_mode() -> WordsCompletionMode {
+    WordsCompletionMode::Fallback
+}
+
+fn lsp_fetch_timeout_ms() -> u64 {
+    500
+}
+
 /// The settings for a particular language.
 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
 pub struct LanguageSettingsContent {
@@ -478,6 +524,8 @@ pub struct LanguageSettingsContent {
     ///
     /// Default: true
     pub show_completion_documentation: Option<bool>,
+    /// Controls how completions are processed for this language.
+    pub completions: Option<CompletionSettings>,
 }
 
 /// The behavior of `editor::Rewrap`.
@@ -1381,6 +1429,7 @@ fn merge_settings(settings: &mut LanguageSettings, src: &LanguageSettingsContent
         &mut settings.show_completion_documentation,
         src.show_completion_documentation,
     );
+    merge(&mut settings.completions, src.completions);
 }
 
 /// Allows to enable/disable formatting with Prettier