Reland invisibles (#19846)

Release Notes:

- Show invisibles in the editor

Relands #19298 

Trying to quantify a performance impact, it doesn't seem to impact much
visible in Instruments or in a micro-benchmark of Editor#layout_lines.
We're still taking a few hundred micro-seconds (+/- a lot) every time.
The ascii file has just ascii, where as the cc file has one control
character per line.

<img width="1055" alt="Screenshot 2024-10-28 at 12 14 53"
src="https://github.com/user-attachments/assets/1c382063-bb19-4e92-bbba-ed5e7c02309f">
<img width="1020" alt="Screenshot 2024-10-28 at 12 15 07"
src="https://github.com/user-attachments/assets/1789f65e-5f83-4c32-be47-7748c62c3703">

crates/editor/src/display_map.rs

@@ -21,6 +21,7 @@ mod block_map;
 mod crease_map;
 mod fold_map;
 mod inlay_map;
+pub(crate) mod invisibles;
 mod tab_map;
 mod wrap_map;
 
@@ -42,6 +43,7 @@ use gpui::{
 pub(crate) use inlay_map::Inlay;
 use inlay_map::{InlayMap, InlaySnapshot};
 pub use inlay_map::{InlayOffset, InlayPoint};
+use invisibles::{is_invisible, replacement};
 use language::{
     language_settings::language_settings, ChunkRenderer, OffsetUtf16, Point,
     Subscription as BufferSubscription,
@@ -56,6 +58,7 @@ use std::{
     any::TypeId,
     borrow::Cow,
     fmt::Debug,
+    iter,
     num::NonZeroU32,
     ops::{Add, Range, Sub},
     sync::Arc,
@@ -63,7 +66,7 @@ use std::{
 use sum_tree::{Bias, TreeMap};
 use tab_map::{TabMap, TabSnapshot};
 use text::LineIndent;
-use ui::WindowContext;
+use ui::{div, px, IntoElement, ParentElement, Styled, WindowContext};
 use wrap_map::{WrapMap, WrapSnapshot};
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -461,6 +464,98 @@ pub struct HighlightedChunk<'a> {
     pub renderer: Option<ChunkRenderer>,
 }
 
+impl<'a> HighlightedChunk<'a> {
+    fn highlight_invisibles(
+        self,
+        editor_style: &'a EditorStyle,
+    ) -> impl Iterator<Item = Self> + 'a {
+        let mut chars = self.text.chars().peekable();
+        let mut text = self.text;
+        let style = self.style;
+        let is_tab = self.is_tab;
+        let renderer = self.renderer;
+        iter::from_fn(move || {
+            let mut prefix_len = 0;
+            while let Some(&ch) = chars.peek() {
+                if !is_invisible(ch) {
+                    prefix_len += ch.len_utf8();
+                    chars.next();
+                    continue;
+                }
+                if prefix_len > 0 {
+                    let (prefix, suffix) = text.split_at(prefix_len);
+                    text = suffix;
+                    return Some(HighlightedChunk {
+                        text: prefix,
+                        style,
+                        is_tab,
+                        renderer: renderer.clone(),
+                    });
+                }
+                chars.next();
+                let (prefix, suffix) = text.split_at(ch.len_utf8());
+                text = suffix;
+                if let Some(replacement) = replacement(ch) {
+                    let background = editor_style.status.hint_background;
+                    let underline = editor_style.status.hint;
+                    return Some(HighlightedChunk {
+                        text: prefix,
+                        style: None,
+                        is_tab: false,
+                        renderer: Some(ChunkRenderer {
+                            render: Arc::new(move |_| {
+                                div()
+                                    .child(replacement)
+                                    .bg(background)
+                                    .text_decoration_1()
+                                    .text_decoration_color(underline)
+                                    .into_any_element()
+                            }),
+                            constrain_width: false,
+                        }),
+                    });
+                } else {
+                    let invisible_highlight = HighlightStyle {
+                        background_color: Some(editor_style.status.hint_background),
+                        underline: Some(UnderlineStyle {
+                            color: Some(editor_style.status.hint),
+                            thickness: px(1.),
+                            wavy: false,
+                        }),
+                        ..Default::default()
+                    };
+                    let invisible_style = if let Some(mut style) = style {
+                        style.highlight(invisible_highlight);
+                        style
+                    } else {
+                        invisible_highlight
+                    };
+
+                    return Some(HighlightedChunk {
+                        text: prefix,
+                        style: Some(invisible_style),
+                        is_tab: false,
+                        renderer: renderer.clone(),
+                    });
+                }
+            }
+
+            if !text.is_empty() {
+                let remainder = text;
+                text = "";
+                Some(HighlightedChunk {
+                    text: remainder,
+                    style,
+                    is_tab,
+                    renderer: renderer.clone(),
+                })
+            } else {
+                None
+            }
+        })
+    }
+}
+
 #[derive(Clone)]
 pub struct DisplaySnapshot {
     pub buffer_snapshot: MultiBufferSnapshot,
@@ -675,7 +770,7 @@ impl DisplaySnapshot {
                 suggestion: Some(editor_style.suggestions_style),
             },
         )
-        .map(|chunk| {
+        .flat_map(|chunk| {
             let mut highlight_style = chunk
                 .syntax_highlight_id
                 .and_then(|id| id.style(&editor_style.syntax));
@@ -718,6 +813,7 @@ impl DisplaySnapshot {
                 is_tab: chunk.is_tab,
                 renderer: chunk.renderer,
             }
+            .highlight_invisibles(editor_style)
         })
     }