Add testing suite for chunk bitmaps

crates/editor/src/display_map/custom_highlights.rs

@@ -144,6 +144,7 @@ impl<'a> Iterator for CustomHighlightsChunks<'a> {
 
         chunk.text = suffix;
         self.offset += prefix.len();
+        // FIXME: chunk cloning is wrong because the bitmaps might be off
         let mut prefix = Chunk {
             text: prefix,
             ..chunk.clone()
@@ -172,3 +173,124 @@ impl Ord for HighlightEndpoint {
             .then_with(|| other.is_start.cmp(&self.is_start))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::MultiBuffer;
+    use gpui::App;
+    use rand::prelude::*;
+    use util::RandomCharIter;
+
+    #[gpui::test(iterations = 100)]
+    fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) {
+        // Generate random buffer using existing test infrastructure
+        let len = rng.gen_range(0..10000);
+        let buffer = if rng.r#gen() {
+            let text = RandomCharIter::new(&mut rng).take(len).collect::<String>();
+            MultiBuffer::build_simple(&text, cx)
+        } else {
+            MultiBuffer::build_random(&mut rng, cx)
+        };
+
+        let buffer_snapshot = buffer.read(cx).snapshot(cx);
+
+        // Create random highlights
+        let mut highlights = TreeMap::default();
+        let highlight_count = rng.gen_range(1..10);
+
+        for _i in 0..highlight_count {
+            let style = HighlightStyle {
+                color: Some(gpui::Hsla {
+                    h: rng.r#gen::<f32>(),
+                    s: rng.r#gen::<f32>(),
+                    l: rng.r#gen::<f32>(),
+                    a: 1.0,
+                }),
+                ..Default::default()
+            };
+
+            let mut ranges = Vec::new();
+            let range_count = rng.gen_range(1..10);
+            for _ in 0..range_count {
+                let start = rng.gen_range(0..buffer_snapshot.len());
+                let end = rng.gen_range(start..buffer_snapshot.len().min(start + 100));
+                let start_anchor = buffer_snapshot.anchor_after(start);
+                let end_anchor = buffer_snapshot.anchor_after(end);
+                ranges.push(start_anchor..end_anchor);
+            }
+
+            let type_id = TypeId::of::<()>(); // Simple type ID for testing
+            highlights.insert(type_id, Arc::new((style, ranges)));
+        }
+
+        // Get all chunks and verify their bitmaps
+        let chunks = CustomHighlightsChunks::new(
+            0..buffer_snapshot.len(),
+            false,
+            Some(&highlights),
+            &buffer_snapshot,
+        );
+
+        for chunk in chunks {
+            let chunk_text = chunk.text;
+            let chars_bitmap = chunk.chars;
+            let tabs_bitmap = chunk.tabs;
+
+            // Check empty chunks have empty bitmaps
+            if chunk_text.is_empty() {
+                assert_eq!(
+                    chars_bitmap, 0,
+                    "Empty chunk should have empty chars bitmap"
+                );
+                assert_eq!(tabs_bitmap, 0, "Empty chunk should have empty tabs bitmap");
+                continue;
+            }
+
+            // Verify that chunk text doesn't exceed 128 bytes
+            assert!(
+                chunk_text.len() <= 128,
+                "Chunk text length {} exceeds 128 bytes",
+                chunk_text.len()
+            );
+
+            // Verify chars bitmap
+            let char_indices = chunk_text
+                .char_indices()
+                .map(|(i, _)| i)
+                .collect::<Vec<_>>();
+
+            for byte_idx in 0..chunk_text.len() {
+                let should_have_bit = char_indices.contains(&byte_idx);
+                let has_bit = chars_bitmap & (1 << byte_idx) != 0;
+
+                if has_bit != should_have_bit {
+                    eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                    eprintln!("Char indices: {:?}", char_indices);
+                    eprintln!("Chars bitmap: {:#b}", chars_bitmap);
+                    assert_eq!(
+                        has_bit, should_have_bit,
+                        "Chars bitmap mismatch at byte index {} in chunk {:?}. Expected bit: {}, Got bit: {}",
+                        byte_idx, chunk_text, should_have_bit, has_bit
+                    );
+                }
+            }
+
+            // Verify tabs bitmap
+            for (byte_idx, byte) in chunk_text.bytes().enumerate() {
+                let is_tab = byte == b'\t';
+                let has_bit = tabs_bitmap & (1 << byte_idx) != 0;
+
+                if has_bit != is_tab {
+                    eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                    eprintln!("Tabs bitmap: {:#b}", tabs_bitmap);
+                    assert_eq!(
+                        has_bit, is_tab,
+                        "Tabs bitmap mismatch at byte index {} in chunk {:?}. Byte: {:?}, Expected bit: {}, Got bit: {}",
+                        byte_idx, chunk_text, byte as char, is_tab, has_bit
+                    );
+                }
+            }
+        }
+    }
+}

crates/editor/src/display_map/fold_map.rs

@@ -2075,6 +2075,97 @@ mod tests {
         );
     }
 
+    #[gpui::test(iterations = 100)]
+    fn test_random_chunk_bitmaps(cx: &mut gpui::App, mut rng: StdRng) {
+        init_test(cx);
+
+        // Generate random buffer using existing test infrastructure
+        let text_len = rng.gen_range(0..10000);
+        let buffer = if rng.r#gen() {
+            let text = RandomCharIter::new(&mut rng)
+                .take(text_len)
+                .collect::<String>();
+            MultiBuffer::build_simple(&text, cx)
+        } else {
+            MultiBuffer::build_random(&mut rng, cx)
+        };
+        let buffer_snapshot = buffer.read(cx).snapshot(cx);
+        let (_, inlay_snapshot) = InlayMap::new(buffer_snapshot.clone());
+        let (mut fold_map, _) = FoldMap::new(inlay_snapshot.clone());
+
+        // Perform random mutations
+        let mutation_count = rng.gen_range(1..10);
+        for _ in 0..mutation_count {
+            fold_map.randomly_mutate(&mut rng);
+        }
+
+        let (snapshot, _) = fold_map.read(inlay_snapshot, vec![]);
+
+        // Get all chunks and verify their bitmaps
+        let chunks = snapshot.chunks(
+            FoldOffset(0)..FoldOffset(snapshot.len().0),
+            false,
+            Highlights::default(),
+        );
+
+        for chunk in chunks {
+            let chunk_text = chunk.text;
+            let chars_bitmap = chunk.chars;
+            let tabs_bitmap = chunk.tabs;
+
+            // Check empty chunks have empty bitmaps
+            if chunk_text.is_empty() {
+                assert_eq!(
+                    chars_bitmap, 0,
+                    "Empty chunk should have empty chars bitmap"
+                );
+                assert_eq!(tabs_bitmap, 0, "Empty chunk should have empty tabs bitmap");
+                continue;
+            }
+
+            // Verify that chunk text doesn't exceed 128 bytes
+            assert!(
+                chunk_text.len() <= 128,
+                "Chunk text length {} exceeds 128 bytes",
+                chunk_text.len()
+            );
+
+            // Verify chars bitmap
+            let char_indices = chunk_text
+                .char_indices()
+                .map(|(i, _)| i)
+                .collect::<Vec<_>>();
+
+            for byte_idx in 0..chunk_text.len() {
+                let should_have_bit = char_indices.contains(&byte_idx);
+                let has_bit = chars_bitmap & (1 << byte_idx) != 0;
+
+                if has_bit != should_have_bit {
+                    eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                    eprintln!("Char indices: {:?}", char_indices);
+                    eprintln!("Chars bitmap: {:#b}", chars_bitmap);
+                    assert_eq!(
+                        has_bit, should_have_bit,
+                        "Chars bitmap mismatch at byte index {} in chunk {:?}. Expected bit: {}, Got bit: {}",
+                        byte_idx, chunk_text, should_have_bit, has_bit
+                    );
+                }
+            }
+
+            // Verify tabs bitmap
+            for (byte_idx, byte) in chunk_text.bytes().enumerate() {
+                let is_tab = byte == b'\t';
+                let has_bit = tabs_bitmap & (1 << byte_idx) != 0;
+
+                assert_eq!(
+                    has_bit, is_tab,
+                    "Tabs bitmap mismatch at byte index {} in chunk {:?}. Byte: {:?}, Expected bit: {}, Got bit: {}",
+                    byte_idx, chunk_text, byte as char, is_tab, has_bit
+                );
+            }
+        }
+    }
+
     fn init_test(cx: &mut gpui::App) {
         let store = SettingsStore::test(cx);
         cx.set_global(store);

crates/editor/src/display_map/inlay_map.rs

@@ -268,6 +268,7 @@ impl<'a> Iterator for InlayChunks<'a> {
 
                 chunk.text = suffix;
                 self.output_offset.0 += prefix.len();
+                // FIXME: chunk cloning is wrong because the bitmaps might be off
                 Chunk {
                     text: prefix,
                     ..chunk.clone()
@@ -1080,7 +1081,7 @@ mod tests {
     use super::*;
     use crate::{
         InlayId, MultiBuffer,
-        display_map::{InlayHighlights, TextHighlights},
+        display_map::{Highlights, InlayHighlights, TextHighlights},
         hover_links::InlayHighlight,
     };
     use gpui::{App, HighlightStyle};
@@ -1090,6 +1091,7 @@ mod tests {
     use std::{any::TypeId, cmp::Reverse, env, sync::Arc};
     use sum_tree::TreeMap;
     use text::Patch;
+    use util::RandomCharIter;
     use util::post_inc;
 
     #[test]
@@ -1809,6 +1811,102 @@ mod tests {
         }
     }
 
+    #[gpui::test(iterations = 100)]
+    fn test_random_chunk_bitmaps(cx: &mut gpui::App, mut rng: StdRng) {
+        init_test(cx);
+
+        // Generate random buffer using existing test infrastructure
+        let text_len = rng.gen_range(0..10000);
+        let buffer = if rng.r#gen() {
+            let text = RandomCharIter::new(&mut rng)
+                .take(text_len)
+                .collect::<String>();
+            MultiBuffer::build_simple(&text, cx)
+        } else {
+            MultiBuffer::build_random(&mut rng, cx)
+        };
+
+        let buffer_snapshot = buffer.read(cx).snapshot(cx);
+        let (mut inlay_map, _) = InlayMap::new(buffer_snapshot.clone());
+
+        // Perform random mutations to add inlays
+        let mut next_inlay_id = 0;
+        let mutation_count = rng.gen_range(1..10);
+        for _ in 0..mutation_count {
+            inlay_map.randomly_mutate(&mut next_inlay_id, &mut rng);
+        }
+
+        let (snapshot, _) = inlay_map.sync(buffer_snapshot, vec![]);
+
+        // Get all chunks and verify their bitmaps
+        let chunks = snapshot.chunks(
+            InlayOffset(0)..InlayOffset(snapshot.len().0),
+            false,
+            Highlights::default(),
+        );
+
+        for chunk in chunks {
+            let chunk_text = chunk.text;
+            let chars_bitmap = chunk.chars;
+            let tabs_bitmap = chunk.tabs;
+
+            // Check empty chunks have empty bitmaps
+            if chunk_text.is_empty() {
+                assert_eq!(
+                    chars_bitmap, 0,
+                    "Empty chunk should have empty chars bitmap"
+                );
+                assert_eq!(tabs_bitmap, 0, "Empty chunk should have empty tabs bitmap");
+                continue;
+            }
+
+            // Verify that chunk text doesn't exceed 128 bytes
+            assert!(
+                chunk_text.len() <= 128,
+                "Chunk text length {} exceeds 128 bytes",
+                chunk_text.len()
+            );
+
+            // Verify chars bitmap
+            let char_indices = chunk_text
+                .char_indices()
+                .map(|(i, _)| i)
+                .collect::<Vec<_>>();
+
+            for byte_idx in 0..chunk_text.len() {
+                let should_have_bit = char_indices.contains(&byte_idx);
+                let has_bit = chars_bitmap & (1 << byte_idx) != 0;
+
+                if has_bit != should_have_bit {
+                    eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                    eprintln!("Char indices: {:?}", char_indices);
+                    eprintln!("Chars bitmap: {:#b}", chars_bitmap);
+                    assert_eq!(
+                        has_bit, should_have_bit,
+                        "Chars bitmap mismatch at byte index {} in chunk {:?}. Expected bit: {}, Got bit: {}",
+                        byte_idx, chunk_text, should_have_bit, has_bit
+                    );
+                }
+            }
+
+            // Verify tabs bitmap
+            for (byte_idx, byte) in chunk_text.bytes().enumerate() {
+                let is_tab = byte == b'\t';
+                let has_bit = tabs_bitmap & (1 << byte_idx) != 0;
+
+                if has_bit != is_tab {
+                    eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                    eprintln!("Tabs bitmap: {:#b}", tabs_bitmap);
+                    assert_eq!(
+                        has_bit, is_tab,
+                        "Tabs bitmap mismatch at byte index {} in chunk {:?}. Byte: {:?}, Expected bit: {}, Got bit: {}",
+                        byte_idx, chunk_text, byte as char, is_tab, has_bit
+                    );
+                }
+            }
+        }
+    }
+
     fn init_test(cx: &mut App) {
         let store = SettingsStore::test(cx);
         cx.set_global(store);

crates/editor/src/display_map/tab_map.rs

@@ -729,6 +729,7 @@ mod tests {
         // assert_eq!(tab_snapshot.expand_tabs("\t".chars(), 0), 0);
         // assert_eq!(tab_snapshot.expand_tabs("\t".chars(), 1), 4);
         // assert_eq!(tab_snapshot.expand_tabs("\ta".chars(), 2), 5);
+        // FIXME: the test
         panic!("Fix this test")
     }
 

crates/language/src/buffer_tests.rs

@@ -3764,3 +3764,80 @@ fn init_settings(cx: &mut App, f: fn(&mut AllLanguageSettingsContent)) {
         settings.update_user_settings::<AllLanguageSettings>(cx, f);
     });
 }
+
+#[gpui::test(iterations = 100)]
+fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) {
+    use util::RandomCharIter;
+
+    // Generate random text
+    let len = rng.gen_range(0..10000);
+    let text = RandomCharIter::new(&mut rng).take(len).collect::<String>();
+
+    let buffer = cx.new(|cx| Buffer::local(text, cx));
+    let snapshot = buffer.read(cx).snapshot();
+
+    // Get all chunks and verify their bitmaps
+    let chunks = snapshot.chunks(0..snapshot.len(), false);
+
+    for chunk in chunks {
+        let chunk_text = chunk.text;
+        let chars_bitmap = chunk.chars;
+        let tabs_bitmap = chunk.tabs;
+
+        // Check empty chunks have empty bitmaps
+        if chunk_text.is_empty() {
+            assert_eq!(
+                chars_bitmap, 0,
+                "Empty chunk should have empty chars bitmap"
+            );
+            assert_eq!(tabs_bitmap, 0, "Empty chunk should have empty tabs bitmap");
+            continue;
+        }
+
+        // Verify that chunk text doesn't exceed 128 bytes
+        assert!(
+            chunk_text.len() <= 128,
+            "Chunk text length {} exceeds 128 bytes",
+            chunk_text.len()
+        );
+
+        // Verify chars bitmap
+        let char_indices = chunk_text
+            .char_indices()
+            .map(|(i, _)| i)
+            .collect::<Vec<_>>();
+
+        for byte_idx in 0..chunk_text.len() {
+            let should_have_bit = char_indices.contains(&byte_idx);
+            let has_bit = chars_bitmap & (1 << byte_idx) != 0;
+
+            if has_bit != should_have_bit {
+                eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                eprintln!("Char indices: {:?}", char_indices);
+                eprintln!("Chars bitmap: {:#b}", chars_bitmap);
+            }
+
+            assert_eq!(
+                has_bit, should_have_bit,
+                "Chars bitmap mismatch at byte index {} in chunk {:?}. Expected bit: {}, Got bit: {}",
+                byte_idx, chunk_text, should_have_bit, has_bit
+            );
+        }
+
+        // Verify tabs bitmap
+        for (byte_idx, byte) in chunk_text.bytes().enumerate() {
+            let is_tab = byte == b'\t';
+            let has_bit = tabs_bitmap & (1 << byte_idx) != 0;
+
+            if has_bit != is_tab {
+                eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                eprintln!("Tabs bitmap: {:#b}", tabs_bitmap);
+                assert_eq!(
+                    has_bit, is_tab,
+                    "Tabs bitmap mismatch at byte index {} in chunk {:?}. Byte: {:?}, Expected bit: {}, Got bit: {}",
+                    byte_idx, chunk_text, byte as char, is_tab, has_bit
+                );
+            }
+        }
+    }
+}

crates/multi_buffer/src/multi_buffer.rs

@@ -7726,6 +7726,8 @@ impl<'a> Iterator for MultiBufferChunks<'a> {
                         chunk.text.split_at(diff_transform_end - self.range.start);
                     self.range.start = diff_transform_end;
                     chunk.text = after;
+                    // FIXME: We should be handling bitmap for tabs and chars here
+                    // Because we do a split at operation the bitmaps will be off
                     Some(Chunk {
                         text: before,
                         ..chunk.clone()

crates/multi_buffer/src/multi_buffer_tests.rs

@@ -7,6 +7,7 @@ use parking_lot::RwLock;
 use rand::prelude::*;
 use settings::SettingsStore;
 use std::env;
+use util::RandomCharIter;
 use util::test::sample_text;
 
 #[ctor::ctor]
@@ -3717,3 +3718,83 @@ fn test_new_empty_buffers_title_can_be_set(cx: &mut App) {
     });
     assert_eq!(multibuffer.read(cx).title(cx), "Hey");
 }
+
+#[gpui::test(iterations = 100)]
+fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) {
+    // Generate random multibuffer using existing test infrastructure
+    let multibuffer = if rng.r#gen() {
+        let len = rng.gen_range(0..10000);
+        let text = RandomCharIter::new(&mut rng).take(len).collect::<String>();
+        let buffer = cx.new(|cx| Buffer::local(text, cx));
+        cx.new(|cx| MultiBuffer::singleton(buffer, cx))
+    } else {
+        MultiBuffer::build_random(&mut rng, cx)
+    };
+
+    let snapshot = multibuffer.read(cx).snapshot(cx);
+
+    // Get all chunks and verify their bitmaps
+    let chunks = snapshot.chunks(0..snapshot.len(), false);
+
+    for chunk in chunks {
+        let chunk_text = chunk.text;
+        let chars_bitmap = chunk.chars;
+        let tabs_bitmap = chunk.tabs;
+
+        // Check empty chunks have empty bitmaps
+        if chunk_text.is_empty() {
+            assert_eq!(
+                chars_bitmap, 0,
+                "Empty chunk should have empty chars bitmap"
+            );
+            assert_eq!(tabs_bitmap, 0, "Empty chunk should have empty tabs bitmap");
+            continue;
+        }
+
+        // Verify that chunk text doesn't exceed 128 bytes
+        assert!(
+            chunk_text.len() <= 128,
+            "Chunk text length {} exceeds 128 bytes",
+            chunk_text.len()
+        );
+
+        // Verify chars bitmap
+        let char_indices = chunk_text
+            .char_indices()
+            .map(|(i, _)| i)
+            .collect::<Vec<_>>();
+
+        for byte_idx in 0..chunk_text.len() {
+            let should_have_bit = char_indices.contains(&byte_idx);
+            let has_bit = chars_bitmap & (1 << byte_idx) != 0;
+
+            if has_bit != should_have_bit {
+                eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                eprintln!("Char indices: {:?}", char_indices);
+                eprintln!("Chars bitmap: {:#b}", chars_bitmap);
+            }
+
+            assert_eq!(
+                has_bit, should_have_bit,
+                "Chars bitmap mismatch at byte index {} in chunk {:?}. Expected bit: {}, Got bit: {}",
+                byte_idx, chunk_text, should_have_bit, has_bit
+            );
+        }
+
+        // Verify tabs bitmap
+        for (byte_idx, byte) in chunk_text.bytes().enumerate() {
+            let is_tab = byte == b'\t';
+            let has_bit = tabs_bitmap & (1 << byte_idx) != 0;
+
+            if has_bit != is_tab {
+                eprintln!("Chunk text bytes: {:?}", chunk_text.as_bytes());
+                eprintln!("Tabs bitmap: {:#b}", tabs_bitmap);
+                assert_eq!(
+                    has_bit, is_tab,
+                    "Tabs bitmap mismatch at byte index {} in chunk {:?}. Byte: {:?}, Expected bit: {}, Got bit: {}",
+                    byte_idx, chunk_text, byte as char, is_tab, has_bit
+                );
+            }
+        }
+    }
+}