Render emojis

Co-Authored-By: Nathan Sobo <nathan@zed.dev>

crates/gpui/src/platform.rs

@@ -129,6 +129,12 @@ pub enum CursorStyle {
     PointingHand,
 }
 
+#[derive(Copy, Clone, Debug)]
+pub enum RasterizationOptions {
+    Alpha,
+    Bgra,
+}
+
 pub trait FontSystem: Send + Sync {
     fn add_fonts(&self, fonts: &[Arc<Vec<u8>>]) -> anyhow::Result<()>;
     fn load_family(&self, name: &str) -> anyhow::Result<Vec<FontId>>;
@@ -148,6 +154,7 @@ pub trait FontSystem: Send + Sync {
         glyph_id: GlyphId,
         subpixel_shift: Vector2F,
         scale_factor: f32,
+        options: RasterizationOptions,
     ) -> Option<(RectI, Vec<u8>)>;
     fn layout_line(&self, text: &str, font_size: f32, runs: &[(usize, RunStyle)]) -> LineLayout;
     fn wrap_line(&self, text: &str, font_id: FontId, font_size: f32, width: f32) -> Vec<usize>;

crates/gpui/src/platform/mac/fonts.rs

@@ -3,9 +3,9 @@ use crate::{
     geometry::{
         rect::{RectF, RectI},
         transform2d::Transform2F,
-        vector::{vec2f, vec2i, Vector2F},
+        vector::{vec2f, Vector2F},
     },
-    platform,
+    platform::{self, RasterizationOptions},
     text_layout::{Glyph, LineLayout, Run, RunStyle},
 };
 use cocoa::appkit::{CGFloat, CGPoint};
@@ -16,12 +16,13 @@ use core_foundation::{
     string::CFString,
 };
 use core_graphics::{
-    base::CGGlyph, color_space::CGColorSpace, context::CGContext, geometry::CGAffineTransform,
+    base::{kCGImageAlphaPremultipliedLast, CGGlyph},
+    color_space::CGColorSpace,
+    context::CGContext,
 };
 use core_text::{font::CTFont, line::CTLine, string_attributes::kCTFontAttributeName};
 use font_kit::{
-    canvas::RasterizationOptions, handle::Handle, hinting::HintingOptions, source::SystemSource,
+    handle::Handle, hinting::HintingOptions, source::SystemSource, sources::mem::MemSource,
-    sources::mem::MemSource,
 };
 use parking_lot::RwLock;
 use std::{cell::RefCell, char, cmp, convert::TryFrom, ffi::c_void, sync::Arc};
@@ -35,15 +36,19 @@ struct FontSystemState {
     memory_source: MemSource,
     system_source: SystemSource,
     fonts: Vec<font_kit::font::Font>,
+    emoji_font_id: FontId,
 }
 
 impl FontSystem {
     pub fn new() -> Self {
-        Self(RwLock::new(FontSystemState {
+        let mut state = FontSystemState {
             memory_source: MemSource::empty(),
             system_source: SystemSource::new(),
             fonts: Vec::new(),
-        }))
+            emoji_font_id: FontId(0), // This will be the first font that we load.
+        };
+        state.load_family("Apple Color Emoji").unwrap();
+        Self(RwLock::new(state))
     }
 }
 
@@ -83,10 +88,16 @@ impl platform::FontSystem for FontSystem {
         glyph_id: GlyphId,
         subpixel_shift: Vector2F,
         scale_factor: f32,
+        options: RasterizationOptions,
     ) -> Option<(RectI, Vec<u8>)> {
-        self.0
+        self.0.read().rasterize_glyph(
-            .read()
+            font_id,
-            .rasterize_glyph(font_id, font_size, glyph_id, subpixel_shift, scale_factor)
+            font_size,
+            glyph_id,
+            subpixel_shift,
+            scale_factor,
+            options,
+        )
     }
 
     fn layout_line(&self, text: &str, font_size: f32, runs: &[(usize, RunStyle)]) -> LineLayout {
@@ -168,6 +179,7 @@ impl FontSystemState {
         glyph_id: GlyphId,
         subpixel_shift: Vector2F,
         scale_factor: f32,
+        options: RasterizationOptions,
     ) -> Option<(RectI, Vec<u8>)> {
         let font = &self.fonts[font_id.0];
         let scale = Transform2F::from_scale(scale_factor);
@@ -177,7 +189,7 @@ impl FontSystemState {
                 font_size,
                 scale,
                 HintingOptions::None,
-                RasterizationOptions::GrayscaleAa,
+                font_kit::canvas::RasterizationOptions::GrayscaleAa,
             )
             .ok()?;
 
@@ -185,18 +197,41 @@ impl FontSystemState {
             None
         } else {
             // Make room for subpixel variants.
-            let cx_bounds = RectI::new(glyph_bounds.origin(), glyph_bounds.size() + vec2i(1, 1));
+            let subpixel_padding = subpixel_shift.ceil().to_i32();
-            let mut bytes = vec![0; cx_bounds.width() as usize * cx_bounds.height() as usize];
+            let cx_bounds = RectI::new(
-            let cx = CGContext::create_bitmap_context(
+                glyph_bounds.origin(),
-                Some(bytes.as_mut_ptr() as *mut _),
+                glyph_bounds.size() + subpixel_padding,
-                cx_bounds.width() as usize,
-                cx_bounds.height() as usize,
-                8,
-                cx_bounds.width() as usize,
-                &CGColorSpace::create_device_gray(),
-                kCGImageAlphaOnly,
             );
 
+            let mut bytes;
+            let cx;
+            match options {
+                RasterizationOptions::Alpha => {
+                    bytes = vec![0; cx_bounds.width() as usize * cx_bounds.height() as usize];
+                    cx = CGContext::create_bitmap_context(
+                        Some(bytes.as_mut_ptr() as *mut _),
+                        cx_bounds.width() as usize,
+                        cx_bounds.height() as usize,
+                        8,
+                        cx_bounds.width() as usize,
+                        &CGColorSpace::create_device_gray(),
+                        kCGImageAlphaOnly,
+                    );
+                }
+                RasterizationOptions::Bgra => {
+                    bytes = vec![0; cx_bounds.width() as usize * 4 * cx_bounds.height() as usize];
+                    cx = CGContext::create_bitmap_context(
+                        Some(bytes.as_mut_ptr() as *mut _),
+                        cx_bounds.width() as usize,
+                        cx_bounds.height() as usize,
+                        8,
+                        cx_bounds.width() as usize * 4,
+                        &CGColorSpace::create_device_rgb(),
+                        kCGImageAlphaPremultipliedLast,
+                    );
+                }
+            }
+
             // Move the origin to bottom left and account for scaling, this
             // makes drawing text consistent with the font-kit's raster_bounds.
             cx.translate(
@@ -220,6 +255,17 @@ impl FontSystemState {
                     cx,
                 );
 
+            if let RasterizationOptions::Bgra = options {
+                // Convert from RGBA with premultiplied alpha to BGRA with straight alpha.
+                for pixel in bytes.chunks_exact_mut(4) {
+                    pixel.swap(0, 2);
+                    let a = pixel[3] as f32 / 255.;
+                    pixel[0] = (pixel[0] as f32 / a) as u8;
+                    pixel[1] = (pixel[1] as f32 / a) as u8;
+                    pixel[2] = (pixel[2] as f32 / a) as u8;
+                }
+            }
+
             Some((cx_bounds, bytes))
         }
     }
@@ -316,6 +362,7 @@ impl FontSystemState {
                     id: *glyph_id as GlyphId,
                     position: vec2f(position.x as f32, position.y as f32),
                     index: ix_converter.utf8_ix,
+                    is_emoji: font_id == self.emoji_font_id,
                 });
             }
 
@@ -520,7 +567,14 @@ mod tests {
         for i in 0..VARIANTS {
             let variant = i as f32 / VARIANTS as f32;
             let (bounds, bytes) = fonts
-                .rasterize_glyph(font_id, 16.0, glyph_id, vec2f(variant, variant), 2.)
+                .rasterize_glyph(
+                    font_id,
+                    16.0,
+                    glyph_id,
+                    vec2f(variant, variant),
+                    2.,
+                    RasterizationOptions::Alpha,
+                )
                 .unwrap();
 
             let name = format!("/Users/as-cii/Desktop/twog-{}.png", i);

crates/gpui/src/platform/mac/image_cache.rs

@@ -1,21 +1,39 @@
-use anyhow::anyhow;
-use metal::{MTLPixelFormat, TextureDescriptor, TextureRef};
-
 use super::atlas::{AllocId, AtlasAllocator};
 use crate::{
+    fonts::{FontId, GlyphId},
     geometry::{rect::RectI, vector::Vector2I},
-    ImageData,
+    platform::RasterizationOptions,
+    scene::ImageGlyph,
+    FontSystem, ImageData,
 };
-use std::{collections::HashMap, mem};
+use anyhow::anyhow;
+use metal::{MTLPixelFormat, TextureDescriptor, TextureRef};
+use ordered_float::OrderedFloat;
+use std::{collections::HashMap, mem, sync::Arc};
+
+#[derive(Hash, Eq, PartialEq)]
+struct GlyphDescriptor {
+    font_id: FontId,
+    font_size: OrderedFloat<f32>,
+    glyph_id: GlyphId,
+}
 
 pub struct ImageCache {
     prev_frame: HashMap<usize, (AllocId, RectI)>,
     curr_frame: HashMap<usize, (AllocId, RectI)>,
+    image_glyphs: HashMap<GlyphDescriptor, Option<(AllocId, RectI, Vector2I)>>,
     atlases: AtlasAllocator,
+    scale_factor: f32,
+    fonts: Arc<dyn FontSystem>,
 }
 
 impl ImageCache {
-    pub fn new(device: metal::Device, size: Vector2I) -> Self {
+    pub fn new(
+        device: metal::Device,
+        size: Vector2I,
+        scale_factor: f32,
+        fonts: Arc<dyn FontSystem>,
+    ) -> Self {
         let descriptor = TextureDescriptor::new();
         descriptor.set_pixel_format(MTLPixelFormat::BGRA8Unorm);
         descriptor.set_width(size.x() as u64);
@@ -23,7 +41,21 @@ impl ImageCache {
         Self {
             prev_frame: Default::default(),
             curr_frame: Default::default(),
+            image_glyphs: Default::default(),
             atlases: AtlasAllocator::new(device, descriptor),
+            scale_factor,
+            fonts,
+        }
+    }
+
+    pub fn set_scale_factor(&mut self, scale_factor: f32) {
+        if scale_factor != self.scale_factor {
+            self.scale_factor = scale_factor;
+            for (_, glyph) in self.image_glyphs.drain() {
+                if let Some((alloc_id, _, _)) = glyph {
+                    self.atlases.deallocate(alloc_id);
+                }
+            }
         }
     }
 
@@ -39,6 +71,37 @@ impl ImageCache {
         (alloc_id, atlas_bounds)
     }
 
+    pub fn render_glyph(&mut self, image_glyph: &ImageGlyph) -> Option<(AllocId, RectI, Vector2I)> {
+        *self
+            .image_glyphs
+            .entry(GlyphDescriptor {
+                font_id: image_glyph.font_id,
+                font_size: OrderedFloat(image_glyph.font_size),
+                glyph_id: image_glyph.id,
+            })
+            .or_insert_with(|| {
+                let (glyph_bounds, bytes) = self.fonts.rasterize_glyph(
+                    image_glyph.font_id,
+                    image_glyph.font_size,
+                    image_glyph.id,
+                    Default::default(),
+                    self.scale_factor,
+                    RasterizationOptions::Bgra,
+                )?;
+                let (alloc_id, atlas_bounds) = self
+                    .atlases
+                    .upload(glyph_bounds.size(), &bytes)
+                    .ok_or_else(|| {
+                        anyhow!(
+                            "could not upload image glyph of size {:?}",
+                            glyph_bounds.size()
+                        )
+                    })
+                    .unwrap();
+                Some((alloc_id, atlas_bounds, glyph_bounds.origin()))
+            })
+    }
+
     pub fn finish_frame(&mut self) {
         mem::swap(&mut self.prev_frame, &mut self.curr_frame);
         for (_, (id, _)) in self.curr_frame.drain() {

crates/gpui/src/platform/mac/renderer.rs

@@ -6,7 +6,7 @@ use crate::{
         vector::{vec2f, vec2i, Vector2F},
     },
     platform,
-    scene::{Glyph, Icon, Image, Layer, Quad, Scene, Shadow, Underline},
+    scene::{Glyph, Icon, Image, ImageGlyph, Layer, Quad, Scene, Shadow, Underline},
 };
 use cocoa::foundation::NSUInteger;
 use log::warn;
@@ -67,8 +67,13 @@ impl Renderer {
             MTLResourceOptions::StorageModeManaged,
         );
 
-        let sprite_cache = SpriteCache::new(device.clone(), vec2i(1024, 768), scale_factor, fonts);
+        let sprite_cache = SpriteCache::new(
-        let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768));
+            device.clone(),
+            vec2i(1024, 768),
+            scale_factor,
+            fonts.clone(),
+        );
+        let image_cache = ImageCache::new(device.clone(), vec2i(1024, 768), scale_factor, fonts);
         let path_atlases =
             AtlasAllocator::new(device.clone(), build_path_atlas_texture_descriptor());
         let quad_pipeline_state = build_pipeline_state(
@@ -141,6 +146,9 @@ impl Renderer {
         command_buffer: &metal::CommandBufferRef,
         output: &metal::TextureRef,
     ) {
+        self.sprite_cache.set_scale_factor(scene.scale_factor());
+        self.image_cache.set_scale_factor(scene.scale_factor());
+
         let mut offset = 0;
 
         let path_sprites = self.render_path_atlases(scene, &mut offset, command_buffer);
@@ -359,6 +367,7 @@ impl Renderer {
             );
             self.render_images(
                 layer.images(),
+                layer.image_glyphs(),
                 scale_factor,
                 offset,
                 drawable_size,
@@ -541,8 +550,6 @@ impl Renderer {
             return;
         }
 
-        self.sprite_cache.set_scale_factor(scale_factor);
-
         let mut sprites_by_atlas = HashMap::new();
 
         for glyph in glyphs {
@@ -653,12 +660,13 @@ impl Renderer {
     fn render_images(
         &mut self,
         images: &[Image],
+        image_glyphs: &[ImageGlyph],
         scale_factor: f32,
         offset: &mut usize,
         drawable_size: Vector2F,
         command_encoder: &metal::RenderCommandEncoderRef,
     ) {
-        if images.is_empty() {
+        if images.is_empty() && image_glyphs.is_empty() {
             return;
         }
 
@@ -686,6 +694,31 @@ impl Renderer {
                 });
         }
 
+        for image_glyph in image_glyphs {
+            let origin = (image_glyph.origin * scale_factor).floor();
+            if let Some((alloc_id, atlas_bounds, glyph_origin)) =
+                self.image_cache.render_glyph(image_glyph)
+            {
+                images_by_atlas
+                    .entry(alloc_id.atlas_id)
+                    .or_insert_with(Vec::new)
+                    .push(shaders::GPUIImage {
+                        origin: (origin + glyph_origin.to_f32()).to_float2(),
+                        target_size: atlas_bounds.size().to_float2(),
+                        source_size: atlas_bounds.size().to_float2(),
+                        atlas_origin: atlas_bounds.origin().to_float2(),
+                        border_top: 0.,
+                        border_right: 0.,
+                        border_bottom: 0.,
+                        border_left: 0.,
+                        border_color: Default::default(),
+                        corner_radius: 0.,
+                    });
+            } else {
+                log::warn!("could not render glyph with id {}", image_glyph.id);
+            }
+        }
+
         command_encoder.set_render_pipeline_state(&self.image_pipeline_state);
         command_encoder.set_vertex_buffer(
             shaders::GPUIImageVertexInputIndex_GPUIImageVertexInputIndexVertices as u64,

crates/gpui/src/platform/mac/sprite_cache.rs

@@ -2,7 +2,7 @@ use super::atlas::AtlasAllocator;
 use crate::{
     fonts::{FontId, GlyphId},
     geometry::vector::{vec2f, Vector2F, Vector2I},
-    platform,
+    platform::{self, RasterizationOptions},
 };
 use collections::hash_map::Entry;
 use metal::{MTLPixelFormat, TextureDescriptor};
@@ -113,6 +113,7 @@ impl SpriteCache {
                     glyph_id,
                     subpixel_shift,
                     scale_factor,
+                    RasterizationOptions::Alpha,
                 )?;
 
                 let (alloc_id, atlas_bounds) = atlases

crates/gpui/src/scene.rs

@@ -29,6 +29,7 @@ pub struct Layer {
     images: Vec<Image>,
     shadows: Vec<Shadow>,
     glyphs: Vec<Glyph>,
+    image_glyphs: Vec<ImageGlyph>,
     icons: Vec<Icon>,
     paths: Vec<Path>,
 }
@@ -58,6 +59,14 @@ pub struct Glyph {
     pub color: Color,
 }
 
+#[derive(Debug)]
+pub struct ImageGlyph {
+    pub font_id: FontId,
+    pub font_size: f32,
+    pub id: GlyphId,
+    pub origin: Vector2F,
+}
+
 pub struct Icon {
     pub bounds: RectF,
     pub svg: usvg::Tree,
@@ -204,6 +213,10 @@ impl Scene {
         self.active_layer().push_glyph(glyph)
     }
 
+    pub fn push_image_glyph(&mut self, image_glyph: ImageGlyph) {
+        self.active_layer().push_image_glyph(image_glyph)
+    }
+
     pub fn push_icon(&mut self, icon: Icon) {
         self.active_layer().push_icon(icon)
     }
@@ -264,13 +277,14 @@ impl Layer {
     pub fn new(clip_bounds: Option<RectF>) -> Self {
         Self {
             clip_bounds,
-            quads: Vec::new(),
+            quads: Default::default(),
-            underlines: Vec::new(),
+            underlines: Default::default(),
-            images: Vec::new(),
+            images: Default::default(),
-            shadows: Vec::new(),
+            shadows: Default::default(),
-            glyphs: Vec::new(),
+            image_glyphs: Default::default(),
-            icons: Vec::new(),
+            glyphs: Default::default(),
-            paths: Vec::new(),
+            icons: Default::default(),
+            paths: Default::default(),
         }
     }
 
@@ -318,6 +332,14 @@ impl Layer {
         self.shadows.as_slice()
     }
 
+    fn push_image_glyph(&mut self, glyph: ImageGlyph) {
+        self.image_glyphs.push(glyph);
+    }
+
+    pub fn image_glyphs(&self) -> &[ImageGlyph] {
+        self.image_glyphs.as_slice()
+    }
+
     fn push_glyph(&mut self, glyph: Glyph) {
         self.glyphs.push(glyph);
     }

crates/gpui/src/text_layout.rs

@@ -191,6 +191,7 @@ pub struct Glyph {
     pub id: GlyphId,
     pub position: Vector2F,
     pub index: usize,
+    pub is_emoji: bool,
 }
 
 impl Line {
@@ -323,13 +324,22 @@ impl Line {
                     });
                 }
 
-                cx.scene.push_glyph(scene::Glyph {
+                if glyph.is_emoji {
-                    font_id: run.font_id,
+                    cx.scene.push_image_glyph(scene::ImageGlyph {
-                    font_size: self.layout.font_size,
+                        font_id: run.font_id,
-                    id: glyph.id,
+                        font_size: self.layout.font_size,
-                    origin: glyph_origin,
+                        id: glyph.id,
-                    color,
+                        origin: glyph_origin,
-                });
+                    });
+                } else {
+                    cx.scene.push_glyph(scene::Glyph {
+                        font_id: run.font_id,
+                        font_size: self.layout.font_size,
+                        id: glyph.id,
+                        origin: glyph_origin,
+                        color,
+                    });
+                }
             }
         }
 
@@ -389,13 +399,22 @@ impl Line {
                         .bounding_box(run.font_id, self.layout.font_size),
                 );
                 if glyph_bounds.intersects(visible_bounds) {
-                    cx.scene.push_glyph(scene::Glyph {
+                    if glyph.is_emoji {
-                        font_id: run.font_id,
+                        cx.scene.push_image_glyph(scene::ImageGlyph {
-                        font_size: self.layout.font_size,
+                            font_id: run.font_id,
-                        id: glyph.id,
+                            font_size: self.layout.font_size,
-                        origin: glyph_bounds.origin() + baseline_origin,
+                            id: glyph.id,
-                        color,
+                            origin: glyph_bounds.origin() + baseline_origin,
-                    });
+                        });
+                    } else {
+                        cx.scene.push_glyph(scene::Glyph {
+                            font_id: run.font_id,
+                            font_size: self.layout.font_size,
+                            id: glyph.id,
+                            origin: glyph_bounds.origin() + baseline_origin,
+                            color,
+                        });
+                    }
                 }
             }
         }