Implement rendering of images with data urls in markdown (#30322)

Fixes #28266

![Screenshot 2025-05-08 at 5 08
21 PM](https://github.com/user-attachments/assets/774d2dde-3f2d-466c-8eb1-c67badbd89e4)

Release Notes:

- Added support for rendering images with data URLs in markdown. This
can show up in hover documentation provided by language servers.

Co-authored-by: Mikayla Maki <mikayla.c.maki@gmail.com>

Cargo.lock

@@ -8517,6 +8517,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "assets",
+ "base64 0.22.1",
  "env_logger 0.11.8",
  "gpui",
  "language",

crates/agent/src/context.rs

@@ -754,11 +754,11 @@ pub enum ImageStatus {
 
 impl ImageContext {
     pub fn eq_for_key(&self, other: &Self) -> bool {
-        self.original_image.id == other.original_image.id
+        self.original_image.id() == other.original_image.id()
     }
 
     pub fn hash_for_key<H: Hasher>(&self, state: &mut H) {
-        self.original_image.id.hash(state);
+        self.original_image.id().hash(state);
     }
 
     pub fn image(&self) -> Option<LanguageModelImage> {

crates/gpui/src/platform.rs

@@ -36,7 +36,7 @@ use crate::{
     ForegroundExecutor, GlyphId, GpuSpecs, ImageSource, Keymap, LineLayout, Pixels, PlatformInput,
     Point, RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, ScaledPixels, Scene,
     ShapedGlyph, ShapedRun, SharedString, Size, SvgRenderer, SvgSize, Task, TaskLabel, Window,
-    point, px, size,
+    hash, point, px, size,
 };
 use anyhow::Result;
 use async_task::Runnable;
@@ -1499,6 +1499,20 @@ impl ImageFormat {
             ImageFormat::Tiff => "image/tiff",
         }
     }
+
+    /// Returns the ImageFormat for the given mime type
+    pub fn from_mime_type(mime_type: &str) -> Option<Self> {
+        match mime_type {
+            "image/png" => Some(Self::Png),
+            "image/jpeg" | "image/jpg" => Some(Self::Jpeg),
+            "image/webp" => Some(Self::Webp),
+            "image/gif" => Some(Self::Gif),
+            "image/svg+xml" => Some(Self::Svg),
+            "image/bmp" => Some(Self::Bmp),
+            "image/tiff" | "image/tif" => Some(Self::Tiff),
+            _ => None,
+        }
+    }
 }
 
 /// An image, with a format and certain bytes
@@ -1509,7 +1523,7 @@ pub struct Image {
     /// The raw image bytes
     pub bytes: Vec<u8>,
     /// The unique ID for the image
-    pub id: u64,
+    id: u64,
 }
 
 impl Hash for Image {
@@ -1521,10 +1535,15 @@ impl Hash for Image {
 impl Image {
     /// An empty image containing no data
     pub fn empty() -> Self {
+        Self::from_bytes(ImageFormat::Png, Vec::new())
+    }
+
+    /// Create an image from a format and bytes
+    pub fn from_bytes(format: ImageFormat, bytes: Vec<u8>) -> Self {
         Self {
-            format: ImageFormat::Png,
+            id: hash(&bytes),
-            bytes: Vec::new(),
+            format,
-            id: 0,
+            bytes,
         }
     }
 

crates/gpui/src/window.rs

@@ -2100,14 +2100,14 @@ impl Window {
         let (task, is_first) = cx.fetch_asset::<A>(source);
         task.clone().now_or_never().or_else(|| {
             if is_first {
-                let entity = self.current_view();
+                let entity_id = self.current_view();
                 self.spawn(cx, {
                     let task = task.clone();
                     async move |cx| {
                         task.await;
 
                         cx.on_next_frame(move |_, cx| {
-                            cx.notify(entity);
+                            cx.notify(entity_id);
                         });
                     }
                 })

crates/markdown/Cargo.toml

@@ -20,6 +20,7 @@ test-support = [
 
 [dependencies]
 anyhow.workspace = true
+base64.workspace = true
 gpui.workspace = true
 language.workspace = true
 linkify.workspace = true

crates/markdown/examples/markdown.rs

@@ -22,6 +22,15 @@ function a(b: T) {
 ```
 
 Remember, markdown processors may have slight differences and extensions, so always refer to the specific documentation or guides relevant to your platform or editor for the best practices and additional features.
+
+## Images
+
+![Alt Text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgIGZpbGw6ICNDNEVFRDA7IC8qIExpZ2h0IG1vZGUgKi8KICAgIH0KCiAgICBAbWVkaWEgKHByZWZlcnMtY29sb3Itc2NoZW1lOiBkYXJrKSB7CiAgICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgICAgZmlsbDogIzEyNTIyNTsgLyogRGFyayBtb2RlICovCiAgICAgIH0KICAgIH0KICA8L3N0eWxlPgogIDxwYXRoIGQ9Ik00MjAgMzBMMzkwIDYwTDQ4MCAxNTBMMzkwIDI0MEwzMzAgMTgwTDMwMCAyMTBMMzkwIDMwMEw1NDAgMTUwTDQyMCAzMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0xNTAgMEwzMCAxMjBMNjAgMTUwTDE1MCA2MEwyMTAgMTIwTDI0MCA5MEwxNTAgMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0zOTAgMEw0MjAgMzBMMTUwIDMwMEwwIDE1MEwzMCAxMjBMMTUwIDI0MEwzOTAgMFoiIGZpbGw9IiMxRUE0NDYiLz4KPC9zdmc+) item one
+
+![other alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmF5LXNoYXBlIHsKICAgICAgZmlsbDogI0M2QzZDNjsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmdyYXktc2hhcGUgewogICAgICAgIGZpbGw6ICM1NjU2NTY7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMjQwIDkwTDIxMCAxMjBMMTIwIDMwTDE1MCAwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik00MjAgMzBMNTQwIDE1MEw0MjAgMjcwTDM5MCAyNDBMNDgwIDE1MEwzOTAgNjBMNDIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzMDAgMjEwTDM5MCAzMDBMNDIwIDI3MEwzMzAgMTgwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik0xMjAgMzBMMTUwIDYwTDYwIDE1MEwxNTAgMjQwTDEyMCAyNzBMMCAxNTBMMTIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDBMNDIwIDMwTDE1MCAzMDBMMTIwIDI3MEwzOTAgMFoiIGZpbGw9IiNGMDk0MDkiLz4KPC9zdmc+) item two
+
+![third alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ibHVlLXNoYXBlIHsKICAgICAgZmlsbDogI0E4QzdGQTsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmJsdWUtc2hhcGUgewogICAgICAgIGZpbGw6ICMyRDUwOUU7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CgogICAgLmRhcmtlci1ibHVlLXNoYXBlIHsKICAgICAgICBmaWxsOiAjMUI2RUYzOwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgICAuZGFya2VyLWJsdWUtc2hhcGUgewogICAgICAgICAgICBmaWxsOiAjNDE4NUZGOwogICAgICAgIH0KICAgIH0KCiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMTgwIDMwTDE1MCA2MEwxMjAgMzBMMTUwIDBaIiBjbGFzcz0iYmx1ZS1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0yMTAgNjBMMjQwIDkwTDIxMCAxMjBMMTgwIDkwTDIxMCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTQ1MCA2MEw0ODAgOTBMNDUwIDEyMEw0MjAgOTBMNDUwIDYwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNTEwIDEyMEw1NDAgMTUwTDUxMCAxODBMNDgwIDE1MEw1MTAgMTIwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNDUwIDE4MEw0ODAgMjEwTDQ1MCAyNDBMNDIwIDIxMEw0NTAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDI0MEw0MjAgMjcwTDM5MCAzMDBMMzYwIDI3MEwzOTAgMjQwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzNjAgMjEwTDMzMCAyNDBMMzAwIDIxMEwzMzAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNOTAgNjBMMTIwIDkwTDkwIDEyMEw2MCA5MEw5MCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTM5MCAwTDQyMCAzMEwxNTAgMzAwTDAgMTUwTDMwIDEyMEwxNTAgMjQwTDM5MCAwWiIgY2xhc3M9ImRhcmtlci1ibHVlLXNoYXBlIi8+Cjwvc3ZnPg==) item three
+
 "#;
 
 pub fn main() {

crates/markdown/src/markdown.rs

@@ -1,9 +1,12 @@
 pub mod parser;
 mod path_range;
 
+use base64::Engine as _;
+use log::Level;
 pub use path_range::{LineCol, PathWithRange};
 
 use std::borrow::Cow;
+use std::collections::HashMap;
 use std::collections::HashSet;
 use std::iter;
 use std::mem;
@@ -15,10 +18,10 @@ use std::time::Duration;
 
 use gpui::{
     AnyElement, App, BorderStyle, Bounds, ClipboardItem, CursorStyle, DispatchPhase, Edges, Entity,
-    FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, KeyContext,
+    FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, Image,
-    Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent, Point, Stateful,
+    ImageFormat, KeyContext, Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent,
-    StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun, TextStyle,
+    Point, Stateful, StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun,
-    TextStyleRefinement, actions, point, quad,
+    TextStyle, TextStyleRefinement, actions, img, point, quad,
 };
 use language::{Language, LanguageRegistry, Rope};
 use parser::CodeBlockMetadata;
@@ -93,6 +96,7 @@ pub struct Markdown {
     pressed_link: Option<RenderedLink>,
     autoscroll_request: Option<usize>,
     parsed_markdown: ParsedMarkdown,
+    images_by_source_offset: HashMap<usize, Arc<Image>>,
     should_reparse: bool,
     pending_parse: Option<Task<Option<()>>>,
     focus_handle: FocusHandle,
@@ -149,6 +153,7 @@ impl Markdown {
             pressed_link: None,
             autoscroll_request: None,
             should_reparse: false,
+            images_by_source_offset: Default::default(),
             parsed_markdown: ParsedMarkdown::default(),
             pending_parse: None,
             focus_handle,
@@ -172,6 +177,7 @@ impl Markdown {
             autoscroll_request: None,
             should_reparse: false,
             parsed_markdown: ParsedMarkdown::default(),
+            images_by_source_offset: Default::default(),
             pending_parse: None,
             focus_handle,
             language_registry: None,
@@ -269,19 +275,23 @@ impl Markdown {
         }
 
         let source = self.source.clone();
-        let parse_text_only = self.options.parse_links_only;
+        let should_parse_links_only = self.options.parse_links_only;
         let language_registry = self.language_registry.clone();
         let fallback = self.fallback_code_block_language.clone();
         let parsed = cx.background_spawn(async move {
-            if parse_text_only {
+            if should_parse_links_only {
-                return anyhow::Ok(ParsedMarkdown {
+                return anyhow::Ok((
-                    events: Arc::from(parse_links_only(source.as_ref())),
+                    ParsedMarkdown {
-                    source,
+                        events: Arc::from(parse_links_only(source.as_ref())),
-                    languages_by_name: TreeMap::default(),
+                        source,
-                    languages_by_path: TreeMap::default(),
+                        languages_by_name: TreeMap::default(),
-                });
+                        languages_by_path: TreeMap::default(),
+                    },
+                    Default::default(),
+                ));
             }
             let (events, language_names, paths) = parse_markdown(&source);
+            let mut images_by_source_offset = HashMap::default();
             let mut languages_by_name = TreeMap::default();
             let mut languages_by_path = TreeMap::default();
             if let Some(registry) = language_registry.as_ref() {
@@ -304,20 +314,52 @@ impl Markdown {
                     }
                 }
             }
-            anyhow::Ok(ParsedMarkdown {
+
-                source,
+            for (range, event) in &events {
-                events: Arc::from(events),
+                if let MarkdownEvent::Start(MarkdownTag::Image { dest_url, .. }) = event {
-                languages_by_name,
+                    if let Some(data_url) = dest_url.strip_prefix("data:") {
-                languages_by_path,
+                        let Some((mime_info, data)) = data_url.split_once(',') else {
-            })
+                            continue;
+                        };
+                        let Some((mime_type, encoding)) = mime_info.split_once(';') else {
+                            continue;
+                        };
+                        let Some(format) = ImageFormat::from_mime_type(mime_type) else {
+                            continue;
+                        };
+                        let is_base64 = encoding == "base64";
+                        if is_base64 {
+                            if let Some(bytes) = base64::prelude::BASE64_STANDARD
+                                .decode(data)
+                                .log_with_level(Level::Debug)
+                            {
+                                let image = Arc::new(Image::from_bytes(format, bytes));
+                                images_by_source_offset.insert(range.start, image);
+                            }
+                        }
+                    }
+                }
+            }
+
+            anyhow::Ok((
+                ParsedMarkdown {
+                    source,
+                    events: Arc::from(events),
+                    languages_by_name,
+                    languages_by_path,
+                },
+                images_by_source_offset,
+            ))
         });
 
         self.should_reparse = false;
         self.pending_parse = Some(cx.spawn(async move |this, cx| {
             async move {
-                let parsed = parsed.await?;
+                let (parsed, images_by_source_offset) = parsed.await?;
+
                 this.update(cx, |this, cx| {
                     this.parsed_markdown = parsed;
+                    this.images_by_source_offset = images_by_source_offset;
                     this.pending_parse.take();
                     if this.should_reparse {
                         this.parse(cx);
@@ -680,7 +722,9 @@ impl Element for MarkdownElement {
             self.style.base_text_style.clone(),
             self.style.syntax.clone(),
         );
-        let parsed_markdown = &self.markdown.read(cx).parsed_markdown;
+        let markdown = self.markdown.read(cx);
+        let parsed_markdown = &markdown.parsed_markdown;
+        let images = &markdown.images_by_source_offset;
         let markdown_end = if let Some(last) = parsed_markdown.events.last() {
             last.0.end
         } else {
@@ -688,11 +732,29 @@ impl Element for MarkdownElement {
         };
 
         let mut current_code_block_metadata = None;
-
+        let mut current_img_block_range: Option<Range<usize>> = None;
         for (range, event) in parsed_markdown.events.iter() {
+            // Skip alt text for images that rendered
+            if let Some(current_img_block_range) = &current_img_block_range {
+                if current_img_block_range.end > range.end {
+                    continue;
+                }
+            }
+
             match event {
                 MarkdownEvent::Start(tag) => {
                     match tag {
+                        MarkdownTag::Image { .. } => {
+                            if let Some(image) = images.get(&range.start) {
+                                current_img_block_range = Some(range.clone());
+                                builder.modify_current_div(|el| {
+                                    el.items_center()
+                                        .flex()
+                                        .flex_row()
+                                        .child(img(image.clone()))
+                                });
+                            }
+                        }
                         MarkdownTag::Paragraph => {
                             builder.push_div(
                                 div().when(!self.style.height_is_multiple_of_line_height, |el| {
@@ -940,6 +1002,9 @@ impl Element for MarkdownElement {
                     }
                 }
                 MarkdownEvent::End(tag) => match tag {
+                    MarkdownTagEnd::Image => {
+                        current_img_block_range.take();
+                    }
                     MarkdownTagEnd::Paragraph => {
                         builder.pop_div();
                     }

crates/project/src/image_store.rs

@@ -6,8 +6,7 @@ use anyhow::{Context as _, Result, anyhow};
 use collections::{HashMap, HashSet, hash_map};
 use futures::{StreamExt, channel::oneshot};
 use gpui::{
-    App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, hash,
+    App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, prelude::*,
-    prelude::*,
 };
 pub use image::ImageFormat;
 use image::{ExtendedColorType, GenericImageView, ImageReader};
@@ -701,9 +700,8 @@ impl LocalImageStore {
 fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
     let format = image::guess_format(&content)?;
 
-    Ok(Arc::new(gpui::Image {
+    Ok(Arc::new(gpui::Image::from_bytes(
-        id: hash(&content),
+        match format {
-        format: match format {
             image::ImageFormat::Png => gpui::ImageFormat::Png,
             image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg,
             image::ImageFormat::WebP => gpui::ImageFormat::Webp,
@@ -712,8 +710,8 @@ fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
             image::ImageFormat::Tiff => gpui::ImageFormat::Tiff,
             _ => Err(anyhow::anyhow!("Image format not supported"))?,
         },
-        bytes: content,
+        content,
-    }))
+    )))
 }
 
 impl ImageStoreImpl for Entity<RemoteImageStore> {

crates/repl/src/outputs/image.rs

@@ -57,11 +57,7 @@ impl ImageView {
         };
 
         // Convert back to a GPUI image for use with the clipboard
-        let clipboard_image = Arc::new(Image {
+        let clipboard_image = Arc::new(Image::from_bytes(format, bytes));
-            format,
-            bytes,
-            id: gpui_image_data.id.0 as u64,
-        });
 
         Ok(ImageView {
             clipboard_image,