From 29c31f020e30004ca9bfae269fd39868d6cdc660 Mon Sep 17 00:00:00 2001 From: Max Brunsfeld Date: Thu, 8 May 2025 18:26:24 -0700 Subject: [PATCH] Implement rendering of images with data urls in markdown (#30322) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #28266 ![Screenshot 2025-05-08 at 5 08 21 PM](https://github.com/user-attachments/assets/774d2dde-3f2d-466c-8eb1-c67badbd89e4) Release Notes: - Added support for rendering images with data URLs in markdown. This can show up in hover documentation provided by language servers. Co-authored-by: Mikayla Maki --- Cargo.lock | 1 + crates/agent/src/context.rs | 4 +- crates/gpui/src/platform.rs | 29 ++++++-- crates/gpui/src/window.rs | 4 +- crates/markdown/Cargo.toml | 1 + crates/markdown/examples/markdown.rs | 9 +++ crates/markdown/src/markdown.rs | 107 +++++++++++++++++++++------ crates/project/src/image_store.rs | 12 ++- crates/repl/src/outputs/image.rs | 6 +- 9 files changed, 131 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6b6d19327..ab9525987d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8517,6 +8517,7 @@ version = "0.1.0" dependencies = [ "anyhow", "assets", + "base64 0.22.1", "env_logger 0.11.8", "gpui", "language", diff --git a/crates/agent/src/context.rs b/crates/agent/src/context.rs index 09e5c92396..b89a6f10c8 100644 --- a/crates/agent/src/context.rs +++ b/crates/agent/src/context.rs @@ -754,11 +754,11 @@ pub enum ImageStatus { impl ImageContext { pub fn eq_for_key(&self, other: &Self) -> bool { - self.original_image.id == other.original_image.id + self.original_image.id() == other.original_image.id() } pub fn hash_for_key(&self, state: &mut H) { - self.original_image.id.hash(state); + self.original_image.id().hash(state); } pub fn image(&self) -> Option { diff --git a/crates/gpui/src/platform.rs b/crates/gpui/src/platform.rs index ae2ee2f2d1..6f470cccec 100644 --- a/crates/gpui/src/platform.rs +++ b/crates/gpui/src/platform.rs @@ -36,7 +36,7 @@ use crate::{ ForegroundExecutor, GlyphId, GpuSpecs, ImageSource, Keymap, LineLayout, Pixels, PlatformInput, Point, RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, ScaledPixels, Scene, ShapedGlyph, ShapedRun, SharedString, Size, SvgRenderer, SvgSize, Task, TaskLabel, Window, - point, px, size, + hash, point, px, size, }; use anyhow::Result; use async_task::Runnable; @@ -1499,6 +1499,20 @@ impl ImageFormat { ImageFormat::Tiff => "image/tiff", } } + + /// Returns the ImageFormat for the given mime type + pub fn from_mime_type(mime_type: &str) -> Option { + match mime_type { + "image/png" => Some(Self::Png), + "image/jpeg" | "image/jpg" => Some(Self::Jpeg), + "image/webp" => Some(Self::Webp), + "image/gif" => Some(Self::Gif), + "image/svg+xml" => Some(Self::Svg), + "image/bmp" => Some(Self::Bmp), + "image/tiff" | "image/tif" => Some(Self::Tiff), + _ => None, + } + } } /// An image, with a format and certain bytes @@ -1509,7 +1523,7 @@ pub struct Image { /// The raw image bytes pub bytes: Vec, /// The unique ID for the image - pub id: u64, + id: u64, } impl Hash for Image { @@ -1521,10 +1535,15 @@ impl Hash for Image { impl Image { /// An empty image containing no data pub fn empty() -> Self { + Self::from_bytes(ImageFormat::Png, Vec::new()) + } + + /// Create an image from a format and bytes + pub fn from_bytes(format: ImageFormat, bytes: Vec) -> Self { Self { - format: ImageFormat::Png, - bytes: Vec::new(), - id: 0, + id: hash(&bytes), + format, + bytes, } } diff --git a/crates/gpui/src/window.rs b/crates/gpui/src/window.rs index 6eac25cb6c..14c12693df 100644 --- a/crates/gpui/src/window.rs +++ b/crates/gpui/src/window.rs @@ -2100,14 +2100,14 @@ impl Window { let (task, is_first) = cx.fetch_asset::(source); task.clone().now_or_never().or_else(|| { if is_first { - let entity = self.current_view(); + let entity_id = self.current_view(); self.spawn(cx, { let task = task.clone(); async move |cx| { task.await; cx.on_next_frame(move |_, cx| { - cx.notify(entity); + cx.notify(entity_id); }); } }) diff --git a/crates/markdown/Cargo.toml b/crates/markdown/Cargo.toml index 5e83049498..e925d6c4a5 100644 --- a/crates/markdown/Cargo.toml +++ b/crates/markdown/Cargo.toml @@ -20,6 +20,7 @@ test-support = [ [dependencies] anyhow.workspace = true +base64.workspace = true gpui.workspace = true language.workspace = true linkify.workspace = true diff --git a/crates/markdown/examples/markdown.rs b/crates/markdown/examples/markdown.rs index fbe2d4326f..263579ef2b 100644 --- a/crates/markdown/examples/markdown.rs +++ b/crates/markdown/examples/markdown.rs @@ -22,6 +22,15 @@ function a(b: T) { ``` Remember, markdown processors may have slight differences and extensions, so always refer to the specific documentation or guides relevant to your platform or editor for the best practices and additional features. + +## Images + +![Alt Text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgIGZpbGw6ICNDNEVFRDA7IC8qIExpZ2h0IG1vZGUgKi8KICAgIH0KCiAgICBAbWVkaWEgKHByZWZlcnMtY29sb3Itc2NoZW1lOiBkYXJrKSB7CiAgICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgICAgZmlsbDogIzEyNTIyNTsgLyogRGFyayBtb2RlICovCiAgICAgIH0KICAgIH0KICA8L3N0eWxlPgogIDxwYXRoIGQ9Ik00MjAgMzBMMzkwIDYwTDQ4MCAxNTBMMzkwIDI0MEwzMzAgMTgwTDMwMCAyMTBMMzkwIDMwMEw1NDAgMTUwTDQyMCAzMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0xNTAgMEwzMCAxMjBMNjAgMTUwTDE1MCA2MEwyMTAgMTIwTDI0MCA5MEwxNTAgMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0zOTAgMEw0MjAgMzBMMTUwIDMwMEwwIDE1MEwzMCAxMjBMMTUwIDI0MEwzOTAgMFoiIGZpbGw9IiMxRUE0NDYiLz4KPC9zdmc+) item one + +![other alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmF5LXNoYXBlIHsKICAgICAgZmlsbDogI0M2QzZDNjsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmdyYXktc2hhcGUgewogICAgICAgIGZpbGw6ICM1NjU2NTY7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMjQwIDkwTDIxMCAxMjBMMTIwIDMwTDE1MCAwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik00MjAgMzBMNTQwIDE1MEw0MjAgMjcwTDM5MCAyNDBMNDgwIDE1MEwzOTAgNjBMNDIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzMDAgMjEwTDM5MCAzMDBMNDIwIDI3MEwzMzAgMTgwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik0xMjAgMzBMMTUwIDYwTDYwIDE1MEwxNTAgMjQwTDEyMCAyNzBMMCAxNTBMMTIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDBMNDIwIDMwTDE1MCAzMDBMMTIwIDI3MEwzOTAgMFoiIGZpbGw9IiNGMDk0MDkiLz4KPC9zdmc+) item two + +![third alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ibHVlLXNoYXBlIHsKICAgICAgZmlsbDogI0E4QzdGQTsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmJsdWUtc2hhcGUgewogICAgICAgIGZpbGw6ICMyRDUwOUU7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CgogICAgLmRhcmtlci1ibHVlLXNoYXBlIHsKICAgICAgICBmaWxsOiAjMUI2RUYzOwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgICAuZGFya2VyLWJsdWUtc2hhcGUgewogICAgICAgICAgICBmaWxsOiAjNDE4NUZGOwogICAgICAgIH0KICAgIH0KCiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMTgwIDMwTDE1MCA2MEwxMjAgMzBMMTUwIDBaIiBjbGFzcz0iYmx1ZS1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0yMTAgNjBMMjQwIDkwTDIxMCAxMjBMMTgwIDkwTDIxMCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTQ1MCA2MEw0ODAgOTBMNDUwIDEyMEw0MjAgOTBMNDUwIDYwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNTEwIDEyMEw1NDAgMTUwTDUxMCAxODBMNDgwIDE1MEw1MTAgMTIwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNDUwIDE4MEw0ODAgMjEwTDQ1MCAyNDBMNDIwIDIxMEw0NTAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDI0MEw0MjAgMjcwTDM5MCAzMDBMMzYwIDI3MEwzOTAgMjQwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzNjAgMjEwTDMzMCAyNDBMMzAwIDIxMEwzMzAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNOTAgNjBMMTIwIDkwTDkwIDEyMEw2MCA5MEw5MCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTM5MCAwTDQyMCAzMEwxNTAgMzAwTDAgMTUwTDMwIDEyMEwxNTAgMjQwTDM5MCAwWiIgY2xhc3M9ImRhcmtlci1ibHVlLXNoYXBlIi8+Cjwvc3ZnPg==) item three + "#; pub fn main() { diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rs index 509b70125e..8ef88901f3 100644 --- a/crates/markdown/src/markdown.rs +++ b/crates/markdown/src/markdown.rs @@ -1,9 +1,12 @@ pub mod parser; mod path_range; +use base64::Engine as _; +use log::Level; pub use path_range::{LineCol, PathWithRange}; use std::borrow::Cow; +use std::collections::HashMap; use std::collections::HashSet; use std::iter; use std::mem; @@ -15,10 +18,10 @@ use std::time::Duration; use gpui::{ AnyElement, App, BorderStyle, Bounds, ClipboardItem, CursorStyle, DispatchPhase, Edges, Entity, - FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, KeyContext, - Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent, Point, Stateful, - StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun, TextStyle, - TextStyleRefinement, actions, point, quad, + FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, Image, + ImageFormat, KeyContext, Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent, + Point, Stateful, StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun, + TextStyle, TextStyleRefinement, actions, img, point, quad, }; use language::{Language, LanguageRegistry, Rope}; use parser::CodeBlockMetadata; @@ -93,6 +96,7 @@ pub struct Markdown { pressed_link: Option, autoscroll_request: Option, parsed_markdown: ParsedMarkdown, + images_by_source_offset: HashMap>, should_reparse: bool, pending_parse: Option>>, focus_handle: FocusHandle, @@ -149,6 +153,7 @@ impl Markdown { pressed_link: None, autoscroll_request: None, should_reparse: false, + images_by_source_offset: Default::default(), parsed_markdown: ParsedMarkdown::default(), pending_parse: None, focus_handle, @@ -172,6 +177,7 @@ impl Markdown { autoscroll_request: None, should_reparse: false, parsed_markdown: ParsedMarkdown::default(), + images_by_source_offset: Default::default(), pending_parse: None, focus_handle, language_registry: None, @@ -269,19 +275,23 @@ impl Markdown { } let source = self.source.clone(); - let parse_text_only = self.options.parse_links_only; + let should_parse_links_only = self.options.parse_links_only; let language_registry = self.language_registry.clone(); let fallback = self.fallback_code_block_language.clone(); let parsed = cx.background_spawn(async move { - if parse_text_only { - return anyhow::Ok(ParsedMarkdown { - events: Arc::from(parse_links_only(source.as_ref())), - source, - languages_by_name: TreeMap::default(), - languages_by_path: TreeMap::default(), - }); + if should_parse_links_only { + return anyhow::Ok(( + ParsedMarkdown { + events: Arc::from(parse_links_only(source.as_ref())), + source, + languages_by_name: TreeMap::default(), + languages_by_path: TreeMap::default(), + }, + Default::default(), + )); } let (events, language_names, paths) = parse_markdown(&source); + let mut images_by_source_offset = HashMap::default(); let mut languages_by_name = TreeMap::default(); let mut languages_by_path = TreeMap::default(); if let Some(registry) = language_registry.as_ref() { @@ -304,20 +314,52 @@ impl Markdown { } } } - anyhow::Ok(ParsedMarkdown { - source, - events: Arc::from(events), - languages_by_name, - languages_by_path, - }) + + for (range, event) in &events { + if let MarkdownEvent::Start(MarkdownTag::Image { dest_url, .. }) = event { + if let Some(data_url) = dest_url.strip_prefix("data:") { + let Some((mime_info, data)) = data_url.split_once(',') else { + continue; + }; + let Some((mime_type, encoding)) = mime_info.split_once(';') else { + continue; + }; + let Some(format) = ImageFormat::from_mime_type(mime_type) else { + continue; + }; + let is_base64 = encoding == "base64"; + if is_base64 { + if let Some(bytes) = base64::prelude::BASE64_STANDARD + .decode(data) + .log_with_level(Level::Debug) + { + let image = Arc::new(Image::from_bytes(format, bytes)); + images_by_source_offset.insert(range.start, image); + } + } + } + } + } + + anyhow::Ok(( + ParsedMarkdown { + source, + events: Arc::from(events), + languages_by_name, + languages_by_path, + }, + images_by_source_offset, + )) }); self.should_reparse = false; self.pending_parse = Some(cx.spawn(async move |this, cx| { async move { - let parsed = parsed.await?; + let (parsed, images_by_source_offset) = parsed.await?; + this.update(cx, |this, cx| { this.parsed_markdown = parsed; + this.images_by_source_offset = images_by_source_offset; this.pending_parse.take(); if this.should_reparse { this.parse(cx); @@ -680,7 +722,9 @@ impl Element for MarkdownElement { self.style.base_text_style.clone(), self.style.syntax.clone(), ); - let parsed_markdown = &self.markdown.read(cx).parsed_markdown; + let markdown = self.markdown.read(cx); + let parsed_markdown = &markdown.parsed_markdown; + let images = &markdown.images_by_source_offset; let markdown_end = if let Some(last) = parsed_markdown.events.last() { last.0.end } else { @@ -688,11 +732,29 @@ impl Element for MarkdownElement { }; let mut current_code_block_metadata = None; - + let mut current_img_block_range: Option> = None; for (range, event) in parsed_markdown.events.iter() { + // Skip alt text for images that rendered + if let Some(current_img_block_range) = ¤t_img_block_range { + if current_img_block_range.end > range.end { + continue; + } + } + match event { MarkdownEvent::Start(tag) => { match tag { + MarkdownTag::Image { .. } => { + if let Some(image) = images.get(&range.start) { + current_img_block_range = Some(range.clone()); + builder.modify_current_div(|el| { + el.items_center() + .flex() + .flex_row() + .child(img(image.clone())) + }); + } + } MarkdownTag::Paragraph => { builder.push_div( div().when(!self.style.height_is_multiple_of_line_height, |el| { @@ -940,6 +1002,9 @@ impl Element for MarkdownElement { } } MarkdownEvent::End(tag) => match tag { + MarkdownTagEnd::Image => { + current_img_block_range.take(); + } MarkdownTagEnd::Paragraph => { builder.pop_div(); } diff --git a/crates/project/src/image_store.rs b/crates/project/src/image_store.rs index dd8db2c675..22861488bf 100644 --- a/crates/project/src/image_store.rs +++ b/crates/project/src/image_store.rs @@ -6,8 +6,7 @@ use anyhow::{Context as _, Result, anyhow}; use collections::{HashMap, HashSet, hash_map}; use futures::{StreamExt, channel::oneshot}; use gpui::{ - App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, hash, - prelude::*, + App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, prelude::*, }; pub use image::ImageFormat; use image::{ExtendedColorType, GenericImageView, ImageReader}; @@ -701,9 +700,8 @@ impl LocalImageStore { fn create_gpui_image(content: Vec) -> anyhow::Result> { let format = image::guess_format(&content)?; - Ok(Arc::new(gpui::Image { - id: hash(&content), - format: match format { + Ok(Arc::new(gpui::Image::from_bytes( + match format { image::ImageFormat::Png => gpui::ImageFormat::Png, image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg, image::ImageFormat::WebP => gpui::ImageFormat::Webp, @@ -712,8 +710,8 @@ fn create_gpui_image(content: Vec) -> anyhow::Result> { image::ImageFormat::Tiff => gpui::ImageFormat::Tiff, _ => Err(anyhow::anyhow!("Image format not supported"))?, }, - bytes: content, - })) + content, + ))) } impl ImageStoreImpl for Entity { diff --git a/crates/repl/src/outputs/image.rs b/crates/repl/src/outputs/image.rs index c5b54df77a..f34d95ae54 100644 --- a/crates/repl/src/outputs/image.rs +++ b/crates/repl/src/outputs/image.rs @@ -57,11 +57,7 @@ impl ImageView { }; // Convert back to a GPUI image for use with the clipboard - let clipboard_image = Arc::new(Image { - format, - bytes, - id: gpui_image_data.id.0 as u64, - }); + let clipboard_image = Arc::new(Image::from_bytes(format, bytes)); Ok(ImageView { clipboard_image,