Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement rendering of images with data urls in markdown (#30322)

Browse source 
Fixes #28266

![Screenshot 2025-05-08 at 5 08
21 PM](https://github.com/user-attachments/assets/774d2dde-3f2d-466c-8eb1-c67badbd89e4)

Release Notes:

- Added support for rendering images with data URLs in markdown. This
can show up in hover documentation provided by language servers.

Co-authored-by: Mikayla Maki <mikayla.c.maki@gmail.com>
This commit is contained in:
Max Brunsfeld 2025-05-08 18:26:24 -07:00 • committed by GitHub
parent c512d43e8c
commit 29c31f020e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 131 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.lock generated
View file

@ -8517,6 +8517,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"assets",
"base64 0.22.1",
"env_logger 0.11.8",
"gpui",
"language",

4
crates/agent/src/context.rs
View file

@ -754,11 +754,11 @@ pub enum ImageStatus {
impl ImageContext {
pub fn eq_for_key(&self, other: &Self) -> bool {
self.original_image.id == other.original_image.id
self.original_image.id() == other.original_image.id()
}
pub fn hash_for_key<H: Hasher>(&self, state: &mut H) {
self.original_image.id.hash(state);
self.original_image.id().hash(state);
}
pub fn image(&self) -> Option<LanguageModelImage> {

29
crates/gpui/src/platform.rs
View file

@ -36,7 +36,7 @@ use crate::{
ForegroundExecutor, GlyphId, GpuSpecs, ImageSource, Keymap, LineLayout, Pixels, PlatformInput,
Point, RenderGlyphParams, RenderImage, RenderImageParams, RenderSvgParams, ScaledPixels, Scene,
ShapedGlyph, ShapedRun, SharedString, Size, SvgRenderer, SvgSize, Task, TaskLabel, Window,
point, px, size,
hash, point, px, size,
};
use anyhow::Result;
use async_task::Runnable;
@ -1499,6 +1499,20 @@ impl ImageFormat {
ImageFormat::Tiff => "image/tiff",
}
}
/// Returns the ImageFormat for the given mime type
pub fn from_mime_type(mime_type: &str) -> Option<Self> {
match mime_type {
"image/png" => Some(Self::Png),
"image/jpeg" | "image/jpg" => Some(Self::Jpeg),
"image/webp" => Some(Self::Webp),
"image/gif" => Some(Self::Gif),
"image/svg+xml" => Some(Self::Svg),
"image/bmp" => Some(Self::Bmp),
"image/tiff" | "image/tif" => Some(Self::Tiff),
_ => None,
}
}
}
/// An image, with a format and certain bytes
@ -1509,7 +1523,7 @@ pub struct Image {
/// The raw image bytes
pub bytes: Vec<u8>,
/// The unique ID for the image
pub id: u64,
id: u64,
}
impl Hash for Image {
@ -1521,10 +1535,15 @@ impl Hash for Image {
impl Image {
/// An empty image containing no data
pub fn empty() -> Self {
Self::from_bytes(ImageFormat::Png, Vec::new())
}
/// Create an image from a format and bytes
pub fn from_bytes(format: ImageFormat, bytes: Vec<u8>) -> Self {
Self {
format: ImageFormat::Png,
bytes: Vec::new(),
id: 0,
id: hash(&bytes),
format,
bytes,
}
}

4
crates/gpui/src/window.rs
View file

@ -2100,14 +2100,14 @@ impl Window {
let (task, is_first) = cx.fetch_asset::<A>(source);
task.clone().now_or_never().or_else(|| {
if is_first {
let entity = self.current_view();
let entity_id = self.current_view();
self.spawn(cx, {
let task = task.clone();
async move |cx| {
task.await;
cx.on_next_frame(move |_, cx| {
cx.notify(entity);
cx.notify(entity_id);
});
}
})

1
crates/markdown/Cargo.toml
View file

@ -20,6 +20,7 @@ test-support = [
[dependencies]
anyhow.workspace = true
base64.workspace = true
gpui.workspace = true
language.workspace = true
linkify.workspace = true

9
crates/markdown/examples/markdown.rs
View file

@ -22,6 +22,15 @@ function a(b: T) {
```
Remember, markdown processors may have slight differences and extensions, so always refer to the specific documentation or guides relevant to your platform or editor for the best practices and additional features.
## Images
![Alt Text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgIGZpbGw6ICNDNEVFRDA7IC8qIExpZ2h0IG1vZGUgKi8KICAgIH0KCiAgICBAbWVkaWEgKHByZWZlcnMtY29sb3Itc2NoZW1lOiBkYXJrKSB7CiAgICAgIC5ncmVlbi1zaGFwZSB7CiAgICAgICAgZmlsbDogIzEyNTIyNTsgLyogRGFyayBtb2RlICovCiAgICAgIH0KICAgIH0KICA8L3N0eWxlPgogIDxwYXRoIGQ9Ik00MjAgMzBMMzkwIDYwTDQ4MCAxNTBMMzkwIDI0MEwzMzAgMTgwTDMwMCAyMTBMMzkwIDMwMEw1NDAgMTUwTDQyMCAzMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0xNTAgMEwzMCAxMjBMNjAgMTUwTDE1MCA2MEwyMTAgMTIwTDI0MCA5MEwxNTAgMFoiIGNsYXNzPSJncmVlbi1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0zOTAgMEw0MjAgMzBMMTUwIDMwMEwwIDE1MEwzMCAxMjBMMTUwIDI0MEwzOTAgMFoiIGZpbGw9IiMxRUE0NDYiLz4KPC9zdmc+) item one
![other alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ncmF5LXNoYXBlIHsKICAgICAgZmlsbDogI0M2QzZDNjsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmdyYXktc2hhcGUgewogICAgICAgIGZpbGw6ICM1NjU2NTY7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMjQwIDkwTDIxMCAxMjBMMTIwIDMwTDE1MCAwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik00MjAgMzBMNTQwIDE1MEw0MjAgMjcwTDM5MCAyNDBMNDgwIDE1MEwzOTAgNjBMNDIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzMDAgMjEwTDM5MCAzMDBMNDIwIDI3MEwzMzAgMTgwWiIgZmlsbD0iI0YwOTQwOSIvPgogIDxwYXRoIGQ9Ik0xMjAgMzBMMTUwIDYwTDYwIDE1MEwxNTAgMjQwTDEyMCAyNzBMMCAxNTBMMTIwIDMwWiIgY2xhc3M9ImdyYXktc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDBMNDIwIDMwTDE1MCAzMDBMMTIwIDI3MEwzOTAgMFoiIGZpbGw9IiNGMDk0MDkiLz4KPC9zdmc+) item two
![third alt text](data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMTgiIGhlaWdodD0iMTAiIHZpZXdCb3g9IjAgMCA1NDAgMzAwIiBmaWxsPSJub25lIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciPgogIDxzdHlsZT4KICAgIC5ibHVlLXNoYXBlIHsKICAgICAgZmlsbDogI0E4QzdGQTsgLyogTGlnaHQgbW9kZSAqLwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgLmJsdWUtc2hhcGUgewogICAgICAgIGZpbGw6ICMyRDUwOUU7IC8qIERhcmsgbW9kZSAqLwogICAgICB9CiAgICB9CgogICAgLmRhcmtlci1ibHVlLXNoYXBlIHsKICAgICAgICBmaWxsOiAjMUI2RUYzOwogICAgfQoKICAgIEBtZWRpYSAocHJlZmVycy1jb2xvci1zY2hlbWU6IGRhcmspIHsKICAgICAgICAuZGFya2VyLWJsdWUtc2hhcGUgewogICAgICAgICAgICBmaWxsOiAjNDE4NUZGOwogICAgICAgIH0KICAgIH0KCiAgPC9zdHlsZT4KICA8cGF0aCBkPSJNMTUwIDBMMTgwIDMwTDE1MCA2MEwxMjAgMzBMMTUwIDBaIiBjbGFzcz0iYmx1ZS1zaGFwZSIvPgogIDxwYXRoIGQ9Ik0yMTAgNjBMMjQwIDkwTDIxMCAxMjBMMTgwIDkwTDIxMCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTQ1MCA2MEw0ODAgOTBMNDUwIDEyMEw0MjAgOTBMNDUwIDYwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNTEwIDEyMEw1NDAgMTUwTDUxMCAxODBMNDgwIDE1MEw1MTAgMTIwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNNDUwIDE4MEw0ODAgMjEwTDQ1MCAyNDBMNDIwIDIxMEw0NTAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzkwIDI0MEw0MjAgMjcwTDM5MCAzMDBMMzYwIDI3MEwzOTAgMjQwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNMzMwIDE4MEwzNjAgMjEwTDMzMCAyNDBMMzAwIDIxMEwzMzAgMTgwWiIgY2xhc3M9ImJsdWUtc2hhcGUiLz4KICA8cGF0aCBkPSJNOTAgNjBMMTIwIDkwTDkwIDEyMEw2MCA5MEw5MCA2MFoiIGNsYXNzPSJibHVlLXNoYXBlIi8+CiAgPHBhdGggZD0iTTM5MCAwTDQyMCAzMEwxNTAgMzAwTDAgMTUwTDMwIDEyMEwxNTAgMjQwTDM5MCAwWiIgY2xhc3M9ImRhcmtlci1ibHVlLXNoYXBlIi8+Cjwvc3ZnPg==) item three
"#;
pub fn main() {

107
crates/markdown/src/markdown.rs
View file

@ -1,9 +1,12 @@
pub mod parser;
mod path_range;
use base64::Engine as _;
use log::Level;
pub use path_range::{LineCol, PathWithRange};
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::iter;
use std::mem;
@ -15,10 +18,10 @@ use std::time::Duration;
use gpui::{
AnyElement, App, BorderStyle, Bounds, ClipboardItem, CursorStyle, DispatchPhase, Edges, Entity,
FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, KeyContext,
Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent, Point, Stateful,
StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun, TextStyle,
TextStyleRefinement, actions, point, quad,
FocusHandle, Focusable, FontStyle, FontWeight, GlobalElementId, Hitbox, Hsla, Image,
ImageFormat, KeyContext, Length, MouseDownEvent, MouseEvent, MouseMoveEvent, MouseUpEvent,
Point, Stateful, StrikethroughStyle, StyleRefinement, StyledText, Task, TextLayout, TextRun,
TextStyle, TextStyleRefinement, actions, img, point, quad,
};
use language::{Language, LanguageRegistry, Rope};
use parser::CodeBlockMetadata;
@ -93,6 +96,7 @@ pub struct Markdown {
pressed_link: Option<RenderedLink>,
autoscroll_request: Option<usize>,
parsed_markdown: ParsedMarkdown,
images_by_source_offset: HashMap<usize, Arc<Image>>,
should_reparse: bool,
pending_parse: Option<Task<Option<()>>>,
focus_handle: FocusHandle,
@ -149,6 +153,7 @@ impl Markdown {
pressed_link: None,
autoscroll_request: None,
should_reparse: false,
images_by_source_offset: Default::default(),
parsed_markdown: ParsedMarkdown::default(),
pending_parse: None,
focus_handle,
@ -172,6 +177,7 @@ impl Markdown {
autoscroll_request: None,
should_reparse: false,
parsed_markdown: ParsedMarkdown::default(),
images_by_source_offset: Default::default(),
pending_parse: None,
focus_handle,
language_registry: None,
@ -269,19 +275,23 @@ impl Markdown {
}
let source = self.source.clone();
let parse_text_only = self.options.parse_links_only;
let should_parse_links_only = self.options.parse_links_only;
let language_registry = self.language_registry.clone();
let fallback = self.fallback_code_block_language.clone();
let parsed = cx.background_spawn(async move {
if parse_text_only {
return anyhow::Ok(ParsedMarkdown {
events: Arc::from(parse_links_only(source.as_ref())),
source,
languages_by_name: TreeMap::default(),
languages_by_path: TreeMap::default(),
});
if should_parse_links_only {
return anyhow::Ok((
ParsedMarkdown {
events: Arc::from(parse_links_only(source.as_ref())),
source,
languages_by_name: TreeMap::default(),
languages_by_path: TreeMap::default(),
},
Default::default(),
));
}
let (events, language_names, paths) = parse_markdown(&source);
let mut images_by_source_offset = HashMap::default();
let mut languages_by_name = TreeMap::default();
let mut languages_by_path = TreeMap::default();
if let Some(registry) = language_registry.as_ref() {
@ -304,20 +314,52 @@ impl Markdown {
}
}
}
anyhow::Ok(ParsedMarkdown {
source,
events: Arc::from(events),
languages_by_name,
languages_by_path,
})
for (range, event) in &events {
if let MarkdownEvent::Start(MarkdownTag::Image { dest_url, .. }) = event {
if let Some(data_url) = dest_url.strip_prefix("data:") {
let Some((mime_info, data)) = data_url.split_once(',') else {
continue;
};
let Some((mime_type, encoding)) = mime_info.split_once(';') else {
continue;
};
let Some(format) = ImageFormat::from_mime_type(mime_type) else {
continue;
};
let is_base64 = encoding == "base64";
if is_base64 {
if let Some(bytes) = base64::prelude::BASE64_STANDARD
.decode(data)
.log_with_level(Level::Debug)
{
let image = Arc::new(Image::from_bytes(format, bytes));
images_by_source_offset.insert(range.start, image);
}
}
}
}
}
anyhow::Ok((
ParsedMarkdown {
source,
events: Arc::from(events),
languages_by_name,
languages_by_path,
},
images_by_source_offset,
))
});
self.should_reparse = false;
self.pending_parse = Some(cx.spawn(async move |this, cx| {
async move {
let parsed = parsed.await?;
let (parsed, images_by_source_offset) = parsed.await?;
this.update(cx, |this, cx| {
this.parsed_markdown = parsed;
this.images_by_source_offset = images_by_source_offset;
this.pending_parse.take();
if this.should_reparse {
this.parse(cx);
@ -680,7 +722,9 @@ impl Element for MarkdownElement {
self.style.base_text_style.clone(),
self.style.syntax.clone(),
);
let parsed_markdown = &self.markdown.read(cx).parsed_markdown;
let markdown = self.markdown.read(cx);
let parsed_markdown = &markdown.parsed_markdown;
let images = &markdown.images_by_source_offset;
let markdown_end = if let Some(last) = parsed_markdown.events.last() {
last.0.end
} else {
@ -688,11 +732,29 @@ impl Element for MarkdownElement {
};
let mut current_code_block_metadata = None;
let mut current_img_block_range: Option<Range<usize>> = None;
for (range, event) in parsed_markdown.events.iter() {
// Skip alt text for images that rendered
if let Some(current_img_block_range) = &current_img_block_range {
if current_img_block_range.end > range.end {
continue;
}
}
match event {
MarkdownEvent::Start(tag) => {
match tag {
MarkdownTag::Image { .. } => {
if let Some(image) = images.get(&range.start) {
current_img_block_range = Some(range.clone());
builder.modify_current_div(|el| {
el.items_center()
.flex()
.flex_row()
.child(img(image.clone()))
});
}
}
MarkdownTag::Paragraph => {
builder.push_div(
div().when(!self.style.height_is_multiple_of_line_height, |el| {
@ -940,6 +1002,9 @@ impl Element for MarkdownElement {
}
}
MarkdownEvent::End(tag) => match tag {
MarkdownTagEnd::Image => {
current_img_block_range.take();
}
MarkdownTagEnd::Paragraph => {
builder.pop_div();
}

12
crates/project/src/image_store.rs
View file

@ -6,8 +6,7 @@ use anyhow::{Context as _, Result, anyhow};
use collections::{HashMap, HashSet, hash_map};
use futures::{StreamExt, channel::oneshot};
use gpui::{
App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, hash,
prelude::*,
App, AsyncApp, Context, Entity, EventEmitter, Img, Subscription, Task, WeakEntity, prelude::*,
};
pub use image::ImageFormat;
use image::{ExtendedColorType, GenericImageView, ImageReader};
@ -701,9 +700,8 @@ impl LocalImageStore {
fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
let format = image::guess_format(&content)?;
Ok(Arc::new(gpui::Image {
id: hash(&content),
format: match format {
Ok(Arc::new(gpui::Image::from_bytes(
match format {
image::ImageFormat::Png => gpui::ImageFormat::Png,
image::ImageFormat::Jpeg => gpui::ImageFormat::Jpeg,
image::ImageFormat::WebP => gpui::ImageFormat::Webp,
@ -712,8 +710,8 @@ fn create_gpui_image(content: Vec<u8>) -> anyhow::Result<Arc<gpui::Image>> {
image::ImageFormat::Tiff => gpui::ImageFormat::Tiff,
_ => Err(anyhow::anyhow!("Image format not supported"))?,
},
bytes: content,
}))
content,
)))
}
impl ImageStoreImpl for Entity<RemoteImageStore> {

6
crates/repl/src/outputs/image.rs
View file

@ -57,11 +57,7 @@ impl ImageView {
};
// Convert back to a GPUI image for use with the clipboard
let clipboard_image = Arc::new(Image {
format,
bytes,
id: gpui_image_data.id.0 as u64,
});
let clipboard_image = Arc::new(Image::from_bytes(format, bytes));
Ok(ImageView {
clipboard_image,