Yehowshua/ZIm
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Introduce surface rendering

Browse source 
Co-Authored-By: Julia <julia@zed.dev>
This commit is contained in:
Antonio Scandurra 2023-11-28 16:40:43 +01:00
parent eac4b2d076
commit 600b564bbf
7 changed files with 332 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

18
crates/call2/src/shared_screen.rs
View file

@ -3,8 +3,8 @@ use anyhow::Result;
use client::{proto::PeerId, User};
use futures::StreamExt;
use gpui::{
div, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement, Render,
SharedString, Task, View, ViewContext, VisualContext, WindowContext,
div, img, AppContext, Div, Element, EventEmitter, FocusHandle, FocusableView, ParentElement,
Render, SharedString, Styled, Task, View, ViewContext, VisualContext, WindowContext,
};
use std::sync::{Arc, Weak};
use workspace::{item::Item, ItemNavHistory, WorkspaceId};
@ -68,15 +68,11 @@ impl Render for SharedScreen {
type Element = Div;
fn render(&mut self, _: &mut ViewContext<Self>) -> Self::Element {
let frame = self.frame.clone();
let frame_id = self.current_frame_id;
self.current_frame_id = self.current_frame_id.wrapping_add(1);
div().children(frame.map(|_| {
ui::Label::new(frame_id.to_string()).color(ui::Color::Error)
// img().data(Arc::new(ImageData::new(image::ImageBuffer::new(
// frame.width() as u32,
// frame.height() as u32,
// ))))
}))
// let frame_id = self.current_frame_id;
// self.current_frame_id = self.current_frame_id.wrapping_add(1);
div()
.size_full()
.children(frame.map(|frame| img().size_full().surface(frame.image())))
}
}
// impl View for SharedScreen {

2
crates/gpui2/build.rs
View file

@ -65,6 +65,8 @@ fn generate_shader_bindings() -> PathBuf {
"MonochromeSprite".into(),
"PolychromeSprite".into(),
"PathSprite".into(),
"SurfaceInputIndex".into(),
"SurfaceBounds".into(),
]);
config.no_includes = true;
config.enumeration.prefix_with_name = true;

41
crates/gpui2/src/elements/img.rs
View file

@ -5,6 +5,7 @@ use crate::{
IntoElement, LayoutId, Pixels, SharedString, StyleRefinement, Styled, WindowContext,
};
use futures::FutureExt;
use media::core_video::CVImageBuffer;
use util::ResultExt;
#[derive(Clone, Debug)]
@ -12,6 +13,7 @@ pub enum ImageSource {
/// Image content will be loaded from provided URI at render time.
Uri(SharedString),
Data(Arc<ImageData>),
Surface(CVImageBuffer),
}
impl From<SharedString> for ImageSource {
@ -26,6 +28,12 @@ impl From<Arc<ImageData>> for ImageSource {
}
}
impl From<CVImageBuffer> for ImageSource {
fn from(value: CVImageBuffer) -> Self {
Self::Surface(value)
}
}
pub struct Img {
interactivity: Interactivity,
source: Option<ImageSource>,
@ -45,11 +53,17 @@ impl Img {
self.source = Some(ImageSource::from(uri.into()));
self
}
pub fn data(mut self, data: Arc<ImageData>) -> Self {
self.source = Some(ImageSource::from(data));
self
}
pub fn surface(mut self, data: CVImageBuffer) -> Self {
self.source = Some(ImageSource::from(data));
self
}
pub fn source(mut self, source: impl Into<ImageSource>) -> Self {
self.source = Some(source.into());
self
@ -85,10 +99,10 @@ impl Element for Img {
element_state,
cx,
|style, _scroll_offset, cx| {
let corner_radii = style.corner_radii;
let corner_radii = style.corner_radii.to_pixels(bounds.size, cx.rem_size());
cx.with_z_index(1, |cx| {
if let Some(source) = self.source {
let image = match source {
match source {
ImageSource::Uri(uri) => {
let image_future = cx.image_cache.get(uri.clone());
if let Some(data) = image_future
@ -96,7 +110,8 @@ impl Element for Img {
.now_or_never()
.and_then(|result| result.ok())
{
data
cx.paint_image(bounds, corner_radii, data, self.grayscale)
.log_err();
} else {
cx.spawn(|mut cx| async move {
if image_future.await.ok().is_some() {
@ -104,17 +119,21 @@ impl Element for Img {
}
})
.detach();
return;
}
}
ImageSource::Data(image) => image,
};
let corner_radii = corner_radii.to_pixels(bounds.size, cx.rem_size());
cx.with_z_index(1, |cx| {
ImageSource::Data(image) => {
cx.paint_image(bounds, corner_radii, image, self.grayscale)
.log_err()
});
.log_err();
}
ImageSource::Surface(surface) => {
// TODO: Add support for corner_radii and grayscale.
cx.paint_surface(bounds, surface);
}
};
}
});
},
)
}

138
crates/gpui2/src/platform/mac/metal_renderer.rs
View file

@ -1,7 +1,7 @@
use crate::{
point, size, AtlasTextureId, AtlasTextureKind, AtlasTile, Bounds, ContentMask, DevicePixels,
Hsla, MetalAtlas, MonochromeSprite, Path, PathId, PathVertex, PolychromeSprite, PrimitiveBatch,
Quad, ScaledPixels, Scene, Shadow, Size, Underline,
Quad, ScaledPixels, Scene, Shadow, Size, Surface, Underline,
};
use cocoa::{
base::{NO, YES},
@ -9,6 +9,9 @@ use cocoa::{
quartzcore::AutoresizingMask,
};
use collections::HashMap;
use core_foundation::base::TCFType;
use foreign_types::ForeignType;
use media::core_video::CVMetalTextureCache;
use metal::{CommandQueue, MTLPixelFormat, MTLResourceOptions, NSRange};
use objc::{self, msg_send, sel, sel_impl};
use smallvec::SmallVec;
@ -27,9 +30,11 @@ pub(crate) struct MetalRenderer {
underlines_pipeline_state: metal::RenderPipelineState,
monochrome_sprites_pipeline_state: metal::RenderPipelineState,
polychrome_sprites_pipeline_state: metal::RenderPipelineState,
surfaces_pipeline_state: metal::RenderPipelineState,
unit_vertices: metal::Buffer,
instances: metal::Buffer,
sprite_atlas: Arc<MetalAtlas>,
core_video_texture_cache: CVMetalTextureCache,
}
impl MetalRenderer {
@ -143,6 +148,14 @@ impl MetalRenderer {
"polychrome_sprite_fragment",
MTLPixelFormat::BGRA8Unorm,
);
let surfaces_pipeline_state = build_pipeline_state(
&device,
&library,
"surfaces",
"surface_vertex",
"surface_fragment",
MTLPixelFormat::BGRA8Unorm,
);
let command_queue = device.new_command_queue();
let sprite_atlas = Arc::new(MetalAtlas::new(device.clone()));
@ -157,9 +170,11 @@ impl MetalRenderer {
underlines_pipeline_state,
monochrome_sprites_pipeline_state,
polychrome_sprites_pipeline_state,
surfaces_pipeline_state,
unit_vertices,
instances,
sprite_atlas,
core_video_texture_cache: CVMetalTextureCache::new(device.as_ptr()).unwrap(),
}
}
@ -268,6 +283,14 @@ impl MetalRenderer {
command_encoder,
);
}
PrimitiveBatch::Surfaces(surfaces) => {
self.draw_surfaces(
surfaces,
&mut instance_offset,
viewport_size,
command_encoder,
);
}
}
}
@ -793,6 +816,102 @@ impl MetalRenderer {
);
*offset = next_offset;
}
fn draw_surfaces(
&mut self,
surfaces: &[Surface],
offset: &mut usize,
viewport_size: Size<DevicePixels>,
command_encoder: &metal::RenderCommandEncoderRef,
) {
command_encoder.set_render_pipeline_state(&self.surfaces_pipeline_state);
command_encoder.set_vertex_buffer(
SurfaceInputIndex::Vertices as u64,
Some(&self.unit_vertices),
0,
);
command_encoder.set_vertex_bytes(
SurfaceInputIndex::ViewportSize as u64,
mem::size_of_val(&viewport_size) as u64,
&viewport_size as *const Size<DevicePixels> as *const _,
);
for surface in surfaces {
let texture_size = size(
DevicePixels::from(surface.image_buffer.width() as i32),
DevicePixels::from(surface.image_buffer.height() as i32),
);
assert_eq!(
surface.image_buffer.pixel_format_type(),
media::core_video::kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
);
let y_texture = self
.core_video_texture_cache
.create_texture_from_image(
surface.image_buffer.as_concrete_TypeRef(),
ptr::null(),
MTLPixelFormat::R8Unorm,
surface.image_buffer.plane_width(0),
surface.image_buffer.plane_height(0),
0,
)
.unwrap();
let cb_cr_texture = self
.core_video_texture_cache
.create_texture_from_image(
surface.image_buffer.as_concrete_TypeRef(),
ptr::null(),
MTLPixelFormat::RG8Unorm,
surface.image_buffer.plane_width(1),
surface.image_buffer.plane_height(1),
1,
)
.unwrap();
align_offset(offset);
let next_offset = *offset + mem::size_of::<Surface>();
assert!(
next_offset <= INSTANCE_BUFFER_SIZE,
"instance buffer exhausted"
);
command_encoder.set_vertex_buffer(
SurfaceInputIndex::Surfaces as u64,
Some(&self.instances),
*offset as u64,
);
command_encoder.set_vertex_bytes(
SurfaceInputIndex::TextureSize as u64,
mem::size_of_val(&texture_size) as u64,
&texture_size as *const Size<DevicePixels> as *const _,
);
command_encoder.set_fragment_texture(
SurfaceInputIndex::YTexture as u64,
Some(y_texture.as_texture_ref()),
);
command_encoder.set_fragment_texture(
SurfaceInputIndex::CbCrTexture as u64,
Some(cb_cr_texture.as_texture_ref()),
);
unsafe {
let buffer_contents =
(self.instances.contents() as *mut u8).add(*offset) as *mut SurfaceBounds;
ptr::write(
buffer_contents,
SurfaceBounds {
bounds: surface.bounds,
content_mask: surface.content_mask.clone(),
},
);
}
command_encoder.draw_primitives(metal::MTLPrimitiveType::Triangle, 0, 6);
*offset = next_offset;
}
}
}
fn build_pipeline_state(
@ -898,6 +1017,16 @@ enum SpriteInputIndex {
AtlasTexture = 4,
}
#[repr(C)]
enum SurfaceInputIndex {
Vertices = 0,
Surfaces = 1,
ViewportSize = 2,
TextureSize = 3,
YTexture = 4,
CbCrTexture = 5,
}
#[repr(C)]
enum PathRasterizationInputIndex {
Vertices = 0,
@ -911,3 +1040,10 @@ pub struct PathSprite {
pub color: Hsla,
pub tile: AtlasTile,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[repr(C)]
pub struct SurfaceBounds {
pub bounds: Bounds<ScaledPixels>,
pub content_mask: ContentMask<ScaledPixels>,
}

52
crates/gpui2/src/platform/mac/shaders.metal
View file

@ -469,6 +469,58 @@ fragment float4 path_sprite_fragment(
return color;
}
struct SurfaceVertexOutput {
float4 position [[position]];
float2 texture_position;
float clip_distance [[clip_distance]][4];
};
struct SurfaceFragmentInput {
float4 position [[position]];
float2 texture_position;
};
vertex SurfaceVertexOutput surface_vertex(
uint unit_vertex_id [[vertex_id]], uint surface_id [[instance_id]],
constant float2 *unit_vertices [[buffer(SurfaceInputIndex_Vertices)]],
constant SurfaceBounds *surfaces [[buffer(SurfaceInputIndex_Surfaces)]],
constant Size_DevicePixels *viewport_size
[[buffer(SurfaceInputIndex_ViewportSize)]],
constant Size_DevicePixels *texture_size
[[buffer(SurfaceInputIndex_TextureSize)]]) {
float2 unit_vertex = unit_vertices[unit_vertex_id];
SurfaceBounds surface = surfaces[surface_id];
float4 device_position =
to_device_position(unit_vertex, surface.bounds, viewport_size);
float4 clip_distance = distance_from_clip_rect(unit_vertex, surface.bounds,
surface.content_mask.bounds);
// We are going to copy the whole texture, so the texture position corresponds
// to the current vertex of the unit triangle.
float2 texture_position = unit_vertex;
return SurfaceVertexOutput{
device_position,
texture_position,
{clip_distance.x, clip_distance.y, clip_distance.z, clip_distance.w}};
}
fragment float4 surface_fragment(SurfaceFragmentInput input [[stage_in]],
texture2d<float> y_texture
[[texture(SurfaceInputIndex_YTexture)]],
texture2d<float> cb_cr_texture
[[texture(SurfaceInputIndex_CbCrTexture)]]) {
constexpr sampler texture_sampler(mag_filter::linear, min_filter::linear);
const float4x4 ycbcrToRGBTransform =
float4x4(float4(+1.0000f, +1.0000f, +1.0000f, +0.0000f),
float4(+0.0000f, -0.3441f, +1.7720f, +0.0000f),
float4(+1.4020f, -0.7141f, +0.0000f, +0.0000f),
float4(-0.7010f, +0.5291f, -0.8860f, +1.0000f));
float4 ycbcr = float4(
y_texture.sample(texture_sampler, input.texture_position).r,
cb_cr_texture.sample(texture_sampler, input.texture_position).rg, 1.0);
return ycbcrToRGBTransform * ycbcr;
}
float4 hsla_to_rgba(Hsla hsla) {
float h = hsla.h * 6.0; // Now, it's an angle but scaled in [0, 6) range
float s = hsla.s;

66
crates/gpui2/src/scene.rs
View file

@ -25,6 +25,7 @@ pub(crate) struct SceneBuilder {
underlines: Vec<Underline>,
monochrome_sprites: Vec<MonochromeSprite>,
polychrome_sprites: Vec<PolychromeSprite>,
surfaces: Vec<Surface>,
}
impl Default for SceneBuilder {
@ -38,6 +39,7 @@ impl Default for SceneBuilder {
underlines: Vec::new(),
monochrome_sprites: Vec::new(),
polychrome_sprites: Vec::new(),
surfaces: Vec::new(),
}
}
}
@ -120,6 +122,7 @@ impl SceneBuilder {
(PrimitiveKind::PolychromeSprite, ix) => {
self.polychrome_sprites[ix].order = draw_order as DrawOrder
}
(PrimitiveKind::Surface, ix) => self.surfaces[ix].order = draw_order as DrawOrder,
}
}
@ -129,6 +132,7 @@ impl SceneBuilder {
self.underlines.sort_unstable();
self.monochrome_sprites.sort_unstable();
self.polychrome_sprites.sort_unstable();
self.surfaces.sort_unstable();
Scene {
shadows: mem::take(&mut self.shadows),
@ -137,6 +141,7 @@ impl SceneBuilder {
underlines: mem::take(&mut self.underlines),
monochrome_sprites: mem::take(&mut self.monochrome_sprites),
polychrome_sprites: mem::take(&mut self.polychrome_sprites),
surfaces: mem::take(&mut self.surfaces),
}
}
@ -185,6 +190,10 @@ impl SceneBuilder {
sprite.order = layer_id;
self.polychrome_sprites.push(sprite);
}
Primitive::Surface(mut surface) => {
surface.order = layer_id;
self.surfaces.push(surface);
}
}
}
}
@ -196,6 +205,7 @@ pub(crate) struct Scene {
pub underlines: Vec<Underline>,
pub monochrome_sprites: Vec<MonochromeSprite>,
pub polychrome_sprites: Vec<PolychromeSprite>,
pub surfaces: Vec<Surface>,
}
impl Scene {
@ -224,6 +234,9 @@ impl Scene {
polychrome_sprites: &self.polychrome_sprites,
polychrome_sprites_start: 0,
polychrome_sprites_iter: self.polychrome_sprites.iter().peekable(),
surfaces: &self.surfaces,
surfaces_start: 0,
surfaces_iter: self.surfaces.iter().peekable(),
}
}
}
@ -247,6 +260,9 @@ struct BatchIterator<'a> {
polychrome_sprites: &'a [PolychromeSprite],
polychrome_sprites_start: usize,
polychrome_sprites_iter: Peekable<slice::Iter<'a, PolychromeSprite>>,
surfaces: &'a [Surface],
surfaces_start: usize,
surfaces_iter: Peekable<slice::Iter<'a, Surface>>,
}
impl<'a> Iterator for BatchIterator<'a> {
@ -272,6 +288,10 @@ impl<'a> Iterator for BatchIterator<'a> {
self.polychrome_sprites_iter.peek().map(|s| s.order),
PrimitiveKind::PolychromeSprite,
),
(
self.surfaces_iter.peek().map(|s| s.order),
PrimitiveKind::Surface,
),
];
orders_and_kinds.sort_by_key(|(order, kind)| (order.unwrap_or(u32::MAX), *kind));
@ -378,6 +398,21 @@ impl<'a> Iterator for BatchIterator<'a> {
sprites: &self.polychrome_sprites[sprites_start..sprites_end],
})
}
PrimitiveKind::Surface => {
let surfaces_start = self.surfaces_start;
let mut surfaces_end = surfaces_start;
while self
.surfaces_iter
.next_if(|surface| surface.order <= max_order)
.is_some()
{
surfaces_end += 1;
}
self.surfaces_start = surfaces_end;
Some(PrimitiveBatch::Surfaces(
&self.surfaces[surfaces_start..surfaces_end],
))
}
}
}
}
@ -391,6 +426,7 @@ pub enum PrimitiveKind {
Underline,
MonochromeSprite,
PolychromeSprite,
Surface,
}
pub enum Primitive {
@ -400,6 +436,7 @@ pub enum Primitive {
Underline(Underline),
MonochromeSprite(MonochromeSprite),
PolychromeSprite(PolychromeSprite),
Surface(Surface),
}
impl Primitive {
@ -411,6 +448,7 @@ impl Primitive {
Primitive::Underline(underline) => &underline.bounds,
Primitive::MonochromeSprite(sprite) => &sprite.bounds,
Primitive::PolychromeSprite(sprite) => &sprite.bounds,
Primitive::Surface(surface) => &surface.bounds,
}
}
@ -422,6 +460,7 @@ impl Primitive {
Primitive::Underline(underline) => &underline.content_mask,
Primitive::MonochromeSprite(sprite) => &sprite.content_mask,
Primitive::PolychromeSprite(sprite) => &sprite.content_mask,
Primitive::Surface(surface) => &surface.content_mask,
}
}
}
@ -440,6 +479,7 @@ pub(crate) enum PrimitiveBatch<'a> {
texture_id: AtlasTextureId,
sprites: &'a [PolychromeSprite],
},
Surfaces(&'a [Surface]),
}
#[derive(Default, Debug, Clone, Eq, PartialEq)]
@ -593,6 +633,32 @@ impl From<PolychromeSprite> for Primitive {
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Surface {
pub order: u32,
pub bounds: Bounds<ScaledPixels>,
pub content_mask: ContentMask<ScaledPixels>,
pub image_buffer: media::core_video::CVImageBuffer,
}
impl Ord for Surface {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.order.cmp(&other.order)
}
}
impl PartialOrd for Surface {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl From<Surface> for Primitive {
fn from(surface: Surface) -> Self {
Primitive::Surface(surface)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub(crate) struct PathId(pub(crate) usize);

22
crates/gpui2/src/window.rs
View file

@ -8,8 +8,8 @@ use crate::{
MouseUpEvent, Path, Pixels, PlatformAtlas, PlatformDisplay, PlatformInputHandler,
PlatformWindow, Point, PolychromeSprite, PromptLevel, Quad, Render, RenderGlyphParams,
RenderImageParams, RenderSvgParams, ScaledPixels, SceneBuilder, Shadow, SharedString, Size,
Style, SubscriberSet, Subscription, TaffyLayoutEngine, Task, Underline, UnderlineStyle, View,
VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS,
Style, SubscriberSet, Subscription, Surface, TaffyLayoutEngine, Task, Underline,
UnderlineStyle, View, VisualContext, WeakView, WindowBounds, WindowOptions, SUBPIXEL_VARIANTS,
};
use anyhow::{anyhow, Context as _, Result};
use collections::HashMap;
@ -18,6 +18,7 @@ use futures::{
channel::{mpsc, oneshot},
StreamExt,
};
use media::core_video::CVImageBuffer;
use parking_lot::RwLock;
use slotmap::SlotMap;
use smallvec::SmallVec;
@ -1090,6 +1091,23 @@ impl<'a> WindowContext<'a> {
Ok(())
}
/// Paint a surface into the scene for the current frame at the current z-index.
pub fn paint_surface(&mut self, bounds: Bounds<Pixels>, image_buffer: CVImageBuffer) {
let scale_factor = self.scale_factor();
let bounds = bounds.scale(scale_factor);
let content_mask = self.content_mask().scale(scale_factor);
let window = &mut *self.window;
window.current_frame.scene_builder.insert(
&window.current_frame.z_index_stack,
Surface {
order: 0,
bounds,
content_mask,
image_buffer,
},
);
}
/// Draw pixels to the display for this window based on the contents of its scene.
pub(crate) fn draw(&mut self) {
let root_view = self.window.root_view.take().unwrap();