mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-12-23 10:30:40 +00:00
Code Issues Projects Releases Wiki Activity

gtk4: Improve handling of RGBA GL textures in GTK

Browse source 
GTK 4.14 comes with a new GL renderer that does not support GL shader
nodes anymore, so the conversion from non-premultiplied alpha to
premultiplied alpha has to happen differently.

For GTK 4.14 or newer we use the correct format directly when building the
texture, but only if a GLES3+ context is used. In that case the NGL renderer is
used by GTK, which supports non-premultiplied formats correctly and fast.

For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
self-mask to pre-multiply the alpha.

For GTK before 4.10, we use a GL shader and hope that it works.

Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/488

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1452>
This commit is contained in:
Sebastian Dröge 2024-02-07 18:12:29 +02:00
parent 09e9c047df
commit 803550111a
6 changed files with 166 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
video/gtk4/Cargo.toml
View file

@ -52,6 +52,9 @@ x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"]
winegl = ["gdk-win32/egl", "gst-gl-egl"]
capi = []
doc = ["gst/v1_18"]
gtk_v4_10 = ["gtk/v4_10"]
gtk_v4_12 = ["gtk/v4_12", "gtk_v4_10"]
gtk_v4_14 = ["gtk/v4_14", "gtk_v4_12"]
[package.metadata.capi]
min_version = "0.9.21"

7
video/gtk4/src/lib.rs
View file

@ -22,6 +22,13 @@ mod utils;
pub use sink::PaintableSink;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
#[cfg(not(feature = "gtk_v4_10"))]
{
if gtk::micro_version() >= 13 {
gst::warning!(sink::imp::CAT, obj: plugin, "GTK 4.13 or newer detected but plugin not compiled with support for this version. Rendering of video frames with alpha will likely be wrong");
}
}
sink::register(plugin)
}

104
video/gtk4/src/sink/frame.rs
View file

@ -94,6 +94,18 @@ impl AsRef<[u8]> for FrameWrapper {
}
}
fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat {
match f {
gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
_ => unreachable!(),
}
}
fn video_frame_to_memory_texture(
frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
@ -109,15 +121,7 @@ fn video_frame_to_memory_texture(
return (texture.clone(), pixel_aspect_ratio);
}
let format = match frame.format() {
gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
_ => unreachable!(),
};
let format = video_format_to_memory_format(frame.format());
let width = frame.width();
let height = frame.height();
let rowstride = frame.plane_stride()[0] as usize;
@ -143,7 +147,7 @@ fn video_frame_to_gl_texture(
cached_textures: &mut HashMap<usize, gdk::Texture>,
used_textures: &mut HashSet<usize>,
gdk_context: &gdk::GLContext,
wrapped_context: &gst_gl::GLContext,
#[allow(unused)] wrapped_context: &gst_gl::GLContext,
) -> (gdk::Texture, f64) {
let texture_id = frame.texture_id(0).expect("Invalid texture id") as usize;
@ -159,19 +163,77 @@ fn video_frame_to_gl_texture(
let height = frame.height();
let sync_meta = frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
sync_meta.wait(wrapped_context);
let texture = unsafe {
gdk::GLTexture::with_release_func(
gdk_context,
texture_id as u32,
width as i32,
height as i32,
move || {
// Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
drop(frame);
},
)
#[cfg(feature = "gtk_v4_12")]
{
let format = {
let format = video_format_to_memory_format(frame.format());
#[cfg(feature = "gtk_v4_14")]
{
use gtk::prelude::*;
if gdk_context.api() != gdk::GLAPI::GLES || gdk_context.version().0 < 3 {
// Map alpha formats to the pre-multiplied versions because we pre-multiply
// ourselves if not GLES3 with the new GL renderer is used as the GTK GL
// backend does not natively support non-premultiplied formats.
match format {
gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied,
gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied,
gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format,
_ => unreachable!(),
}
} else {
format
}
}
#[cfg(not(feature = "gtk_v4_14"))]
{
// Map alpha formats to the pre-multiplied versions because we pre-multiply
// ourselves in pre-4.14 versions as the GTK GL backend does not natively
// support non-premultiplied formats
match format {
gdk::MemoryFormat::B8g8r8a8 => gdk::MemoryFormat::B8g8r8a8Premultiplied,
gdk::MemoryFormat::A8r8g8b8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
gdk::MemoryFormat::R8g8b8a8 => gdk::MemoryFormat::R8g8b8a8Premultiplied,
gdk::MemoryFormat::A8b8g8r8 => gdk::MemoryFormat::A8r8g8b8Premultiplied,
gdk::MemoryFormat::R8g8b8 | gdk::MemoryFormat::B8g8r8 => format,
_ => unreachable!(),
}
}
};
let sync_point = (*sync_meta.as_ptr()).data;
gdk::GLTextureBuilder::new()
.set_context(Some(gdk_context))
.set_id(texture_id as u32)
.set_width(width as i32)
.set_height(height as i32)
.set_format(format)
.set_sync(Some(sync_point))
.build_with_release_func(move || {
// Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
drop(frame);
})
}
#[cfg(not(feature = "gtk_v4_12"))]
{
sync_meta.wait(wrapped_context);
gdk::GLTexture::with_release_func(
gdk_context,
texture_id as u32,
width as i32,
height as i32,
move || {
// Unmap and drop the GStreamer GL texture once GTK is done with it and not earlier
drop(frame);
},
)
}
.upcast::<gdk::Texture>()
};

2
video/gtk4/src/sink/imp.rs
View file

@ -48,7 +48,7 @@ enum GLContext {
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
static GL_CONTEXT: Mutex<GLContext> = Mutex::new(GLContext::Uninitialized);
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
pub(crate) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"gtk4paintablesink",
gst::DebugColorFlags::empty(),

2
video/gtk4/src/sink/mod.rs
View file

@ -13,7 +13,7 @@ use gtk::glib;
use gtk::glib::prelude::*;
mod frame;
mod imp;
pub(super) mod imp;
mod paintable;
enum SinkEvent {

82
video/gtk4/src/sink/paintable/imp.rs
View file

@ -33,6 +33,7 @@ pub struct Paintable {
paintables: RefCell<Vec<Texture>>,
cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
gl_context: RefCell<Option<gdk::GLContext>>,
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader,
}
@ -42,6 +43,7 @@ impl Default for Paintable {
paintables: Default::default(),
cached_textures: Default::default(),
gl_context: Default::default(),
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
"premult.glsl"
))),
@ -167,20 +169,78 @@ impl PaintableImpl for Paintable {
let bounds = graphene::Rect::new(*x, *y, *paintable_width, *paintable_height);
// Only premultiply GL textures that expect to be in premultiplied RGBA format.
let do_premult = texture.is::<gdk::GLTexture>() && *has_alpha;
if do_premult {
snapshot.push_gl_shader(
&self.premult_shader,
&bounds,
gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
);
//
// For GTK 4.14 or newer we use the correct format directly when building the
// texture, but only if a GLES3+ context is used. In that case the NGL renderer is
// used by GTK, which supports non-premultiplied formats correctly and fast.
//
// For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
// self-mask to pre-multiply the alpha.
//
// For GTK before 4.10, we use a GL shader and hope that it works.
#[cfg(feature = "gtk_v4_10")]
{
let context_requires_premult = {
#[cfg(feature = "gtk_v4_14")]
{
self.gl_context.borrow().as_ref().map_or(false, |context| {
context.api() != gdk::GLAPI::GLES || context.version().0 < 3
})
}
#[cfg(not(feature = "gtk_v4_14"))]
{
true
}
};
let do_premult =
context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
if do_premult {
snapshot.push_mask(gsk::MaskMode::Alpha);
snapshot.append_texture(texture, &bounds);
snapshot.pop(); // pop mask
// color matrix to set alpha of the source to 1.0 as it was
// already applied via the mask just above.
snapshot.push_color_matrix(
&graphene::Matrix::from_float({
[
1.0, 0.0, 0.0, 0.0, //
0.0, 1.0, 0.0, 0.0, //
0.0, 0.0, 1.0, 0.0, //
0.0, 0.0, 0.0, 0.0,
]
}),
&graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
);
}
snapshot.append_texture(texture, &bounds);
if do_premult {
snapshot.pop(); // pop color matrix
snapshot.pop(); // pop mask 2
}
}
#[cfg(not(feature = "gtk_v4_10"))]
{
let do_premult =
texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
if do_premult {
snapshot.push_gl_shader(
&self.premult_shader,
&bounds,
gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
);
}
snapshot.append_texture(texture, &bounds);
snapshot.append_texture(texture, &bounds);
if do_premult {
snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
snapshot.pop(); // pop shader
if do_premult {
snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
snapshot.pop(); // pop shader
}
}
snapshot.pop(); // pop opacity