From c92462b2406cacb82efab0d8f67140b9f574753b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 20 Apr 2024 23:19:20 +0300 Subject: [PATCH 1/6] gtk4: Implement support for directly importing dmabufs Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/441 Part-of: --- Cargo.lock | 25 ++ Cargo.toml | 1 + docs/plugins/gst_plugins_cache.json | 2 +- video/gtk4/Cargo.toml | 2 + video/gtk4/examples/gtksink.rs | 27 +- video/gtk4/src/sink/frame.rs | 357 ++++++++++++++++++++------- video/gtk4/src/sink/imp.rs | 196 ++++++++++++--- video/gtk4/src/sink/paintable/imp.rs | 68 ++--- video/gtk4/src/sink/paintable/mod.rs | 4 +- 9 files changed, 519 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b41c3e7..fb42c823 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2454,6 +2454,7 @@ dependencies = [ "gdk4-x11", "gst-plugin-version-helper", "gstreamer", + "gstreamer-allocators", "gstreamer-base", "gstreamer-gl", "gstreamer-gl-egl", @@ -3059,6 +3060,30 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gstreamer-allocators" +version = "0.23.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-allocators-sys", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-allocators-sys" +version = "0.23.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-app" version = "0.23.0" diff --git a/Cargo.toml b/Cargo.toml index ce7dfae8..9d803a18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,6 +134,7 @@ gdk-wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4- gdk-x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"} gdk-win32 = { package = "gdk4-win32", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"} gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } +gst-allocators = { package = "gstreamer-allocators", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-app = { package = "gstreamer-app", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index c5f304c4..ad9844ab 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -2476,7 +2476,7 @@ "long-name": "GTK 4 Paintable Sink", "pad-templates": { "sink": { - "caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "caps": "video/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", "direction": "sink", "presence": "always" } diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index fce75fd9..ce011d65 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -17,6 +17,7 @@ gst = { workspace = true, features = ["v1_16"] } gst-base.workspace = true gst-video.workspace = true gst-gl = { workspace = true, features = ["v1_16"], optional = true } +gst-allocators = { workspace = true, features = ["v1_24"], optional = true } gst-gl-wayland = { workspace = true, features = ["v1_16"], optional = true } gst-gl-x11 = { workspace = true, features = ["v1_16"], optional = true } @@ -50,6 +51,7 @@ wayland = ["gtk/v4_6", "gdk-wayland", "gst-gl", "gst-gl-wayland"] x11glx = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-x11"] x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"] winegl = ["gdk-win32/egl", "gst-gl-egl"] +dmabuf = ["gst-allocators", "wayland", "gtk_v4_14", "gst-video/v1_24"] capi = [] doc = ["gst/v1_18"] gtk_v4_10 = ["gtk/v4_10"] diff --git a/video/gtk4/examples/gtksink.rs b/video/gtk4/examples/gtksink.rs index b431b5c5..abda23c6 100644 --- a/video/gtk4/examples/gtksink.rs +++ b/video/gtk4/examples/gtksink.rs @@ -6,13 +6,6 @@ use gtk::{gdk, gio, glib}; use std::cell::RefCell; fn create_ui(app: >k::Application) { - let window = gtk::ApplicationWindow::new(app); - window.set_default_size(640, 480); - - let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0); - let picture = gtk::Picture::new(); - let label = gtk::Label::new(Some("Position: 00:00:00")); - let pipeline = gst::Pipeline::new(); let overlay = gst::ElementFactory::make("clockoverlay") @@ -64,8 +57,26 @@ fn create_ui(app: >k::Application) { src.link_filtered(&overlay, &caps).unwrap(); overlay.link(&sink).unwrap(); + let window = gtk::ApplicationWindow::new(app); + window.set_default_size(640, 480); + + let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0); + + let picture = gtk::Picture::new(); picture.set_paintable(Some(&paintable)); - vbox.append(&picture); + + #[cfg(feature = "gtk_v4_14")] + { + let offload = gtk::GraphicsOffload::new(Some(&picture)); + offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled); + vbox.append(&offload); + } + #[cfg(not(feature = "gtk_v4_14"))] + { + vbox.append(&picture); + } + + let label = gtk::Label::new(Some("Position: 00:00:00")); vbox.append(&label); window.set_child(Some(&vbox)); diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index aaa7e224..f9f1c61b 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -14,7 +14,61 @@ use gst_video::prelude::*; #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] use gst_gl::prelude::*; use gtk::{gdk, glib}; -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + ops, +}; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum VideoInfo { + VideoInfo(gst_video::VideoInfo), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaDrm(gst_video::VideoInfoDmaDrm), +} + +impl From for VideoInfo { + fn from(v: gst_video::VideoInfo) -> Self { + VideoInfo::VideoInfo(v) + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl From for VideoInfo { + fn from(v: gst_video::VideoInfoDmaDrm) -> Self { + VideoInfo::DmaDrm(v) + } +} + +impl ops::Deref for VideoInfo { + type Target = gst_video::VideoInfo; + + fn deref(&self) -> &Self::Target { + match self { + VideoInfo::VideoInfo(info) => info, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + VideoInfo::DmaDrm(info) => info, + } + } +} + +impl VideoInfo { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + fn dma_drm(&self) -> Option<&gst_video::VideoInfoDmaDrm> { + match self { + VideoInfo::VideoInfo(..) => None, + VideoInfo::DmaDrm(info) => Some(info), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum TextureCacheId { + Memory(usize), + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] + GL(usize), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaBuf([i32; 4]), +} #[derive(Debug)] enum MappedFrame { @@ -24,6 +78,17 @@ enum MappedFrame { frame: gst_gl::GLVideoFrame, wrapped_context: gst_gl::GLContext, }, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaBuf { + buffer: gst::Buffer, + info: gst_video::VideoInfoDmaDrm, + n_planes: u32, + fds: [i32; 4], + offsets: [usize; 4], + strides: [usize; 4], + width: u32, + height: u32, + }, } impl MappedFrame { @@ -32,6 +97,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.buffer(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.buffer(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { buffer, .. } => buffer, } } @@ -40,6 +107,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.width(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.width(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.width(), } } @@ -48,6 +117,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.height(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.height(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.height(), } } @@ -56,6 +127,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.format_info(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.format_info(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.format_info(), } } } @@ -108,16 +181,16 @@ fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat fn video_frame_to_memory_texture( frame: gst_video::VideoFrame, - cached_textures: &mut HashMap, - used_textures: &mut HashSet, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, ) -> (gdk::Texture, f64) { - let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize; + let ptr = frame.plane_data(0).unwrap().as_ptr() as usize; let pixel_aspect_ratio = (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64); - if let Some(texture) = cached_textures.get(&texture_id) { - used_textures.insert(texture_id); + if let Some(texture) = cached_textures.get(&TextureCacheId::Memory(ptr)) { + used_textures.insert(TextureCacheId::Memory(ptr)); return (texture.clone(), pixel_aspect_ratio); } @@ -135,8 +208,8 @@ fn video_frame_to_memory_texture( ) .upcast::(); - cached_textures.insert(texture_id, texture.clone()); - used_textures.insert(texture_id); + cached_textures.insert(TextureCacheId::Memory(ptr), texture.clone()); + used_textures.insert(TextureCacheId::Memory(ptr)); (texture, pixel_aspect_ratio) } @@ -144,8 +217,8 @@ fn video_frame_to_memory_texture( #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] fn video_frame_to_gl_texture( frame: gst_gl::GLVideoFrame, - cached_textures: &mut HashMap, - used_textures: &mut HashSet, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, gdk_context: &gdk::GLContext, #[allow(unused)] wrapped_context: &gst_gl::GLContext, ) -> (gdk::Texture, f64) { @@ -154,8 +227,8 @@ fn video_frame_to_gl_texture( let pixel_aspect_ratio = (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64); - if let Some(texture) = cached_textures.get(&(texture_id)) { - used_textures.insert(texture_id); + if let Some(texture) = cached_textures.get(&TextureCacheId::GL(texture_id)) { + used_textures.insert(TextureCacheId::GL(texture_id)); return (texture.clone(), pixel_aspect_ratio); } @@ -237,18 +310,64 @@ fn video_frame_to_gl_texture( .upcast::() }; - cached_textures.insert(texture_id, texture.clone()); - used_textures.insert(texture_id); + cached_textures.insert(TextureCacheId::GL(texture_id), texture.clone()); + used_textures.insert(TextureCacheId::GL(texture_id)); (texture, pixel_aspect_ratio) } +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +#[allow(clippy::too_many_arguments)] +fn video_frame_to_dmabuf_texture( + buffer: gst::Buffer, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, + info: &gst_video::VideoInfoDmaDrm, + n_planes: u32, + fds: &[i32; 4], + offsets: &[usize; 4], + strides: &[usize; 4], + width: u32, + height: u32, +) -> Result<(gdk::Texture, f64), glib::Error> { + let pixel_aspect_ratio = (info.par().numer() as f64) / (info.par().denom() as f64); + + if let Some(texture) = cached_textures.get(&TextureCacheId::DmaBuf(*fds)) { + used_textures.insert(TextureCacheId::DmaBuf(*fds)); + return Ok((texture.clone(), pixel_aspect_ratio)); + } + + let builder = gdk::DmabufTextureBuilder::new(); + builder.set_display(&gdk::Display::default().unwrap()); + builder.set_fourcc(info.fourcc()); + builder.set_modifier(info.modifier()); + builder.set_width(width); + builder.set_height(height); + builder.set_n_planes(n_planes); + for plane in 0..(n_planes as usize) { + builder.set_fd(plane as u32, fds[plane]); + builder.set_offset(plane as u32, offsets[plane] as u32); + builder.set_stride(plane as u32, strides[plane] as u32); + } + + let texture = unsafe { + builder.build_with_release_func(move || { + drop(buffer); + })? + }; + + cached_textures.insert(TextureCacheId::DmaBuf(*fds), texture.clone()); + used_textures.insert(TextureCacheId::DmaBuf(*fds)); + + Ok((texture, pixel_aspect_ratio)) +} + impl Frame { pub(crate) fn into_textures( self, #[allow(unused_variables)] gdk_context: Option<&gdk::GLContext>, - cached_textures: &mut HashMap, - ) -> Vec { + cached_textures: &mut HashMap, + ) -> Result, glib::Error> { let mut textures = Vec::with_capacity(1 + self.overlays.len()); let mut used_textures = HashSet::with_capacity(1 + self.overlays.len()); @@ -278,6 +397,28 @@ impl Frame { &wrapped_context, ) } + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { + buffer, + info, + n_planes, + fds, + offsets, + strides, + width, + height, + } => video_frame_to_dmabuf_texture( + buffer, + cached_textures, + &mut used_textures, + &info, + n_planes, + &fds, + &offsets, + &strides, + width, + height, + )?, }; textures.push(Texture { @@ -309,14 +450,14 @@ impl Frame { // Remove textures that were not used this time cached_textures.retain(|id, _| used_textures.contains(id)); - textures + Ok(textures) } } impl Frame { pub(crate) fn new( buffer: &gst::Buffer, - info: &gst_video::VideoInfo, + info: &VideoInfo, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option< &gst_gl::GLContext, >, @@ -327,77 +468,125 @@ impl Frame { // Empty buffers get filtered out in show_frame debug_assert!(buffer.n_memory() > 0); - let mut frame; + #[allow(unused_mut)] + let mut frame = None; - #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))] + #[cfg(all(target_os = "linux", feature = "dmabuf"))] { - frame = Self { - frame: MappedFrame::SysMem( + // Check we received a buffer with dmabuf memory and if so do some checks before + // passing it onwards + if frame.is_none() + && buffer + .peek_memory(0) + .is_memory_type::() + { + if let Some((vmeta, info)) = + Option::zip(buffer.meta::(), info.dma_drm()) + { + let mut fds = [-1i32; 4]; + let mut offsets = [0; 4]; + let mut strides = [0; 4]; + let n_planes = vmeta.n_planes() as usize; + + let vmeta_offsets = vmeta.offset(); + let vmeta_strides = vmeta.stride(); + + for plane in 0..n_planes { + let Some((range, skip)) = + buffer.find_memory(vmeta_offsets[plane]..(vmeta_offsets[plane] + 1)) + else { + break; + }; + + let mem = buffer.peek_memory(range.start); + let Some(mem) = mem.downcast_memory_ref::() + else { + break; + }; + + let fd = mem.fd(); + fds[plane] = fd; + offsets[plane] = mem.offset() + skip; + strides[plane] = vmeta_strides[plane] as usize; + } + + // All fds valid? + if fds[0..n_planes].iter().all(|fd| *fd != -1) { + frame = Some(MappedFrame::DmaBuf { + buffer: buffer.clone(), + info: info.clone(), + n_planes: n_planes as u32, + fds, + offsets, + strides, + width: vmeta.width(), + height: vmeta.height(), + }); + } + } + } + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] + { + if frame.is_none() { + // Check we received a buffer with GL memory and if the context of that memory + // can share with the wrapped context around the GDK GL context. + // + // If not it has to be uploaded to the GPU. + let memory_ctx = buffer + .peek_memory(0) + .downcast_memory_ref::() + .and_then(|m| { + let ctx = m.context(); + if wrapped_context + .map_or(false, |wrapped_context| wrapped_context.can_share(ctx)) + { + Some(ctx) + } else { + None + } + }); + + if let Some(memory_ctx) = memory_ctx { + // If there is no GLSyncMeta yet then we need to add one here now, which requires + // obtaining a writable buffer. + let mapped_frame = if buffer.meta::().is_some() { + gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)? + } else { + let mut buffer = buffer.clone(); + { + let buffer = buffer.make_mut(); + gst_gl::GLSyncMeta::add(buffer, memory_ctx); + } + gst_gl::GLVideoFrame::from_buffer_readable(buffer, info) + .map_err(|_| gst::FlowError::Error)? + }; + + // Now that it's guaranteed that there is a sync meta and the frame is mapped, set + // a sync point so we can ensure that the texture is ready later when making use of + // it as gdk::GLTexture. + let meta = mapped_frame.buffer().meta::().unwrap(); + meta.set_sync_point(memory_ctx); + + frame = Some(MappedFrame::GL { + frame: mapped_frame, + wrapped_context: wrapped_context.unwrap().clone(), + }); + } + } + } + } + + let mut frame = Self { + frame: match frame { + Some(frame) => frame, + None => MappedFrame::SysMem( gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) .map_err(|_| gst::FlowError::Error)?, ), - overlays: vec![], - }; - } - #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] - { - // Check we received a buffer with GL memory and if the context of that memory - // can share with the wrapped context around the GDK GL context. - // - // If not it has to be uploaded to the GPU. - let memory_ctx = buffer - .peek_memory(0) - .downcast_memory_ref::() - .and_then(|m| { - let ctx = m.context(); - if wrapped_context - .map_or(false, |wrapped_context| wrapped_context.can_share(ctx)) - { - Some(ctx) - } else { - None - } - }); - - if let Some(memory_ctx) = memory_ctx { - // If there is no GLSyncMeta yet then we need to add one here now, which requires - // obtaining a writable buffer. - let mapped_frame = if buffer.meta::().is_some() { - gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)? - } else { - let mut buffer = buffer.clone(); - { - let buffer = buffer.make_mut(); - gst_gl::GLSyncMeta::add(buffer, memory_ctx); - } - gst_gl::GLVideoFrame::from_buffer_readable(buffer, info) - .map_err(|_| gst::FlowError::Error)? - }; - - // Now that it's guaranteed that there is a sync meta and the frame is mapped, set - // a sync point so we can ensure that the texture is ready later when making use of - // it as gdk::GLTexture. - let meta = mapped_frame.buffer().meta::().unwrap(); - meta.set_sync_point(memory_ctx); - - frame = Self { - frame: MappedFrame::GL { - frame: mapped_frame, - wrapped_context: wrapped_context.unwrap().clone(), - }, - overlays: vec![], - }; - } else { - frame = Self { - frame: MappedFrame::SysMem( - gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)?, - ), - overlays: vec![], - }; - } - } + }, + overlays: vec![], + }; frame.overlays = frame .frame diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index dcad8d42..eff4b2ec 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -1,7 +1,7 @@ // // Copyright (C) 2021 Bilal Elmoussaoui // Copyright (C) 2021 Jordan Petridis -// Copyright (C) 2021 Sebastian Dröge +// Copyright (C) 2021-2024 Sebastian Dröge // // This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. // If a copy of the MPL was not distributed with this file, You can obtain one at @@ -62,7 +62,7 @@ pub(crate) static CAT: Lazy = Lazy::new(|| { pub struct PaintableSink { paintable: Mutex>>, window: Mutex>>, - info: Mutex>, + info: Mutex>, sender: Mutex>>, pending_frame: Mutex>, cached_caps: Mutex>, @@ -163,53 +163,99 @@ impl ElementImpl for PaintableSink { { let caps = caps.get_mut().unwrap(); + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + for features in [ + [ + gst_allocators::CAPS_FEATURE_MEMORY_DMABUF, + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + ] + .as_slice(), + [gst_allocators::CAPS_FEATURE_MEMORY_DMABUF].as_slice(), + ] { + let c = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::DmaDrm) + .features(features.iter().copied()) + .build(); + caps.append(c); + } + } + for features in [ - None, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] Some(gst::CapsFeatures::new([ - "memory:GLMemory", - "meta:GstVideoOverlayComposition", + gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY, + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, ])), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] - Some(gst::CapsFeatures::new(["memory:GLMemory"])), + Some(gst::CapsFeatures::new([ + gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY, + ])), Some(gst::CapsFeatures::new([ "memory:SystemMemory", - "meta:GstVideoOverlayComposition", + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, ])), - Some(gst::CapsFeatures::new(["meta:GstVideoOverlayComposition"])), + Some(gst::CapsFeatures::new([ + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + ])), + None, ] { - const GL_FORMATS: &[gst_video::VideoFormat] = - &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb]; - const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[ - gst_video::VideoFormat::Bgra, - gst_video::VideoFormat::Argb, - gst_video::VideoFormat::Rgba, - gst_video::VideoFormat::Abgr, - gst_video::VideoFormat::Rgb, - gst_video::VideoFormat::Bgr, - ]; - - let formats = if features - .as_ref() - .is_some_and(|features| features.contains("memory:GLMemory")) + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] { - GL_FORMATS - } else { - NON_GL_FORMATS - }; + const GL_FORMATS: &[gst_video::VideoFormat] = + &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb]; + const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[ + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Argb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Abgr, + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Bgr, + ]; - let mut c = gst_video::video_make_raw_caps(formats).build(); + let formats = if features.as_ref().is_some_and(|features| { + features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) + }) { + GL_FORMATS + } else { + NON_GL_FORMATS + }; - if let Some(features) = features { - let c = c.get_mut().unwrap(); + let mut c = gst_video::video_make_raw_caps(formats).build(); - if features.contains("memory:GLMemory") { - c.set("texture-target", "2D") + if let Some(features) = features { + let c = c.get_mut().unwrap(); + + if features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) { + c.set("texture-target", "2D") + } + c.set_features_simple(Some(features)); } - c.set_features_simple(Some(features)); + caps.append(c); } + #[cfg(not(any( + target_os = "macos", + target_os = "windows", + feature = "gst-gl" + )))] + { + const FORMATS: &[gst_video::VideoFormat] = &[ + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Argb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Abgr, + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Bgr, + ]; - caps.append(c); + let mut c = gst_video::video_make_raw_caps(FORMATS).build(); + + if let Some(features) = features { + let c = c.get_mut().unwrap(); + c.set_features_simple(Some(features)); + } + caps.append(c); + } } } @@ -361,8 +407,21 @@ impl BaseSinkImpl for PaintableSink { fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> { gst::debug!(CAT, imp: self, "Setting caps {caps:?}"); - let video_info = gst_video::VideoInfo::from_caps(caps) - .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?; + #[allow(unused_mut)] + let mut video_info = None; + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + if let Ok(info) = gst_video::VideoInfoDmaDrm::from_caps(caps) { + video_info = Some(info.into()); + } + } + + let video_info = match video_info { + Some(info) => info, + None => gst_video::VideoInfo::from_caps(caps) + .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))? + .into(), + }; self.info.lock().unwrap().replace(video_info); @@ -516,10 +575,11 @@ impl PaintableSink { match action { SinkEvent::FrameChanged => { + let Some(frame) = self.pending_frame() else { + return glib::ControlFlow::Continue; + }; gst::trace!(CAT, imp: self, "Frame changed"); - paintable - .get_ref() - .handle_frame_changed(self.pending_frame()) + paintable.get_ref().handle_frame_changed(&self.obj(), frame); } } @@ -530,13 +590,59 @@ impl PaintableSink { #[allow(unused_mut)] let mut tmp_caps = Self::pad_templates()[0].caps().clone(); + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + let formats = utils::invoke_on_main_thread(move || { + let Some(display) = gdk::Display::default() else { + return vec![]; + }; + let dmabuf_formats = display.dmabuf_formats(); + + let mut formats = vec![]; + let n_formats = dmabuf_formats.n_formats(); + for i in 0..n_formats { + let (fourcc, modifier) = dmabuf_formats.format(i); + + if fourcc == 0 || modifier == (u64::MAX >> 8) { + continue; + } + + formats.push(gst_video::dma_drm_fourcc_to_string(fourcc, modifier)); + } + + formats + }); + + if formats.is_empty() { + // Filter out dmabufs caps from the template pads if we have no supported formats + if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) { + tmp_caps = tmp_caps + .iter_with_features() + .filter(|(_, features)| { + !features.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) + }) + .map(|(s, c)| (s.to_owned(), c.to_owned())) + .collect::(); + } + } else { + let tmp_caps = tmp_caps.make_mut(); + for (s, f) in tmp_caps.iter_with_features_mut() { + if f.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) { + s.set("drm-format", gst::List::new(&formats)); + } + } + } + } + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] { // Filter out GL caps from the template pads if we have no context if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) { tmp_caps = tmp_caps .iter_with_features() - .filter(|(_, features)| !features.contains("memory:GLMemory")) + .filter(|(_, features)| { + !features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) + }) .map(|(s, c)| (s.to_owned(), c.to_owned())) .collect::(); } @@ -564,7 +670,17 @@ impl PaintableSink { let window = gtk::Window::new(); let picture = gtk::Picture::new(); picture.set_paintable(Some(&paintable)); - window.set_child(Some(&picture)); + + #[cfg(feature = "gtk_v4_14")] + { + let offload = gtk::GraphicsOffload::new(Some(&picture)); + offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled); + window.set_child(Some(&offload)); + } + #[cfg(not(feature = "gtk_v4_14"))] + { + window.set_child(Some(&picture)); + } window.set_default_size(640, 480); window.connect_close_request({ diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index eccff6b8..f71cc9ec 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -31,7 +31,7 @@ static CAT: Lazy = Lazy::new(|| { #[derive(Debug)] pub struct Paintable { paintables: RefCell>, - cached_textures: RefCell>, + cached_textures: RefCell>, gl_context: RefCell>, background_color: Cell, #[cfg(feature = "gtk_v4_10")] @@ -197,12 +197,14 @@ impl PaintableImpl for Paintable { ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0; scale_y = scale_x; } - } - snapshot.append_color( - &background_color, - &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), - ); + if !background_color.is_clear() { + snapshot.append_color( + &background_color, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); + } + } snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32)); @@ -331,34 +333,44 @@ impl PaintableImpl for Paintable { } impl Paintable { - pub(super) fn handle_frame_changed(&self, frame: Option) { + pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) { let context = self.gl_context.borrow(); - if let Some(frame) = frame { - gst::trace!(CAT, imp: self, "Received new frame"); - let new_paintables = - frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()); - let new_size = new_paintables - .first() - .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) - .unwrap(); + gst::trace!(CAT, imp: self, "Received new frame"); - let old_paintables = self.paintables.replace(new_paintables); - let old_size = old_paintables - .first() - .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)); + let new_paintables = + match frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()) { + Ok(textures) => textures, + Err(err) => { + gst::element_error!( + sink, + gst::ResourceError::Failed, + ["Failed to transform frame into textures: {err}"] + ); + return; + } + }; - if Some(new_size) != old_size { - gst::debug!( - CAT, - imp: self, - "Size changed from {old_size:?} to {new_size:?}", - ); - self.obj().invalidate_size(); - } + let new_size = new_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) + .unwrap(); - self.obj().invalidate_contents(); + let old_paintables = self.paintables.replace(new_paintables); + let old_size = old_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)); + + if Some(new_size) != old_size { + gst::debug!( + CAT, + imp: self, + "Size changed from {old_size:?} to {new_size:?}", + ); + self.obj().invalidate_size(); } + + self.obj().invalidate_contents(); } pub(super) fn handle_flush_frames(&self) { diff --git a/video/gtk4/src/sink/paintable/mod.rs b/video/gtk4/src/sink/paintable/mod.rs index 835c43de..63e42b3d 100644 --- a/video/gtk4/src/sink/paintable/mod.rs +++ b/video/gtk4/src/sink/paintable/mod.rs @@ -30,8 +30,8 @@ impl Paintable { } impl Paintable { - pub(crate) fn handle_frame_changed(&self, frame: Option) { - self.imp().handle_frame_changed(frame); + pub(crate) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) { + self.imp().handle_frame_changed(sink, frame); } pub(crate) fn handle_flush_frames(&self) { From 3dd800ac7728ea4b3cb6ef15213d66c762e5f8ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 24 Apr 2024 15:22:57 +0200 Subject: [PATCH 2/6] gtk4paintablesink: Implement child proxy interface This allows setting properties on the paintable from gst-launch-1.0. Part-of: --- docs/plugins/gst_plugins_cache.json | 3 + video/gtk4/src/sink/imp.rs | 89 ++++++++++++++++++++++++----- video/gtk4/src/sink/mod.rs | 3 +- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index ad9844ab..1c08c051 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -2472,6 +2472,9 @@ "GInitiallyUnowned", "GObject" ], + "interfaces": [ + "GstChildProxy" + ], "klass": "Sink/Video", "long-name": "GTK 4 Paintable Sink", "pad-templates": { diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index eff4b2ec..8b98c4cb 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -82,6 +82,7 @@ impl ObjectSubclass for PaintableSink { const NAME: &'static str = "GstGtk4PaintableSink"; type Type = super::PaintableSink; type ParentType = gst_video::VideoSink; + type Interfaces = (gst::ChildProxy,); } impl ObjectImpl for PaintableSink { @@ -110,12 +111,14 @@ impl ObjectImpl for PaintableSink { return None::<&gdk::Paintable>.to_value(); } - let mut paintable = self.paintable.lock().unwrap(); - if paintable.is_none() { - self.create_paintable(&mut paintable); + let mut paintable_guard = self.paintable.lock().unwrap(); + let mut created = false; + if paintable_guard.is_none() { + created = true; + self.create_paintable(&mut paintable_guard); } - let paintable = match &*paintable { + let paintable = match &*paintable_guard { Some(ref paintable) => paintable, None => { gst::error!(CAT, imp: self, "Failed to create paintable"); @@ -124,16 +127,31 @@ impl ObjectImpl for PaintableSink { }; // Getter must be called from the main thread - if paintable.is_owner() { - paintable.get_ref().to_value() - } else { + if !paintable.is_owner() { gst::error!( CAT, imp: self, "Can't retrieve Paintable from non-main thread" ); - None::<&gdk::Paintable>.to_value() + return None::<&gdk::Paintable>.to_value(); } + + let paintable = paintable.get_ref().clone(); + drop(paintable_guard); + + if created { + let self_ = self.to_owned(); + glib::MainContext::default().invoke(move || { + let paintable_guard = self_.paintable.lock().unwrap(); + if let Some(paintable) = &*paintable_guard { + let paintable_clone = paintable.get_ref().clone(); + drop(paintable_guard); + self_.obj().child_added(&paintable_clone, "paintable"); + } + }); + } + + paintable.to_value() } _ => unimplemented!(), } @@ -290,18 +308,31 @@ impl ElementImpl for PaintableSink { } } - let mut paintable = self.paintable.lock().unwrap(); - - if paintable.is_none() { - self.create_paintable(&mut paintable); + let mut paintable_guard = self.paintable.lock().unwrap(); + let mut created = false; + if paintable_guard.is_none() { + created = true; + self.create_paintable(&mut paintable_guard); } - if paintable.is_none() { + if paintable_guard.is_none() { gst::error!(CAT, imp: self, "Failed to create paintable"); return Err(gst::StateChangeError); } - drop(paintable); + drop(paintable_guard); + + if created { + let self_ = self.to_owned(); + glib::MainContext::default().invoke(move || { + let paintable_guard = self_.paintable.lock().unwrap(); + if let Some(paintable) = &*paintable_guard { + let paintable_clone = paintable.get_ref().clone(); + drop(paintable_guard); + self_.obj().child_added(&paintable_clone, "paintable"); + } + }); + } // Notify the pipeline about the GL display and wrapped context so that any other // elements in the pipeline ideally use the same / create GL contexts that are @@ -1189,3 +1220,33 @@ impl PaintableSink { } } } + +impl ChildProxyImpl for PaintableSink { + fn child_by_index(&self, index: u32) -> Option { + if index != 0 { + return None; + } + + let paintable = self.paintable.lock().unwrap(); + paintable + .as_ref() + .filter(|p| p.is_owner()) + .map(|p| p.get_ref().upcast_ref::().clone()) + } + + fn child_by_name(&self, name: &str) -> Option { + if name == "paintable" { + return self.child_by_index(0); + } + None + } + + fn children_count(&self) -> u32 { + let paintable = self.paintable.lock().unwrap(); + if paintable.is_some() { + 1 + } else { + 0 + } + } +} diff --git a/video/gtk4/src/sink/mod.rs b/video/gtk4/src/sink/mod.rs index 96ff9299..5008ed00 100644 --- a/video/gtk4/src/sink/mod.rs +++ b/video/gtk4/src/sink/mod.rs @@ -22,7 +22,8 @@ enum SinkEvent { glib::wrapper! { pub struct PaintableSink(ObjectSubclass) - @extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object; + @extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object, + @implements gst::ChildProxy; } impl PaintableSink { From b42bd3d02619520233d269c310fb17cd96c05dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 24 Apr 2024 15:42:24 +0200 Subject: [PATCH 3/6] gtk4paintablesink: Add force-aspect-ratio property like in other video sinks Unlike in other sinks this defaults to false as generally every user of GDK paintables already ensures that the aspect ratio is kept and the paintable is layed out in the most optimal way based on the context. Part-of: --- video/gtk4/src/sink/paintable/imp.rs | 44 ++++++++++++++++++---------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index f71cc9ec..b520fc02 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -37,6 +37,7 @@ pub struct Paintable { #[cfg(feature = "gtk_v4_10")] scaling_filter: Cell, use_scaling_filter: Cell, + force_aspect_ratio: Cell, #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader, } @@ -51,6 +52,7 @@ impl Default for Paintable { #[cfg(feature = "gtk_v4_10")] scaling_filter: Cell::new(gsk::ScalingFilter::Linear), use_scaling_filter: Cell::new(false), + force_aspect_ratio: Cell::new(false), #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!( "premult.glsl" @@ -94,6 +96,11 @@ impl ObjectImpl for Paintable { .blurb("Use selected scaling filter or GTK default for rendering") .default_value(false) .build(), + glib::ParamSpecBoolean::builder("force-aspect-ratio") + .nick("Force Aspect Ratio") + .blurb("When enabled, scaling will respect original aspect ratio") + .default_value(true) + .build(), ] }); @@ -117,6 +124,7 @@ impl ObjectImpl for Paintable { "scaling-filter" => self.scaling_filter.get().to_value(), #[cfg(feature = "gtk_v4_10")] "use-scaling-filter" => self.use_scaling_filter.get().to_value(), + "force-aspect-ratio" => self.force_aspect_ratio.get().to_value(), _ => unimplemented!(), } } @@ -139,6 +147,7 @@ impl ObjectImpl for Paintable { "scaling-filter" => self.scaling_filter.set(value.get().unwrap()), #[cfg(feature = "gtk_v4_10")] "use-scaling-filter" => self.use_scaling_filter.set(value.get().unwrap()), + "force-aspect-ratio" => self.force_aspect_ratio.set(value.get().unwrap()), _ => unimplemented!(), } } @@ -173,6 +182,7 @@ impl PaintableImpl for Paintable { let snapshot = snapshot.downcast_ref::().unwrap(); let background_color = self.background_color.get(); + let force_aspect_ratio = self.force_aspect_ratio.get(); let paintables = self.paintables.borrow(); if !paintables.is_empty() { @@ -186,23 +196,25 @@ impl PaintableImpl for Paintable { let mut trans_x = 0.0; let mut trans_y = 0.0; - // TODO: Property for keeping aspect ratio or not - if (scale_x - scale_y).abs() > f64::EPSILON { - if scale_x > scale_y { - trans_x = - ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0; - scale_x = scale_y; - } else { - trans_y = - ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0; - scale_y = scale_x; - } + if force_aspect_ratio { + if (scale_x - scale_y).abs() > f64::EPSILON { + if scale_x > scale_y { + trans_x = + ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0; + scale_x = scale_y; + } else { + trans_y = ((frame_height as f64 * scale_y) + - (frame_height as f64 * scale_x)) + / 2.0; + scale_y = scale_x; + } - if !background_color.is_clear() { - snapshot.append_color( - &background_color, - &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), - ); + if !background_color.is_clear() { + snapshot.append_color( + &background_color, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); + } } } From c95e07a897c0800f2ca997833dba0909281e701a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 24 Apr 2024 15:48:53 +0200 Subject: [PATCH 4/6] gtk4paintablesink: Improve scaling logic If force-aspect-ratio=false then make sure to fully fill the given width/height with the video frame and avoid rounding errors. This makes sure that the video is rendered in the exact position selected by the caller and that graphics offloading is going to work more likely. In other cases and for all overlays, make sure that the calculated positions are staying inside (0, 0, width, height) as rendering outside is not allowed by GTK. Part-of: --- video/gtk4/src/sink/paintable/imp.rs | 269 ++++++++++++++------------- 1 file changed, 144 insertions(+), 125 deletions(-) diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index b520fc02..dd9b6749 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -185,42 +185,63 @@ impl PaintableImpl for Paintable { let force_aspect_ratio = self.force_aspect_ratio.get(); let paintables = self.paintables.borrow(); - if !paintables.is_empty() { - gst::trace!(CAT, imp: self, "Snapshotting frame"); + let Some(first_paintable) = paintables.first() else { + gst::trace!(CAT, imp: self, "Snapshotting black frame"); + snapshot.append_color( + &background_color, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); - let (frame_width, frame_height) = - paintables.first().map(|p| (p.width, p.height)).unwrap(); + return; + }; - let mut scale_x = width / frame_width as f64; - let mut scale_y = height / frame_height as f64; + gst::trace!(CAT, imp: self, "Snapshotting frame"); + + // The first paintable is the actual video frame and defines the overall size. + // + // Based on its size relative to the snapshot width/height, all other paintables are + // scaled accordingly. + let (frame_width, frame_height) = (first_paintable.width, first_paintable.height); + + let mut scale_x = width / frame_width as f64; + let mut scale_y = height / frame_height as f64; + + // Usually the caller makes sure that the aspect ratio is preserved. To enforce this here + // optionally, we scale the frame equally in both directions and center it. In addition the + // background color is drawn behind the frame to fill the gaps. + // + // This is not done by default for performance reasons and usually would draw a <1px + // background. + if force_aspect_ratio { let mut trans_x = 0.0; let mut trans_y = 0.0; - if force_aspect_ratio { - if (scale_x - scale_y).abs() > f64::EPSILON { - if scale_x > scale_y { - trans_x = - ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0; - scale_x = scale_y; - } else { - trans_y = ((frame_height as f64 * scale_y) - - (frame_height as f64 * scale_x)) - / 2.0; - scale_y = scale_x; - } - - if !background_color.is_clear() { - snapshot.append_color( - &background_color, - &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), - ); - } + if (scale_x - scale_y).abs() > f64::EPSILON { + if scale_x > scale_y { + trans_x = (width - (frame_width as f64 * scale_y)) / 2.0; + scale_x = scale_y; + } else { + trans_y = (height - (frame_height as f64 * scale_x)) / 2.0; + scale_y = scale_x; } } + if !background_color.is_clear() && (trans_x > f64::EPSILON || trans_y > f64::EPSILON) { + snapshot.append_color( + &background_color, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); + } snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32)); + } - for Texture { + // Make immutable + let scale_x = scale_x; + let scale_y = scale_y; + + for ( + idx, + Texture { texture, x, y, @@ -228,118 +249,116 @@ impl PaintableImpl for Paintable { height: paintable_height, global_alpha, has_alpha, - } in &*paintables + }, + ) in paintables.iter().enumerate() + { + snapshot.push_opacity(*global_alpha as f64); + + let bounds = if !force_aspect_ratio && idx == 0 { + // While this should end up with width again, be explicit in this case to avoid + // rounding errors and fill the whole area with the video frame. + graphene::Rect::new(0.0, 0.0, width as f32, height as f32) + } else { + // Scale texture position and size with the same scale factor as the main video + // frame, and make sure to not render outside (0, 0, width, height). + let x = f32::clamp(*x * scale_x as f32, 0.0, width as f32); + let y = f32::clamp(*y * scale_y as f32, 0.0, height as f32); + let texture_width = f32::min(*paintable_width * scale_x as f32, width as f32); + let texture_height = f32::min(*paintable_height * scale_y as f32, height as f32); + graphene::Rect::new(x, y, texture_width, texture_height) + }; + + // Only premultiply GL textures that expect to be in premultiplied RGBA format. + // + // For GTK 4.14 or newer we use the correct format directly when building the + // texture, but only if a GLES3+ context is used. In that case the NGL renderer is + // used by GTK, which supports non-premultiplied formats correctly and fast. + // + // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a + // self-mask to pre-multiply the alpha. + // + // For GTK before 4.10, we use a GL shader and hope that it works. + #[cfg(feature = "gtk_v4_10")] { - snapshot.push_opacity(*global_alpha as f64); - - let texture_width = *paintable_width * scale_x as f32; - let texture_height = *paintable_height * scale_y as f32; - let x = *x * scale_x as f32; - let y = *y * scale_y as f32; - let bounds = graphene::Rect::new(x, y, texture_width, texture_height); - - // Only premultiply GL textures that expect to be in premultiplied RGBA format. - // - // For GTK 4.14 or newer we use the correct format directly when building the - // texture, but only if a GLES3+ context is used. In that case the NGL renderer is - // used by GTK, which supports non-premultiplied formats correctly and fast. - // - // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a - // self-mask to pre-multiply the alpha. - // - // For GTK before 4.10, we use a GL shader and hope that it works. - #[cfg(feature = "gtk_v4_10")] - { - let context_requires_premult = { - #[cfg(feature = "gtk_v4_14")] - { - self.gl_context.borrow().as_ref().map_or(false, |context| { - context.api() != gdk::GLAPI::GLES || context.version().0 < 3 - }) - } - - #[cfg(not(feature = "gtk_v4_14"))] - { - true - } - }; - - let do_premult = - context_requires_premult && texture.is::() && *has_alpha; - if do_premult { - snapshot.push_mask(gsk::MaskMode::Alpha); - if self.use_scaling_filter.get() { - #[cfg(feature = "gtk_v4_10")] - snapshot.append_scaled_texture( - texture, - self.scaling_filter.get(), - &bounds, - ); - } else { - snapshot.append_texture(texture, &bounds); - } - snapshot.pop(); // pop mask - - // color matrix to set alpha of the source to 1.0 as it was - // already applied via the mask just above. - snapshot.push_color_matrix( - &graphene::Matrix::from_float({ - [ - 1.0, 0.0, 0.0, 0.0, // - 0.0, 1.0, 0.0, 0.0, // - 0.0, 0.0, 1.0, 0.0, // - 0.0, 0.0, 0.0, 0.0, - ] - }), - &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0), - ); + let context_requires_premult = { + #[cfg(feature = "gtk_v4_14")] + { + self.gl_context.borrow().as_ref().map_or(false, |context| { + context.api() != gdk::GLAPI::GLES || context.version().0 < 3 + }) } + #[cfg(not(feature = "gtk_v4_14"))] + { + true + } + }; + + let do_premult = + context_requires_premult && texture.is::() && *has_alpha; + if do_premult { + snapshot.push_mask(gsk::MaskMode::Alpha); if self.use_scaling_filter.get() { #[cfg(feature = "gtk_v4_10")] snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds); } else { snapshot.append_texture(texture, &bounds); } + snapshot.pop(); // pop mask - if do_premult { - snapshot.pop(); // pop color matrix - snapshot.pop(); // pop mask 2 - } - } - #[cfg(not(feature = "gtk_v4_10"))] - { - let do_premult = - texture.is::() && *has_alpha && gtk::micro_version() < 13; - if do_premult { - snapshot.push_gl_shader( - &self.premult_shader, - &bounds, - gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(), - ); - } - - if self.use_scaling_filter.get() { - #[cfg(feature = "gtk_v4_10")] - snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds); - } else { - snapshot.append_texture(texture, &bounds); - } - - if do_premult { - snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader - snapshot.pop(); // pop shader - } + // color matrix to set alpha of the source to 1.0 as it was + // already applied via the mask just above. + snapshot.push_color_matrix( + &graphene::Matrix::from_float({ + [ + 1.0, 0.0, 0.0, 0.0, // + 0.0, 1.0, 0.0, 0.0, // + 0.0, 0.0, 1.0, 0.0, // + 0.0, 0.0, 0.0, 0.0, + ] + }), + &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0), + ); } - snapshot.pop(); // pop opacity + if self.use_scaling_filter.get() { + #[cfg(feature = "gtk_v4_10")] + snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds); + } else { + snapshot.append_texture(texture, &bounds); + } + + if do_premult { + snapshot.pop(); // pop color matrix + snapshot.pop(); // pop mask 2 + } } - } else { - gst::trace!(CAT, imp: self, "Snapshotting black frame"); - snapshot.append_color( - &background_color, - &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), - ); + #[cfg(not(feature = "gtk_v4_10"))] + { + let do_premult = + texture.is::() && *has_alpha && gtk::micro_version() < 13; + if do_premult { + snapshot.push_gl_shader( + &self.premult_shader, + &bounds, + gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(), + ); + } + + if self.use_scaling_filter.get() { + #[cfg(feature = "gtk_v4_10")] + snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds); + } else { + snapshot.append_texture(texture, &bounds); + } + + if do_premult { + snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader + snapshot.pop(); // pop shader + } + } + + snapshot.pop(); // pop opacity } } } From 5803904deb45184d0dad579fc62e1804ea7467d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 24 Apr 2024 15:59:11 +0200 Subject: [PATCH 5/6] gtk4paintablesink: meson: Add auto-detection of GTK4 versions and dmabuf feature Part-of: --- meson.build | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/meson.build b/meson.build index fe48240d..a069e709 100644 --- a/meson.build +++ b/meson.build @@ -304,6 +304,23 @@ if get_option('gtk4').allowed() gtk4_features += 'winegl' endif endif + + gst_allocators_dep = dependency('gstreamer-allocators-1.0', version: '>=1.24', required: false) + gtk_dep = dependency('gtk4', version: '>=4.6', required: get_option('gtk4')) + if gtk_dep.found() + if host_system == 'linux' and gtk_dep.version().version_compare('>=4.14') and gst_allocators_dep.found() + gtk4_features += 'dmabuf' + endif + + if gtk_dep.version().version_compare('>=4.14') + gtk4_features += 'gtk_v4_14' + elif gtk_dep.version().version_compare('>=4.12') + gtk4_features += 'gtk_v4_12' + elif gtk_dep.version().version_compare('>=4.10') + gtk4_features += 'gtk_v4_10' + endif + endif + plugins += { 'gtk4': { 'library': 'libgstgtk4', From 927c3fcdb600bf36eea9800ad6c216e833a18c9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 25 Apr 2024 13:51:40 +0300 Subject: [PATCH 6/6] gtk4paintablesink: Update README.md with all the new features Part-of: --- video/gtk4/README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/video/gtk4/README.md b/video/gtk4/README.md index 4955d6a9..aaf66f33 100644 --- a/video/gtk4/README.md +++ b/video/gtk4/README.md @@ -1,10 +1,20 @@ -# Gtk 4 Sink & Paintable +# GTK 4 Sink & Paintable GTK 4 provides `gtk::Video` & `gtk::Picture` for rendering media such as videos. As the default `gtk::Video` widget doesn't offer the possibility to use a custom `gst::Pipeline`. The plugin provides a `gst_video::VideoSink` along with a `gdk::Paintable` that's capable of rendering the sink's frames. -The Sink can generate GL Textures if the system is capable of it, but it needs to be compiled -with either `wayland`, `x11glx` or `x11egl` cargo features. +The sink can generate GL Textures if the system is capable of it, but it needs +to be compiled with either `wayland`, `x11glx` or `x11egl` cargo features. On +Windows and macOS this is enabled by default. + +Additionally, the sink can render DMABufs directly on Linux if GTK 4.14 or +newer is used. For this the `dmabuf` feature needs to be enabled. + +Depending on the GTK version that is used and should be supported as minimum, +new features or more efficient processing can be opted in with the `gtk_v4_10`, +`gtk_v4_12` and `gtk_v4_14` features. The minimum GTK version required by the +sink is GTK 4.4 on Linux without GL support, and 4.6 on Windows and macOS, and +on Linux with GL support. # Flatpak Integration @@ -44,7 +54,7 @@ To build and include the plugin in a Flatpak manifest, you can add the following { "type": "git", "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs", - "branch": "0.10" + "branch": "0.12" } ], "build-options": {