From c92462b2406cacb82efab0d8f67140b9f574753b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 20 Apr 2024 23:19:20 +0300 Subject: [PATCH] gtk4: Implement support for directly importing dmabufs Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/441 Part-of: --- Cargo.lock | 25 ++ Cargo.toml | 1 + docs/plugins/gst_plugins_cache.json | 2 +- video/gtk4/Cargo.toml | 2 + video/gtk4/examples/gtksink.rs | 27 +- video/gtk4/src/sink/frame.rs | 357 ++++++++++++++++++++------- video/gtk4/src/sink/imp.rs | 196 ++++++++++++--- video/gtk4/src/sink/paintable/imp.rs | 68 ++--- video/gtk4/src/sink/paintable/mod.rs | 4 +- 9 files changed, 519 insertions(+), 163 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b41c3e7..fb42c823 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2454,6 +2454,7 @@ dependencies = [ "gdk4-x11", "gst-plugin-version-helper", "gstreamer", + "gstreamer-allocators", "gstreamer-base", "gstreamer-gl", "gstreamer-gl-egl", @@ -3059,6 +3060,30 @@ dependencies = [ "thiserror", ] +[[package]] +name = "gstreamer-allocators" +version = "0.23.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae" +dependencies = [ + "glib", + "gstreamer", + "gstreamer-allocators-sys", + "libc", + "once_cell", +] + +[[package]] +name = "gstreamer-allocators-sys" +version = "0.23.0" +source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae" +dependencies = [ + "glib-sys", + "gobject-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-app" version = "0.23.0" diff --git a/Cargo.toml b/Cargo.toml index ce7dfae8..9d803a18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,6 +134,7 @@ gdk-wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4- gdk-x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"} gdk-win32 = { package = "gdk4-win32", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"} gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } +gst-allocators = { package = "gstreamer-allocators", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-app = { package = "gstreamer-app", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" } diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index c5f304c4..ad9844ab 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -2476,7 +2476,7 @@ "long-name": "GTK 4 Paintable Sink", "pad-templates": { "sink": { - "caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "caps": "video/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { RGBA, RGB }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", "direction": "sink", "presence": "always" } diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index fce75fd9..ce011d65 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -17,6 +17,7 @@ gst = { workspace = true, features = ["v1_16"] } gst-base.workspace = true gst-video.workspace = true gst-gl = { workspace = true, features = ["v1_16"], optional = true } +gst-allocators = { workspace = true, features = ["v1_24"], optional = true } gst-gl-wayland = { workspace = true, features = ["v1_16"], optional = true } gst-gl-x11 = { workspace = true, features = ["v1_16"], optional = true } @@ -50,6 +51,7 @@ wayland = ["gtk/v4_6", "gdk-wayland", "gst-gl", "gst-gl-wayland"] x11glx = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-x11"] x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"] winegl = ["gdk-win32/egl", "gst-gl-egl"] +dmabuf = ["gst-allocators", "wayland", "gtk_v4_14", "gst-video/v1_24"] capi = [] doc = ["gst/v1_18"] gtk_v4_10 = ["gtk/v4_10"] diff --git a/video/gtk4/examples/gtksink.rs b/video/gtk4/examples/gtksink.rs index b431b5c5..abda23c6 100644 --- a/video/gtk4/examples/gtksink.rs +++ b/video/gtk4/examples/gtksink.rs @@ -6,13 +6,6 @@ use gtk::{gdk, gio, glib}; use std::cell::RefCell; fn create_ui(app: >k::Application) { - let window = gtk::ApplicationWindow::new(app); - window.set_default_size(640, 480); - - let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0); - let picture = gtk::Picture::new(); - let label = gtk::Label::new(Some("Position: 00:00:00")); - let pipeline = gst::Pipeline::new(); let overlay = gst::ElementFactory::make("clockoverlay") @@ -64,8 +57,26 @@ fn create_ui(app: >k::Application) { src.link_filtered(&overlay, &caps).unwrap(); overlay.link(&sink).unwrap(); + let window = gtk::ApplicationWindow::new(app); + window.set_default_size(640, 480); + + let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0); + + let picture = gtk::Picture::new(); picture.set_paintable(Some(&paintable)); - vbox.append(&picture); + + #[cfg(feature = "gtk_v4_14")] + { + let offload = gtk::GraphicsOffload::new(Some(&picture)); + offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled); + vbox.append(&offload); + } + #[cfg(not(feature = "gtk_v4_14"))] + { + vbox.append(&picture); + } + + let label = gtk::Label::new(Some("Position: 00:00:00")); vbox.append(&label); window.set_child(Some(&vbox)); diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index aaa7e224..f9f1c61b 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -14,7 +14,61 @@ use gst_video::prelude::*; #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] use gst_gl::prelude::*; use gtk::{gdk, glib}; -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + ops, +}; + +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum VideoInfo { + VideoInfo(gst_video::VideoInfo), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaDrm(gst_video::VideoInfoDmaDrm), +} + +impl From for VideoInfo { + fn from(v: gst_video::VideoInfo) -> Self { + VideoInfo::VideoInfo(v) + } +} + +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +impl From for VideoInfo { + fn from(v: gst_video::VideoInfoDmaDrm) -> Self { + VideoInfo::DmaDrm(v) + } +} + +impl ops::Deref for VideoInfo { + type Target = gst_video::VideoInfo; + + fn deref(&self) -> &Self::Target { + match self { + VideoInfo::VideoInfo(info) => info, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + VideoInfo::DmaDrm(info) => info, + } + } +} + +impl VideoInfo { + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + fn dma_drm(&self) -> Option<&gst_video::VideoInfoDmaDrm> { + match self { + VideoInfo::VideoInfo(..) => None, + VideoInfo::DmaDrm(info) => Some(info), + } + } +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum TextureCacheId { + Memory(usize), + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] + GL(usize), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaBuf([i32; 4]), +} #[derive(Debug)] enum MappedFrame { @@ -24,6 +78,17 @@ enum MappedFrame { frame: gst_gl::GLVideoFrame, wrapped_context: gst_gl::GLContext, }, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + DmaBuf { + buffer: gst::Buffer, + info: gst_video::VideoInfoDmaDrm, + n_planes: u32, + fds: [i32; 4], + offsets: [usize; 4], + strides: [usize; 4], + width: u32, + height: u32, + }, } impl MappedFrame { @@ -32,6 +97,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.buffer(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.buffer(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { buffer, .. } => buffer, } } @@ -40,6 +107,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.width(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.width(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.width(), } } @@ -48,6 +117,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.height(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.height(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.height(), } } @@ -56,6 +127,8 @@ impl MappedFrame { MappedFrame::SysMem(frame) => frame.format_info(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.format_info(), + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { info, .. } => info.format_info(), } } } @@ -108,16 +181,16 @@ fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat fn video_frame_to_memory_texture( frame: gst_video::VideoFrame, - cached_textures: &mut HashMap, - used_textures: &mut HashSet, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, ) -> (gdk::Texture, f64) { - let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize; + let ptr = frame.plane_data(0).unwrap().as_ptr() as usize; let pixel_aspect_ratio = (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64); - if let Some(texture) = cached_textures.get(&texture_id) { - used_textures.insert(texture_id); + if let Some(texture) = cached_textures.get(&TextureCacheId::Memory(ptr)) { + used_textures.insert(TextureCacheId::Memory(ptr)); return (texture.clone(), pixel_aspect_ratio); } @@ -135,8 +208,8 @@ fn video_frame_to_memory_texture( ) .upcast::(); - cached_textures.insert(texture_id, texture.clone()); - used_textures.insert(texture_id); + cached_textures.insert(TextureCacheId::Memory(ptr), texture.clone()); + used_textures.insert(TextureCacheId::Memory(ptr)); (texture, pixel_aspect_ratio) } @@ -144,8 +217,8 @@ fn video_frame_to_memory_texture( #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] fn video_frame_to_gl_texture( frame: gst_gl::GLVideoFrame, - cached_textures: &mut HashMap, - used_textures: &mut HashSet, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, gdk_context: &gdk::GLContext, #[allow(unused)] wrapped_context: &gst_gl::GLContext, ) -> (gdk::Texture, f64) { @@ -154,8 +227,8 @@ fn video_frame_to_gl_texture( let pixel_aspect_ratio = (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64); - if let Some(texture) = cached_textures.get(&(texture_id)) { - used_textures.insert(texture_id); + if let Some(texture) = cached_textures.get(&TextureCacheId::GL(texture_id)) { + used_textures.insert(TextureCacheId::GL(texture_id)); return (texture.clone(), pixel_aspect_ratio); } @@ -237,18 +310,64 @@ fn video_frame_to_gl_texture( .upcast::() }; - cached_textures.insert(texture_id, texture.clone()); - used_textures.insert(texture_id); + cached_textures.insert(TextureCacheId::GL(texture_id), texture.clone()); + used_textures.insert(TextureCacheId::GL(texture_id)); (texture, pixel_aspect_ratio) } +#[cfg(all(target_os = "linux", feature = "dmabuf"))] +#[allow(clippy::too_many_arguments)] +fn video_frame_to_dmabuf_texture( + buffer: gst::Buffer, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, + info: &gst_video::VideoInfoDmaDrm, + n_planes: u32, + fds: &[i32; 4], + offsets: &[usize; 4], + strides: &[usize; 4], + width: u32, + height: u32, +) -> Result<(gdk::Texture, f64), glib::Error> { + let pixel_aspect_ratio = (info.par().numer() as f64) / (info.par().denom() as f64); + + if let Some(texture) = cached_textures.get(&TextureCacheId::DmaBuf(*fds)) { + used_textures.insert(TextureCacheId::DmaBuf(*fds)); + return Ok((texture.clone(), pixel_aspect_ratio)); + } + + let builder = gdk::DmabufTextureBuilder::new(); + builder.set_display(&gdk::Display::default().unwrap()); + builder.set_fourcc(info.fourcc()); + builder.set_modifier(info.modifier()); + builder.set_width(width); + builder.set_height(height); + builder.set_n_planes(n_planes); + for plane in 0..(n_planes as usize) { + builder.set_fd(plane as u32, fds[plane]); + builder.set_offset(plane as u32, offsets[plane] as u32); + builder.set_stride(plane as u32, strides[plane] as u32); + } + + let texture = unsafe { + builder.build_with_release_func(move || { + drop(buffer); + })? + }; + + cached_textures.insert(TextureCacheId::DmaBuf(*fds), texture.clone()); + used_textures.insert(TextureCacheId::DmaBuf(*fds)); + + Ok((texture, pixel_aspect_ratio)) +} + impl Frame { pub(crate) fn into_textures( self, #[allow(unused_variables)] gdk_context: Option<&gdk::GLContext>, - cached_textures: &mut HashMap, - ) -> Vec { + cached_textures: &mut HashMap, + ) -> Result, glib::Error> { let mut textures = Vec::with_capacity(1 + self.overlays.len()); let mut used_textures = HashSet::with_capacity(1 + self.overlays.len()); @@ -278,6 +397,28 @@ impl Frame { &wrapped_context, ) } + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { + buffer, + info, + n_planes, + fds, + offsets, + strides, + width, + height, + } => video_frame_to_dmabuf_texture( + buffer, + cached_textures, + &mut used_textures, + &info, + n_planes, + &fds, + &offsets, + &strides, + width, + height, + )?, }; textures.push(Texture { @@ -309,14 +450,14 @@ impl Frame { // Remove textures that were not used this time cached_textures.retain(|id, _| used_textures.contains(id)); - textures + Ok(textures) } } impl Frame { pub(crate) fn new( buffer: &gst::Buffer, - info: &gst_video::VideoInfo, + info: &VideoInfo, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option< &gst_gl::GLContext, >, @@ -327,77 +468,125 @@ impl Frame { // Empty buffers get filtered out in show_frame debug_assert!(buffer.n_memory() > 0); - let mut frame; + #[allow(unused_mut)] + let mut frame = None; - #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))] + #[cfg(all(target_os = "linux", feature = "dmabuf"))] { - frame = Self { - frame: MappedFrame::SysMem( + // Check we received a buffer with dmabuf memory and if so do some checks before + // passing it onwards + if frame.is_none() + && buffer + .peek_memory(0) + .is_memory_type::() + { + if let Some((vmeta, info)) = + Option::zip(buffer.meta::(), info.dma_drm()) + { + let mut fds = [-1i32; 4]; + let mut offsets = [0; 4]; + let mut strides = [0; 4]; + let n_planes = vmeta.n_planes() as usize; + + let vmeta_offsets = vmeta.offset(); + let vmeta_strides = vmeta.stride(); + + for plane in 0..n_planes { + let Some((range, skip)) = + buffer.find_memory(vmeta_offsets[plane]..(vmeta_offsets[plane] + 1)) + else { + break; + }; + + let mem = buffer.peek_memory(range.start); + let Some(mem) = mem.downcast_memory_ref::() + else { + break; + }; + + let fd = mem.fd(); + fds[plane] = fd; + offsets[plane] = mem.offset() + skip; + strides[plane] = vmeta_strides[plane] as usize; + } + + // All fds valid? + if fds[0..n_planes].iter().all(|fd| *fd != -1) { + frame = Some(MappedFrame::DmaBuf { + buffer: buffer.clone(), + info: info.clone(), + n_planes: n_planes as u32, + fds, + offsets, + strides, + width: vmeta.width(), + height: vmeta.height(), + }); + } + } + } + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] + { + if frame.is_none() { + // Check we received a buffer with GL memory and if the context of that memory + // can share with the wrapped context around the GDK GL context. + // + // If not it has to be uploaded to the GPU. + let memory_ctx = buffer + .peek_memory(0) + .downcast_memory_ref::() + .and_then(|m| { + let ctx = m.context(); + if wrapped_context + .map_or(false, |wrapped_context| wrapped_context.can_share(ctx)) + { + Some(ctx) + } else { + None + } + }); + + if let Some(memory_ctx) = memory_ctx { + // If there is no GLSyncMeta yet then we need to add one here now, which requires + // obtaining a writable buffer. + let mapped_frame = if buffer.meta::().is_some() { + gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)? + } else { + let mut buffer = buffer.clone(); + { + let buffer = buffer.make_mut(); + gst_gl::GLSyncMeta::add(buffer, memory_ctx); + } + gst_gl::GLVideoFrame::from_buffer_readable(buffer, info) + .map_err(|_| gst::FlowError::Error)? + }; + + // Now that it's guaranteed that there is a sync meta and the frame is mapped, set + // a sync point so we can ensure that the texture is ready later when making use of + // it as gdk::GLTexture. + let meta = mapped_frame.buffer().meta::().unwrap(); + meta.set_sync_point(memory_ctx); + + frame = Some(MappedFrame::GL { + frame: mapped_frame, + wrapped_context: wrapped_context.unwrap().clone(), + }); + } + } + } + } + + let mut frame = Self { + frame: match frame { + Some(frame) => frame, + None => MappedFrame::SysMem( gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) .map_err(|_| gst::FlowError::Error)?, ), - overlays: vec![], - }; - } - #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] - { - // Check we received a buffer with GL memory and if the context of that memory - // can share with the wrapped context around the GDK GL context. - // - // If not it has to be uploaded to the GPU. - let memory_ctx = buffer - .peek_memory(0) - .downcast_memory_ref::() - .and_then(|m| { - let ctx = m.context(); - if wrapped_context - .map_or(false, |wrapped_context| wrapped_context.can_share(ctx)) - { - Some(ctx) - } else { - None - } - }); - - if let Some(memory_ctx) = memory_ctx { - // If there is no GLSyncMeta yet then we need to add one here now, which requires - // obtaining a writable buffer. - let mapped_frame = if buffer.meta::().is_some() { - gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)? - } else { - let mut buffer = buffer.clone(); - { - let buffer = buffer.make_mut(); - gst_gl::GLSyncMeta::add(buffer, memory_ctx); - } - gst_gl::GLVideoFrame::from_buffer_readable(buffer, info) - .map_err(|_| gst::FlowError::Error)? - }; - - // Now that it's guaranteed that there is a sync meta and the frame is mapped, set - // a sync point so we can ensure that the texture is ready later when making use of - // it as gdk::GLTexture. - let meta = mapped_frame.buffer().meta::().unwrap(); - meta.set_sync_point(memory_ctx); - - frame = Self { - frame: MappedFrame::GL { - frame: mapped_frame, - wrapped_context: wrapped_context.unwrap().clone(), - }, - overlays: vec![], - }; - } else { - frame = Self { - frame: MappedFrame::SysMem( - gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)?, - ), - overlays: vec![], - }; - } - } + }, + overlays: vec![], + }; frame.overlays = frame .frame diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index dcad8d42..eff4b2ec 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -1,7 +1,7 @@ // // Copyright (C) 2021 Bilal Elmoussaoui // Copyright (C) 2021 Jordan Petridis -// Copyright (C) 2021 Sebastian Dröge +// Copyright (C) 2021-2024 Sebastian Dröge // // This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. // If a copy of the MPL was not distributed with this file, You can obtain one at @@ -62,7 +62,7 @@ pub(crate) static CAT: Lazy = Lazy::new(|| { pub struct PaintableSink { paintable: Mutex>>, window: Mutex>>, - info: Mutex>, + info: Mutex>, sender: Mutex>>, pending_frame: Mutex>, cached_caps: Mutex>, @@ -163,53 +163,99 @@ impl ElementImpl for PaintableSink { { let caps = caps.get_mut().unwrap(); + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + for features in [ + [ + gst_allocators::CAPS_FEATURE_MEMORY_DMABUF, + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + ] + .as_slice(), + [gst_allocators::CAPS_FEATURE_MEMORY_DMABUF].as_slice(), + ] { + let c = gst_video::VideoCapsBuilder::new() + .format(gst_video::VideoFormat::DmaDrm) + .features(features.iter().copied()) + .build(); + caps.append(c); + } + } + for features in [ - None, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] Some(gst::CapsFeatures::new([ - "memory:GLMemory", - "meta:GstVideoOverlayComposition", + gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY, + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, ])), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] - Some(gst::CapsFeatures::new(["memory:GLMemory"])), + Some(gst::CapsFeatures::new([ + gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY, + ])), Some(gst::CapsFeatures::new([ "memory:SystemMemory", - "meta:GstVideoOverlayComposition", + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, ])), - Some(gst::CapsFeatures::new(["meta:GstVideoOverlayComposition"])), + Some(gst::CapsFeatures::new([ + gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION, + ])), + None, ] { - const GL_FORMATS: &[gst_video::VideoFormat] = - &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb]; - const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[ - gst_video::VideoFormat::Bgra, - gst_video::VideoFormat::Argb, - gst_video::VideoFormat::Rgba, - gst_video::VideoFormat::Abgr, - gst_video::VideoFormat::Rgb, - gst_video::VideoFormat::Bgr, - ]; - - let formats = if features - .as_ref() - .is_some_and(|features| features.contains("memory:GLMemory")) + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] { - GL_FORMATS - } else { - NON_GL_FORMATS - }; + const GL_FORMATS: &[gst_video::VideoFormat] = + &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb]; + const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[ + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Argb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Abgr, + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Bgr, + ]; - let mut c = gst_video::video_make_raw_caps(formats).build(); + let formats = if features.as_ref().is_some_and(|features| { + features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) + }) { + GL_FORMATS + } else { + NON_GL_FORMATS + }; - if let Some(features) = features { - let c = c.get_mut().unwrap(); + let mut c = gst_video::video_make_raw_caps(formats).build(); - if features.contains("memory:GLMemory") { - c.set("texture-target", "2D") + if let Some(features) = features { + let c = c.get_mut().unwrap(); + + if features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) { + c.set("texture-target", "2D") + } + c.set_features_simple(Some(features)); } - c.set_features_simple(Some(features)); + caps.append(c); } + #[cfg(not(any( + target_os = "macos", + target_os = "windows", + feature = "gst-gl" + )))] + { + const FORMATS: &[gst_video::VideoFormat] = &[ + gst_video::VideoFormat::Bgra, + gst_video::VideoFormat::Argb, + gst_video::VideoFormat::Rgba, + gst_video::VideoFormat::Abgr, + gst_video::VideoFormat::Rgb, + gst_video::VideoFormat::Bgr, + ]; - caps.append(c); + let mut c = gst_video::video_make_raw_caps(FORMATS).build(); + + if let Some(features) = features { + let c = c.get_mut().unwrap(); + c.set_features_simple(Some(features)); + } + caps.append(c); + } } } @@ -361,8 +407,21 @@ impl BaseSinkImpl for PaintableSink { fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> { gst::debug!(CAT, imp: self, "Setting caps {caps:?}"); - let video_info = gst_video::VideoInfo::from_caps(caps) - .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?; + #[allow(unused_mut)] + let mut video_info = None; + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + if let Ok(info) = gst_video::VideoInfoDmaDrm::from_caps(caps) { + video_info = Some(info.into()); + } + } + + let video_info = match video_info { + Some(info) => info, + None => gst_video::VideoInfo::from_caps(caps) + .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))? + .into(), + }; self.info.lock().unwrap().replace(video_info); @@ -516,10 +575,11 @@ impl PaintableSink { match action { SinkEvent::FrameChanged => { + let Some(frame) = self.pending_frame() else { + return glib::ControlFlow::Continue; + }; gst::trace!(CAT, imp: self, "Frame changed"); - paintable - .get_ref() - .handle_frame_changed(self.pending_frame()) + paintable.get_ref().handle_frame_changed(&self.obj(), frame); } } @@ -530,13 +590,59 @@ impl PaintableSink { #[allow(unused_mut)] let mut tmp_caps = Self::pad_templates()[0].caps().clone(); + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + { + let formats = utils::invoke_on_main_thread(move || { + let Some(display) = gdk::Display::default() else { + return vec![]; + }; + let dmabuf_formats = display.dmabuf_formats(); + + let mut formats = vec![]; + let n_formats = dmabuf_formats.n_formats(); + for i in 0..n_formats { + let (fourcc, modifier) = dmabuf_formats.format(i); + + if fourcc == 0 || modifier == (u64::MAX >> 8) { + continue; + } + + formats.push(gst_video::dma_drm_fourcc_to_string(fourcc, modifier)); + } + + formats + }); + + if formats.is_empty() { + // Filter out dmabufs caps from the template pads if we have no supported formats + if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) { + tmp_caps = tmp_caps + .iter_with_features() + .filter(|(_, features)| { + !features.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) + }) + .map(|(s, c)| (s.to_owned(), c.to_owned())) + .collect::(); + } + } else { + let tmp_caps = tmp_caps.make_mut(); + for (s, f) in tmp_caps.iter_with_features_mut() { + if f.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) { + s.set("drm-format", gst::List::new(&formats)); + } + } + } + } + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] { // Filter out GL caps from the template pads if we have no context if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) { tmp_caps = tmp_caps .iter_with_features() - .filter(|(_, features)| !features.contains("memory:GLMemory")) + .filter(|(_, features)| { + !features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) + }) .map(|(s, c)| (s.to_owned(), c.to_owned())) .collect::(); } @@ -564,7 +670,17 @@ impl PaintableSink { let window = gtk::Window::new(); let picture = gtk::Picture::new(); picture.set_paintable(Some(&paintable)); - window.set_child(Some(&picture)); + + #[cfg(feature = "gtk_v4_14")] + { + let offload = gtk::GraphicsOffload::new(Some(&picture)); + offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled); + window.set_child(Some(&offload)); + } + #[cfg(not(feature = "gtk_v4_14"))] + { + window.set_child(Some(&picture)); + } window.set_default_size(640, 480); window.connect_close_request({ diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index eccff6b8..f71cc9ec 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -31,7 +31,7 @@ static CAT: Lazy = Lazy::new(|| { #[derive(Debug)] pub struct Paintable { paintables: RefCell>, - cached_textures: RefCell>, + cached_textures: RefCell>, gl_context: RefCell>, background_color: Cell, #[cfg(feature = "gtk_v4_10")] @@ -197,12 +197,14 @@ impl PaintableImpl for Paintable { ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0; scale_y = scale_x; } - } - snapshot.append_color( - &background_color, - &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), - ); + if !background_color.is_clear() { + snapshot.append_color( + &background_color, + &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), + ); + } + } snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32)); @@ -331,34 +333,44 @@ impl PaintableImpl for Paintable { } impl Paintable { - pub(super) fn handle_frame_changed(&self, frame: Option) { + pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) { let context = self.gl_context.borrow(); - if let Some(frame) = frame { - gst::trace!(CAT, imp: self, "Received new frame"); - let new_paintables = - frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()); - let new_size = new_paintables - .first() - .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) - .unwrap(); + gst::trace!(CAT, imp: self, "Received new frame"); - let old_paintables = self.paintables.replace(new_paintables); - let old_size = old_paintables - .first() - .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)); + let new_paintables = + match frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()) { + Ok(textures) => textures, + Err(err) => { + gst::element_error!( + sink, + gst::ResourceError::Failed, + ["Failed to transform frame into textures: {err}"] + ); + return; + } + }; - if Some(new_size) != old_size { - gst::debug!( - CAT, - imp: self, - "Size changed from {old_size:?} to {new_size:?}", - ); - self.obj().invalidate_size(); - } + let new_size = new_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) + .unwrap(); - self.obj().invalidate_contents(); + let old_paintables = self.paintables.replace(new_paintables); + let old_size = old_paintables + .first() + .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)); + + if Some(new_size) != old_size { + gst::debug!( + CAT, + imp: self, + "Size changed from {old_size:?} to {new_size:?}", + ); + self.obj().invalidate_size(); } + + self.obj().invalidate_contents(); } pub(super) fn handle_flush_frames(&self) { diff --git a/video/gtk4/src/sink/paintable/mod.rs b/video/gtk4/src/sink/paintable/mod.rs index 835c43de..63e42b3d 100644 --- a/video/gtk4/src/sink/paintable/mod.rs +++ b/video/gtk4/src/sink/paintable/mod.rs @@ -30,8 +30,8 @@ impl Paintable { } impl Paintable { - pub(crate) fn handle_frame_changed(&self, frame: Option) { - self.imp().handle_frame_changed(frame); + pub(crate) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) { + self.imp().handle_frame_changed(sink, frame); } pub(crate) fn handle_flush_frames(&self) {