From 0b2aa2646f73af6736f5ab3d0d040aaf3aca1ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 30 Nov 2022 11:55:05 +0200 Subject: [PATCH] gtk4: Make GL support fully optional Don't depend on gstreamer-gl if it's not enabled, and don't try doing anything with the GDK GL context at all. Part-of: --- video/gtk4/Cargo.toml | 8 +- video/gtk4/src/sink/frame.rs | 125 +++++++++++++--------- video/gtk4/src/sink/imp.rs | 197 ++++++++++++++++++++--------------- 3 files changed, 196 insertions(+), 134 deletions(-) diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index 7defdd22..de9bc799 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -16,7 +16,7 @@ gdk_x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", bra gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19.1", features = ["v1_16"] } gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19" } gst_video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19" } -gst_gl = { package = "gstreamer-gl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19", features = ["v1_16"] } +gst_gl = { package = "gstreamer-gl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19", features = ["v1_16"], optional = true } gst_gl_wayland = { package = "gstreamer-gl-wayland", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19", features = ["v1_16"], optional = true } gst_gl_x11 = { package = "gstreamer-gl-x11", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "0.19", version = "0.19", features = ["v1_16"], optional = true } @@ -36,9 +36,9 @@ gst-plugin-version-helper = { version = "0.7", path="../../version-helper" } [features] default = [] static = [] -wayland = ["gdk_wayland", "gst_gl_wayland"] -x11glx = ["gdk_x11", "gst_gl_x11"] -x11egl = ["gdk_x11", "gst_gl_egl"] +wayland = ["gdk_wayland", "gst_gl", "gst_gl_wayland"] +x11glx = ["gdk_x11", "gst_gl", "gst_gl_x11"] +x11egl = ["gdk_x11", "gst_gl", "gst_gl_egl"] capi = [] doc = ["gst/v1_18"] diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index 119bed37..430150ae 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -9,6 +9,9 @@ // // SPDX-License-Identifier: MPL-2.0 +use gst_video::prelude::*; + +#[cfg(feature = "gst_gl")] use gst_gl::prelude::*; use gtk::{gdk, glib}; use std::collections::{HashMap, HashSet}; @@ -17,6 +20,7 @@ use std::collections::{HashMap, HashSet}; pub(crate) struct Frame { frame: gst_video::VideoFrame, overlays: Vec, + #[cfg(feature = "gst_gl")] gst_context: Option, } @@ -90,6 +94,7 @@ fn video_frame_to_memory_texture( (texture, pixel_aspect_ratio) } +#[cfg(feature = "gst_gl")] fn video_frame_to_gl_texture( frame: &gst_video::VideoFrame, cached_textures: &mut HashMap, @@ -127,7 +132,7 @@ fn video_frame_to_gl_texture( impl Frame { pub(crate) fn into_textures( self, - gdk_context: Option<&gdk::GLContext>, + #[allow(unused_variables)] gdk_context: Option<&gdk::GLContext>, cached_textures: &mut HashMap, ) -> Vec { let mut textures = Vec::with_capacity(1 + self.overlays.len()); @@ -135,18 +140,26 @@ impl Frame { let width = self.frame.width(); let height = self.frame.height(); - let (texture, pixel_aspect_ratio) = - if let (Some(gdk_ctx), Some(gst_ctx)) = (gdk_context, self.gst_context.as_ref()) { - video_frame_to_gl_texture( - &self.frame, - cached_textures, - &mut used_textures, - gdk_ctx, - gst_ctx, - ) - } else { + let (texture, pixel_aspect_ratio) = { + #[cfg(not(feature = "gst_gl"))] + { video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures) - }; + } + #[cfg(feature = "gst_gl")] + { + if let (Some(gdk_ctx), Some(gst_ctx)) = (gdk_context, self.gst_context.as_ref()) { + video_frame_to_gl_texture( + &self.frame, + cached_textures, + &mut used_textures, + gdk_ctx, + gst_ctx, + ) + } else { + video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures) + } + } + }; textures.push(Texture { texture, @@ -182,47 +195,67 @@ impl Frame { pub(crate) fn new( buffer: &gst::Buffer, info: &gst_video::VideoInfo, - have_gl_context: bool, + #[allow(unused_variables)] have_gl_context: bool, ) -> Result { - let mut gst_context = None; - // Empty buffers get filtered out in show_frame debug_assert!(buffer.n_memory() > 0); - let is_buffer_gl = buffer - .peek_memory(0) - .downcast_memory_ref::() - .is_some(); + let mut frame; - let frame = if !is_buffer_gl || !have_gl_context { - gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)? - } else { - let gst_ctx = buffer + #[cfg(not(feature = "gst_gl"))] + { + frame = Self { + frame: gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)?, + overlays: vec![], + }; + } + #[cfg(feature = "gst_gl")] + { + let is_buffer_gl = buffer .peek_memory(0) .downcast_memory_ref::() - .map(|m| m.context()) - .expect("Failed to retrieve the GstGL Context."); + .is_some(); - gst_context = Some(gst_ctx.clone()); - - if let Some(meta) = buffer.meta::() { - meta.set_sync_point(gst_ctx); - gst_video::VideoFrame::from_buffer_readable_gl(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)? + if !is_buffer_gl || !have_gl_context { + frame = Self { + frame: gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)?, + overlays: vec![], + gst_context: None, + }; } else { - let mut buffer = buffer.clone(); - { - let buffer = buffer.make_mut(); - let meta = gst_gl::GLSyncMeta::add(buffer, gst_ctx); - meta.set_sync_point(gst_ctx); - } - gst_video::VideoFrame::from_buffer_readable_gl(buffer, info) - .map_err(|_| gst::FlowError::Error)? - } - }; + let gst_ctx = buffer + .peek_memory(0) + .downcast_memory_ref::() + .map(|m| m.context()) + .expect("Failed to retrieve the GstGL Context."); - let overlays = frame + let mapped_frame = if let Some(meta) = buffer.meta::() { + meta.set_sync_point(gst_ctx); + gst_video::VideoFrame::from_buffer_readable_gl(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)? + } else { + let mut buffer = buffer.clone(); + { + let buffer = buffer.make_mut(); + let meta = gst_gl::GLSyncMeta::add(buffer, gst_ctx); + meta.set_sync_point(gst_ctx); + } + gst_video::VideoFrame::from_buffer_readable_gl(buffer, info) + .map_err(|_| gst::FlowError::Error)? + }; + + frame = Self { + frame: mapped_frame, + overlays: vec![], + gst_context: Some(gst_ctx.clone()), + }; + } + } + + frame.overlays = frame + .frame .buffer() .iter_meta::() .flat_map(|meta| { @@ -258,10 +291,6 @@ impl Frame { }) .collect(); - Ok(Self { - frame, - overlays, - gst_context, - }) + Ok(frame) } } diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index 0a690ab4..ad436af5 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -13,25 +13,29 @@ use super::SinkEvent; use crate::sink::frame::Frame; use crate::sink::paintable::Paintable; -use glib::translate::*; use glib::Sender; use gtk::prelude::*; use gtk::{gdk, glib}; -use gst::prelude::*; use gst::subclass::prelude::*; use gst_base::subclass::prelude::*; -use gst_gl::prelude::GLContextExt as GstGLContextExt; -use gst_gl::prelude::*; use gst_video::subclass::prelude::*; use once_cell::sync::Lazy; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Mutex, MutexGuard}; use crate::utils; use fragile::Fragile; +#[cfg(feature = "gst_gl")] +use glib::translate::*; +#[cfg(feature = "gst_gl")] +use gst_gl::prelude::GLContextExt as GstGLContextExt; +#[cfg(feature = "gst_gl")] +use gst_gl::prelude::*; +#[cfg(feature = "gst_gl")] +use std::sync::atomic::{AtomicBool, Ordering}; + static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "gstgtk4paintablesink", @@ -46,10 +50,14 @@ pub struct PaintableSink { info: Mutex>, sender: Mutex>>, pending_frame: Mutex>, + #[cfg(feature = "gst_gl")] gst_display: Mutex>, + #[cfg(feature = "gst_gl")] gst_app_context: Mutex>, + #[cfg(feature = "gst_gl")] gst_context: Mutex>, cached_caps: Mutex>, + #[cfg(feature = "gst_gl")] have_gl_context: AtomicBool, } @@ -274,78 +282,87 @@ impl BaseSinkImpl for PaintableSink { // TODO: Provide a preferred "window size" here for higher-resolution rendering query.add_allocation_meta::(None); + #[cfg(not(feature = "gst_gl"))] { - // Early return if there is no context initialized - let gst_context_guard = self.gst_context.lock().unwrap(); - if gst_context_guard.is_none() { - gst::debug!( - CAT, - imp: self, - "Found no GL Context during propose_allocation." - ); - return Ok(()); - } + Ok(()) } - // GL specific things - let (caps, need_pool) = query.get_owned(); - - if caps.is_empty() { - return Err(gst::loggable_error!(CAT, "No caps where specified.")); - } - - if let Some(f) = caps.features(0) { - if !f.contains("memory:GLMemory") { - gst::debug!( - CAT, - imp: self, - "No 'memory:GLMemory' feature in caps: {}", - caps - ) - } - } - - let info = gst_video::VideoInfo::from_caps(&caps) - .map_err(|_| gst::loggable_error!(CAT, "Failed to get VideoInfo from caps"))?; - - let size = info.size() as u32; - + #[cfg(feature = "gst_gl")] { - let gst_context = { self.gst_context.lock().unwrap().clone().unwrap() }; - let buffer_pool = gst_gl::GLBufferPool::new(&gst_context); - - if need_pool { - gst::debug!(CAT, imp: self, "Creating new Pool"); - - let mut config = buffer_pool.config(); - config.set_params(Some(&caps), size, 0, 0); - config.add_option("GstBufferPoolOptionGLSyncMeta"); - - if let Err(err) = buffer_pool.set_config(config) { - return Err(gst::loggable_error!( + { + // Early return if there is no context initialized + let gst_context_guard = self.gst_context.lock().unwrap(); + if gst_context_guard.is_none() { + gst::debug!( CAT, - format!("Failed to set config in the GL BufferPool.: {}", err) - )); + imp: self, + "Found no GL Context during propose_allocation." + ); + return Ok(()); } } - // we need at least 2 buffer because we hold on to the last one - query.add_allocation_pool(Some(&buffer_pool), size, 2, 0); + // GL specific things + let (caps, need_pool) = query.get_owned(); - if gst_context.check_feature("GL_ARB_sync") - || gst_context.check_feature("GL_EXT_EGL_sync") - { - query.add_allocation_meta::(None) + if caps.is_empty() { + return Err(gst::loggable_error!(CAT, "No caps where specified.")); } - } - Ok(()) + if let Some(f) = caps.features(0) { + if !f.contains("memory:GLMemory") { + gst::debug!( + CAT, + imp: self, + "No 'memory:GLMemory' feature in caps: {}", + caps + ) + } + } + + let info = gst_video::VideoInfo::from_caps(&caps) + .map_err(|_| gst::loggable_error!(CAT, "Failed to get VideoInfo from caps"))?; + + let size = info.size() as u32; + + { + let gst_context = { self.gst_context.lock().unwrap().clone().unwrap() }; + let buffer_pool = gst_gl::GLBufferPool::new(&gst_context); + + if need_pool { + gst::debug!(CAT, imp: self, "Creating new Pool"); + + let mut config = buffer_pool.config(); + config.set_params(Some(&caps), size, 0, 0); + config.add_option("GstBufferPoolOptionGLSyncMeta"); + + if let Err(err) = buffer_pool.set_config(config) { + return Err(gst::loggable_error!( + CAT, + format!("Failed to set config in the GL BufferPool.: {}", err) + )); + } + } + + // we need at least 2 buffer because we hold on to the last one + query.add_allocation_pool(Some(&buffer_pool), size, 2, 0); + + if gst_context.check_feature("GL_ARB_sync") + || gst_context.check_feature("GL_EXT_EGL_sync") + { + query.add_allocation_meta::(None) + } + } + + Ok(()) + } } fn query(&self, query: &mut gst::QueryRef) -> bool { gst::log!(CAT, imp: self, "Handling query {:?}", query); match query.view_mut() { + #[cfg(feature = "gst_gl")] gst::QueryViewMut::Context(q) => { // Avoid holding the locks while we respond to the query // The objects are ref-counted anyway. @@ -396,11 +413,20 @@ impl VideoSinkImpl for PaintableSink { gst::FlowError::NotNegotiated })?; - let frame = Frame::new(buffer, info, self.have_gl_context.load(Ordering::Relaxed)) - .map_err(|err| { - gst::error!(CAT, imp: self, "Failed to map video frame"); - err - })?; + let have_gl_context = { + #[cfg(not(feature = "gst_gl"))] + { + false + } + #[cfg(feature = "gst_gl")] + { + self.have_gl_context.load(Ordering::Relaxed) + } + }; + let frame = Frame::new(buffer, info, have_gl_context).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to map video frame"); + err + })?; self.pending_frame.lock().unwrap().replace(frame); let sender = self.sender.lock().unwrap(); @@ -441,15 +467,19 @@ impl PaintableSink { } fn configure_caps(&self) { + #[allow(unused_mut)] let mut tmp_caps = Self::pad_templates()[0].caps().clone(); - // Filter out GL caps from the template pads if we have no context - if !self.have_gl_context.load(Ordering::Relaxed) { - tmp_caps = tmp_caps - .iter_with_features() - .filter(|(_, features)| !features.contains("memory:GLMemory")) - .map(|(s, c)| (s.to_owned(), c.to_owned())) - .collect::(); + #[cfg(feature = "gst_gl")] + { + // Filter out GL caps from the template pads if we have no context + if !self.have_gl_context.load(Ordering::Relaxed) { + tmp_caps = tmp_caps + .iter_with_features() + .filter(|(_, features)| !features.contains("memory:GLMemory")) + .map(|(s, c)| (s.to_owned(), c.to_owned())) + .collect::(); + } } self.cached_caps @@ -459,18 +489,18 @@ impl PaintableSink { } fn create_paintable(&self, paintable_storage: &mut MutexGuard>>) { - let ctx = self.realize_context(); + #[allow(unused_mut)] + let mut ctx = None; - let ctx = if let Some(c) = ctx { - if let Ok(c) = self.initialize_gl_wrapper(c) { - self.have_gl_context.store(true, Ordering::Relaxed); - Some(c) - } else { - None + #[cfg(feature = "gst_gl")] + { + if let Some(c) = self.realize_context() { + if let Ok(c) = self.initialize_gl_wrapper(c) { + self.have_gl_context.store(true, Ordering::Relaxed); + ctx = Some(c); + } } - } else { - None - }; + } self.configure_caps(); self.initialize_paintable(ctx, paintable_storage); @@ -505,6 +535,7 @@ impl PaintableSink { *self.sender.lock().unwrap() = Some(sender); } + #[cfg(feature = "gst_gl")] fn realize_context(&self) -> Option> { gst::debug!(CAT, imp: self, "Realizing GDK GL Context"); @@ -551,6 +582,7 @@ impl PaintableSink { utils::invoke_on_main_thread(cb) } + #[cfg(feature = "gst_gl")] fn initialize_gl_wrapper( &self, context: Fragile, @@ -560,6 +592,7 @@ impl PaintableSink { utils::invoke_on_main_thread(move || self_.imp().initialize_gl(context)) } + #[cfg(feature = "gst_gl")] fn initialize_gl( &self, context: Fragile,