diff --git a/ci/generate-static-test.py b/ci/generate-static-test.py index 0f5bb6d3..68c5d4cd 100755 --- a/ci/generate-static-test.py +++ b/ci/generate-static-test.py @@ -8,7 +8,7 @@ from utils import iterate_plugins # the csound version used on ci does not ship a .pc file # threadshare we skip in meson static build as well -IGNORE = ['csound', 'threadshare'] +IGNORE = ['csound', 'threadshare', 'gtk4'] outdir = sys.argv[1] diff --git a/ci/run_windows_tests.ps1 b/ci/run_windows_tests.ps1 index 7c2b5405..14e50645 100644 --- a/ci/run_windows_tests.ps1 +++ b/ci/run_windows_tests.ps1 @@ -17,11 +17,18 @@ function Run-Tests { param ( $Features ) + $local_exclude = $exclude_crates; + + # In this case the plugin will pull x11/wayland features + # which will fail to build on windows. + if (($Features -eq '--all-features') -or ($Features -eq '')) { + $local_exclude += @("--exclude", "gst-plugin-gtk4") + } Write-Host "Features: $Features" - Write-Host "Exclude string: $exclude_crates" + Write-Host "Exclude string: $local_exclude" - cargo build --color=always --workspace $exclude_crates --all-targets $Features + cargo build --color=always --workspace $local_exclude --all-targets $Features if (!$?) { Write-Host "Build failed" @@ -29,7 +36,7 @@ function Run-Tests { } $env:G_DEBUG="fatal_warnings" - cargo test --no-fail-fast --color=always --workspace $exclude_crates --all-targets $Features + cargo test --no-fail-fast --color=always --workspace $local_exclude --all-targets $Features if (!$?) { Write-Host "Tests failed" diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 7d67b603..f6b39921 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -1940,7 +1940,7 @@ "long-name": "GTK 4 Paintable Sink", "pad-templates": { "sink": { - "caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", "direction": "sink", "presence": "always" } diff --git a/meson.build b/meson.build index ce7846e5..a800e3cd 100644 --- a/meson.build +++ b/meson.build @@ -140,7 +140,7 @@ else message('csound not found, disabling its plugin') endif -if dependency('gtk4', required : get_option('gtk4')).found() +if dependency('gtk4', version: '>= 4.6.0', required : get_option('gtk4')).found() plugins += {'gst-plugin-gtk4' : 'libgstgtk4',} endif @@ -278,7 +278,7 @@ foreach plugin : plugins ) meson.override_dependency(plugin_name, dep) - if static_build and plugin_name in ['gstcsound', 'gstthreadshare'] + if static_build and plugin_name in ['gstcsound', 'gstthreadshare', 'gstgtk4'] warning('Static plugin @0@ is known to fail. It will not be included in libgstreamer-full.'.format(plugin_name)) else gst_plugins += dep diff --git a/video/gtk4/Cargo.toml b/video/gtk4/Cargo.toml index 02875993..f7f3388c 100644 --- a/video/gtk4/Cargo.toml +++ b/video/gtk4/Cargo.toml @@ -9,11 +9,18 @@ rust-version = "1.63" description = "GStreamer GTK 4 Sink element and Paintable widget" [dependencies] -gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs" } +gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_6"] } +gdk_wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_4"], optional = true} +gdk_x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_4"], optional = true} gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] } gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } gst_video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst_gl = { package = "gstreamer-gl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] } + +gst_gl_wayland = { package = "gstreamer-gl-wayland", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true } +gst_gl_x11 = { package = "gstreamer-gl-x11", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true } +gst_gl_egl = { package = "gstreamer-gl-egl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true } once_cell = "1.0" fragile = "2" @@ -27,7 +34,11 @@ path = "src/lib.rs" gst-plugin-version-helper = { path="../../version-helper" } [features] +default = [] static = [] +wayland = ["gdk_wayland", "gst_gl_wayland"] +x11glx = ["gdk_x11", "gst_gl_x11"] +x11egl = ["gdk_x11", "gst_gl_egl"] capi = [] doc = ["gst/v1_18"] diff --git a/video/gtk4/README.md b/video/gtk4/README.md index 44d2ff5c..82f87057 100644 --- a/video/gtk4/README.md +++ b/video/gtk4/README.md @@ -2,3 +2,6 @@ GTK 4 provides `gtk::Video` & `gtk::Picture` for rendering media such as videos. As the default `gtk::Video` widget doesn't offer the possibility to use a custom `gst::Pipeline`. The plugin provides a `gst_video::VideoSink` along with a `gdk::Paintable` that's capable of rendering the sink's frames. + +The Sink can generate GL Textures if the system is capable of it, but it needs to be compiled +with either `wayland`, `x11glx` or `x11egl` cargo features. diff --git a/video/gtk4/examples/gtksink.rs b/video/gtk4/examples/gtksink.rs index ee2b4e2f..bf271dd8 100644 --- a/video/gtk4/examples/gtksink.rs +++ b/video/gtk4/examples/gtksink.rs @@ -6,30 +6,6 @@ use gtk::{gdk, gio, glib}; use std::cell::RefCell; fn create_ui(app: >k::Application) { - let pipeline = gst::Pipeline::default(); - let src = gst::ElementFactory::make("videotestsrc").build().unwrap(); - - let overlay = gst::ElementFactory::make("clockoverlay") - .property("font-desc", "Monospace 42") - .build() - .unwrap(); - - let sink = gst::ElementFactory::make("gtk4paintablesink") - .build() - .unwrap(); - let paintable = sink.property::("paintable"); - - pipeline.add_many(&[&src, &overlay, &sink]).unwrap(); - src.link_filtered( - &overlay, - &gst_video::VideoCapsBuilder::new() - .width(640) - .height(480) - .build(), - ) - .unwrap(); - overlay.link(&sink).unwrap(); - let window = gtk::ApplicationWindow::new(app); window.set_default_size(640, 480); @@ -37,6 +13,48 @@ fn create_ui(app: >k::Application) { let picture = gtk::Picture::new(); let label = gtk::Label::new(Some("Position: 00:00:00")); + let pipeline = gst::Pipeline::new(None); + + let overlay = gst::ElementFactory::make("clockoverlay") + .property("font-desc", "Monospace 42") + .build() + .unwrap(); + + let gtksink = gst::ElementFactory::make("gtk4paintablesink") + .build() + .unwrap(); + + // Need to set state to Ready to get a GL context + gtksink.set_state(gst::State::Ready).unwrap(); + let paintable = gtksink.property::("paintable"); + + // TODO: future plans to provide a bin-like element that works with less setup + let (src, sink) = if paintable + .property::>("gl-context") + .is_some() + { + let src = gst::ElementFactory::make("gltestsrc").build().unwrap(); + + let sink = gst::ElementFactory::make("glsinkbin") + .property("sink", >ksink) + .build() + .unwrap(); + (src, sink) + } else { + let src = gst::ElementFactory::make("videotestsrc").build().unwrap(); + (src, gtksink) + }; + + pipeline.add_many(&[&src, &overlay, &sink]).unwrap(); + let caps = gst_video::VideoCapsBuilder::new() + .width(640) + .height(480) + .any_features() + .build(); + + src.link_filtered(&overlay, &caps).unwrap(); + overlay.link(&sink).unwrap(); + picture.set_paintable(Some(&paintable)); vbox.append(&picture); vbox.append(&label); @@ -115,12 +133,10 @@ fn main() { gstgtk4::plugin_register_static().expect("Failed to register gstgtk4 plugin"); - { - let app = gtk::Application::new(None, gio::ApplicationFlags::FLAGS_NONE); + let app = gtk::Application::new(None, gio::ApplicationFlags::FLAGS_NONE); - app.connect_activate(create_ui); - app.run(); - } + app.connect_activate(create_ui); + app.run(); unsafe { gst::deinit(); diff --git a/video/gtk4/src/lib.rs b/video/gtk4/src/lib.rs index a666d3e6..1ccd6116 100644 --- a/video/gtk4/src/lib.rs +++ b/video/gtk4/src/lib.rs @@ -18,6 +18,7 @@ use gst::glib; mod sink; +mod utils; pub use sink::PaintableSink; fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index 390cec06..119bed37 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -9,7 +9,7 @@ // // SPDX-License-Identifier: MPL-2.0 -use gtk::prelude::*; +use gst_gl::prelude::*; use gtk::{gdk, glib}; use std::collections::{HashMap, HashSet}; @@ -17,6 +17,7 @@ use std::collections::{HashMap, HashSet}; pub(crate) struct Frame { frame: gst_video::VideoFrame, overlays: Vec, + gst_context: Option, } #[derive(Debug)] @@ -31,7 +32,6 @@ struct Overlay { #[derive(Debug)] pub(crate) struct Texture { - //FIXME: create getters instead of having the fields public pub texture: gdk::Texture, pub x: f32, pub y: f32, @@ -90,15 +90,63 @@ fn video_frame_to_memory_texture( (texture, pixel_aspect_ratio) } +fn video_frame_to_gl_texture( + frame: &gst_video::VideoFrame, + cached_textures: &mut HashMap, + used_textures: &mut HashSet, + gdk_context: &gdk::GLContext, + gst_context: &gst_gl::GLContext, +) -> (gdk::Texture, f64) { + let texture_id = frame.texture_id(0).expect("Invalid texture id") as usize; + + let pixel_aspect_ratio = + (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64); + + if let Some(texture) = cached_textures.get(&(texture_id)) { + used_textures.insert(texture_id); + return (texture.clone(), pixel_aspect_ratio); + } + + let width = frame.width(); + let height = frame.height(); + + let sync_meta = frame.buffer().meta::().unwrap(); + sync_meta.wait(gst_context); + + let texture = unsafe { + gdk::GLTexture::new(gdk_context, texture_id as u32, width as i32, height as i32) + .upcast::() + }; + + cached_textures.insert(texture_id, texture.clone()); + used_textures.insert(texture_id); + + (texture, pixel_aspect_ratio) +} + impl Frame { - pub fn into_textures(self, cached_textures: &mut HashMap) -> Vec { + pub(crate) fn into_textures( + self, + gdk_context: Option<&gdk::GLContext>, + cached_textures: &mut HashMap, + ) -> Vec { let mut textures = Vec::with_capacity(1 + self.overlays.len()); let mut used_textures = HashSet::with_capacity(1 + self.overlays.len()); let width = self.frame.width(); let height = self.frame.height(); let (texture, pixel_aspect_ratio) = - video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures); + if let (Some(gdk_ctx), Some(gst_ctx)) = (gdk_context, self.gst_context.as_ref()) { + video_frame_to_gl_texture( + &self.frame, + cached_textures, + &mut used_textures, + gdk_ctx, + gst_ctx, + ) + } else { + video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures) + }; textures.push(Texture { texture, @@ -131,9 +179,48 @@ impl Frame { } impl Frame { - pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Result { - let frame = gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) - .map_err(|_| gst::FlowError::Error)?; + pub(crate) fn new( + buffer: &gst::Buffer, + info: &gst_video::VideoInfo, + have_gl_context: bool, + ) -> Result { + let mut gst_context = None; + + // Empty buffers get filtered out in show_frame + debug_assert!(buffer.n_memory() > 0); + + let is_buffer_gl = buffer + .peek_memory(0) + .downcast_memory_ref::() + .is_some(); + + let frame = if !is_buffer_gl || !have_gl_context { + gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)? + } else { + let gst_ctx = buffer + .peek_memory(0) + .downcast_memory_ref::() + .map(|m| m.context()) + .expect("Failed to retrieve the GstGL Context."); + + gst_context = Some(gst_ctx.clone()); + + if let Some(meta) = buffer.meta::() { + meta.set_sync_point(gst_ctx); + gst_video::VideoFrame::from_buffer_readable_gl(buffer.clone(), info) + .map_err(|_| gst::FlowError::Error)? + } else { + let mut buffer = buffer.clone(); + { + let buffer = buffer.make_mut(); + let meta = gst_gl::GLSyncMeta::add(buffer, gst_ctx); + meta.set_sync_point(gst_ctx); + } + gst_video::VideoFrame::from_buffer_readable_gl(buffer, info) + .map_err(|_| gst::FlowError::Error)? + } + }; let overlays = frame .buffer() @@ -171,6 +258,10 @@ impl Frame { }) .collect(); - Ok(Self { frame, overlays }) + Ok(Self { + frame, + overlays, + gst_context, + }) } } diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index f0127b38..0a690ab4 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -13,21 +13,26 @@ use super::SinkEvent; use crate::sink::frame::Frame; use crate::sink::paintable::Paintable; -use glib::prelude::*; +use glib::translate::*; use glib::Sender; +use gtk::prelude::*; +use gtk::{gdk, glib}; +use gst::prelude::*; use gst::subclass::prelude::*; use gst_base::subclass::prelude::*; +use gst_gl::prelude::GLContextExt as GstGLContextExt; +use gst_gl::prelude::*; use gst_video::subclass::prelude::*; -use gtk::glib; - use once_cell::sync::Lazy; -use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard}; +use crate::utils; use fragile::Fragile; -pub(super) static CAT: Lazy = Lazy::new(|| { +static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "gstgtk4paintablesink", gst::DebugColorFlags::empty(), @@ -35,25 +40,24 @@ pub(super) static CAT: Lazy = Lazy::new(|| { ) }); -#[derive(Default)] +#[derive(Default, Debug)] pub struct PaintableSink { - pub(super) paintable: Mutex>>, + paintable: Mutex>>, info: Mutex>, - pub(super) sender: Mutex>>, - pub(super) pending_frame: Mutex>, + sender: Mutex>>, + pending_frame: Mutex>, + gst_display: Mutex>, + gst_app_context: Mutex>, + gst_context: Mutex>, + cached_caps: Mutex>, + have_gl_context: AtomicBool, } impl Drop for PaintableSink { fn drop(&mut self) { let mut paintable = self.paintable.lock().unwrap(); - - // Drop the paintable from the main thread if let Some(paintable) = paintable.take() { - let context = glib::MainContext::default(); - - context.invoke(move || { - drop(paintable); - }); + glib::MainContext::default().invoke(|| drop(paintable)) } } } @@ -69,7 +73,7 @@ impl ObjectImpl for PaintableSink { fn properties() -> &'static [glib::ParamSpec] { static PROPERTIES: Lazy> = Lazy::new(|| { vec![ - glib::ParamSpecObject::builder::("paintable") + glib::ParamSpecObject::builder::("paintable") .nick("Paintable") .blurb("The Paintable the sink renders to") .read_only() @@ -85,14 +89,14 @@ impl ObjectImpl for PaintableSink { "paintable" => { let mut paintable = self.paintable.lock().unwrap(); if paintable.is_none() { - self.obj().initialize_paintable(&mut paintable); + self.create_paintable(&mut paintable); } let paintable = match &*paintable { Some(ref paintable) => paintable, None => { gst::error!(CAT, imp: self, "Failed to create paintable"); - return None::<>k::gdk::Paintable>.to_value(); + return None::<&gdk::Paintable>.to_value(); } }; @@ -105,7 +109,7 @@ impl ObjectImpl for PaintableSink { imp: self, "Can't retrieve Paintable from non-main thread" ); - None::<>k::gdk::Paintable>.to_value() + None::<&gdk::Paintable>.to_value() } } } @@ -139,6 +143,8 @@ impl ElementImpl for PaintableSink { for features in [ None, + Some(&["memory:GLMemory", "meta:GstVideoOverlayComposition"][..]), + Some(&["memory:GLMemory"][..]), Some(&["memory:SystemMemory", "meta:GstVideoOverlayComposition"][..]), Some(&["meta:GstVideoOverlayComposition"][..]), ] { @@ -156,6 +162,12 @@ impl ElementImpl for PaintableSink { c.get_mut() .unwrap() .set_features_simple(Some(gst::CapsFeatures::new(features))); + + if features.contains(&"memory:GLMemory") { + c.get_mut() + .unwrap() + .set_simple(&[("texture-target", &"2D")]) + } } caps.append(c); @@ -182,8 +194,9 @@ impl ElementImpl for PaintableSink { match transition { gst::StateChange::NullToReady => { let mut paintable = self.paintable.lock().unwrap(); + if paintable.is_none() { - self.obj().initialize_paintable(&mut paintable); + self.create_paintable(&mut paintable); } if paintable.is_none() { @@ -209,6 +222,35 @@ impl ElementImpl for PaintableSink { } impl BaseSinkImpl for PaintableSink { + fn caps(&self, filter: Option<&gst::Caps>) -> Option { + let cached_caps = self + .cached_caps + .lock() + .expect("Failed to lock cached caps mutex") + .clone(); + + let mut tmp_caps = cached_caps.unwrap_or_else(|| { + let templ = Self::pad_templates(); + templ[0].caps().clone() + }); + + gst::debug!(CAT, imp: self, "Advertising our own caps: {:?}", &tmp_caps); + + if let Some(filter_caps) = filter { + gst::debug!( + CAT, + imp: self, + "Intersecting with filter caps: {:?}", + &filter_caps + ); + + tmp_caps = filter_caps.intersect_with_mode(&tmp_caps, gst::CapsIntersectMode::First); + }; + + gst::debug!(CAT, imp: self, "Returning caps: {:?}", &tmp_caps); + Some(tmp_caps) + } + fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> { gst::debug!(CAT, imp: self, "Setting caps {:?}", caps); @@ -224,12 +266,113 @@ impl BaseSinkImpl for PaintableSink { &self, query: &mut gst::query::Allocation, ) -> Result<(), gst::LoggableError> { - query.add_allocation_meta::(None); + gst::debug!(CAT, imp: self, "Proposing Allocation query"); + self.parent_propose_allocation(query)?; + + query.add_allocation_meta::(None); // TODO: Provide a preferred "window size" here for higher-resolution rendering query.add_allocation_meta::(None); - self.parent_propose_allocation(query) + { + // Early return if there is no context initialized + let gst_context_guard = self.gst_context.lock().unwrap(); + if gst_context_guard.is_none() { + gst::debug!( + CAT, + imp: self, + "Found no GL Context during propose_allocation." + ); + return Ok(()); + } + } + + // GL specific things + let (caps, need_pool) = query.get_owned(); + + if caps.is_empty() { + return Err(gst::loggable_error!(CAT, "No caps where specified.")); + } + + if let Some(f) = caps.features(0) { + if !f.contains("memory:GLMemory") { + gst::debug!( + CAT, + imp: self, + "No 'memory:GLMemory' feature in caps: {}", + caps + ) + } + } + + let info = gst_video::VideoInfo::from_caps(&caps) + .map_err(|_| gst::loggable_error!(CAT, "Failed to get VideoInfo from caps"))?; + + let size = info.size() as u32; + + { + let gst_context = { self.gst_context.lock().unwrap().clone().unwrap() }; + let buffer_pool = gst_gl::GLBufferPool::new(&gst_context); + + if need_pool { + gst::debug!(CAT, imp: self, "Creating new Pool"); + + let mut config = buffer_pool.config(); + config.set_params(Some(&caps), size, 0, 0); + config.add_option("GstBufferPoolOptionGLSyncMeta"); + + if let Err(err) = buffer_pool.set_config(config) { + return Err(gst::loggable_error!( + CAT, + format!("Failed to set config in the GL BufferPool.: {}", err) + )); + } + } + + // we need at least 2 buffer because we hold on to the last one + query.add_allocation_pool(Some(&buffer_pool), size, 2, 0); + + if gst_context.check_feature("GL_ARB_sync") + || gst_context.check_feature("GL_EXT_EGL_sync") + { + query.add_allocation_meta::(None) + } + } + + Ok(()) + } + + fn query(&self, query: &mut gst::QueryRef) -> bool { + gst::log!(CAT, imp: self, "Handling query {:?}", query); + + match query.view_mut() { + gst::QueryViewMut::Context(q) => { + // Avoid holding the locks while we respond to the query + // The objects are ref-counted anyway. + let gst_display = { self.gst_display.lock().unwrap().clone() }; + if let Some(display) = gst_display { + let (app_ctx, gst_ctx) = { + ( + self.gst_app_context.lock().unwrap().clone(), + self.gst_context.lock().unwrap().clone(), + ) + }; + assert_ne!(app_ctx, None); + assert_ne!(gst_ctx, None); + + return gst_gl::functions::gl_handle_context_query( + &*self.instance(), + q, + Some(&display), + gst_ctx.as_ref(), + app_ctx.as_ref(), + ); + } + + BaseSinkImplExt::parent_query(self, query) + } + _ => BaseSinkImplExt::parent_query(self, query), + } } } @@ -237,16 +380,27 @@ impl VideoSinkImpl for PaintableSink { fn show_frame(&self, buffer: &gst::Buffer) -> Result { gst::trace!(CAT, imp: self, "Rendering buffer {:?}", buffer); + // Empty buffer, nothing to render + if buffer.n_memory() == 0 { + gst::trace!( + CAT, + imp: self, + "Empty buffer, nothing to render. Returning." + ); + return Ok(gst::FlowSuccess::Ok); + }; + let info = self.info.lock().unwrap(); let info = info.as_ref().ok_or_else(|| { gst::error!(CAT, imp: self, "Received no caps yet"); gst::FlowError::NotNegotiated })?; - let frame = Frame::new(buffer, info).map_err(|err| { - gst::error!(CAT, imp: self, "Failed to map video frame"); - err - })?; + let frame = Frame::new(buffer, info, self.have_gl_context.load(Ordering::Relaxed)) + .map_err(|err| { + gst::error!(CAT, imp: self, "Failed to map video frame"); + err + })?; self.pending_frame.lock().unwrap().replace(frame); let sender = self.sender.lock().unwrap(); @@ -263,3 +417,392 @@ impl VideoSinkImpl for PaintableSink { Ok(gst::FlowSuccess::Ok) } } + +impl PaintableSink { + fn pending_frame(&self) -> Option { + self.pending_frame.lock().unwrap().take() + } + + fn do_action(&self, action: SinkEvent) -> glib::Continue { + let paintable = self.paintable.lock().unwrap().clone(); + let paintable = match paintable { + Some(paintable) => paintable, + None => return glib::Continue(false), + }; + + match action { + SinkEvent::FrameChanged => { + gst::trace!(CAT, imp: self, "Frame changed"); + paintable.get().handle_frame_changed(self.pending_frame()) + } + } + + glib::Continue(true) + } + + fn configure_caps(&self) { + let mut tmp_caps = Self::pad_templates()[0].caps().clone(); + + // Filter out GL caps from the template pads if we have no context + if !self.have_gl_context.load(Ordering::Relaxed) { + tmp_caps = tmp_caps + .iter_with_features() + .filter(|(_, features)| !features.contains("memory:GLMemory")) + .map(|(s, c)| (s.to_owned(), c.to_owned())) + .collect::(); + } + + self.cached_caps + .lock() + .expect("Failed to lock Mutex") + .replace(tmp_caps); + } + + fn create_paintable(&self, paintable_storage: &mut MutexGuard>>) { + let ctx = self.realize_context(); + + let ctx = if let Some(c) = ctx { + if let Ok(c) = self.initialize_gl_wrapper(c) { + self.have_gl_context.store(true, Ordering::Relaxed); + Some(c) + } else { + None + } + } else { + None + }; + + self.configure_caps(); + self.initialize_paintable(ctx, paintable_storage); + } + + fn initialize_paintable( + &self, + gl_context: Option>, + paintable_storage: &mut MutexGuard>>, + ) { + gst::debug!(CAT, imp: self, "Initializing paintable"); + + let paintable = utils::invoke_on_main_thread(|| { + // grab the context out of the fragile + let ctx = gl_context.map(|f| f.into_inner()); + Fragile::new(Paintable::new(ctx)) + }); + + // The channel for the SinkEvents + let (sender, receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT); + let sink = self.instance(); + receiver.attach( + None, + glib::clone!( + @weak sink => @default-return glib::Continue(false), + move |action| sink.imp().do_action(action) + ), + ); + + **paintable_storage = Some(paintable); + + *self.sender.lock().unwrap() = Some(sender); + } + + fn realize_context(&self) -> Option> { + gst::debug!(CAT, imp: self, "Realizing GDK GL Context"); + + let weak = self.instance().downgrade(); + let cb = move || -> Option> { + let obj = weak + .upgrade() + .expect("Failed to upgrade Weak ref during gl initialization."); + + gst::debug!(CAT, obj: &obj, "Realizing GDK GL Context from main context"); + + // This can return NULL but only happens in 2 situations: + // * If the function is called before gtk_init + // * If the function is called after gdk_display_close(default_display) + // Both of which are treated as programming errors. + // + // However, when we are building the docs, gtk_init doesn't get called + // and this would cause the documentation generation to error. + // Thus its okayish to return None here and fallback to software + // rendering, since this path isn't going to be used by applications + // anyway. + // + // FIXME: add a couple more gtk_init checks across the codebase where + // applicable since this is no longer going to panic. + let display = gdk::Display::default()?; + let ctx = display.create_gl_context(); + + if let Ok(ctx) = ctx { + gst::info!(CAT, obj: &obj, "Realizing GDK GL Context",); + + if ctx.realize().is_ok() { + gst::info!(CAT, obj: &obj, "Successfully realized GDK GL Context",); + return Some(Fragile::new(ctx)); + } else { + gst::warning!(CAT, obj: &obj, "Failed to realize GDK GL Context",); + } + } else { + gst::warning!(CAT, obj: &obj, "Failed to create GDK GL Context",); + }; + + None + }; + + utils::invoke_on_main_thread(cb) + } + + fn initialize_gl_wrapper( + &self, + context: Fragile, + ) -> Result, glib::Error> { + gst::info!(CAT, imp: self, "Initializing GDK GL Context"); + let self_ = self.instance().clone(); + utils::invoke_on_main_thread(move || self_.imp().initialize_gl(context)) + } + + fn initialize_gl( + &self, + context: Fragile, + ) -> Result, glib::Error> { + let ctx = context.get(); + let display = gtk::prelude::GLContextExt::display(ctx) + .expect("Failed to get GDK Display from GDK Context."); + ctx.make_current(); + + let mut app_ctx_guard = self.gst_app_context.lock().unwrap(); + let mut display_ctx_guard = self.gst_display.lock().unwrap(); + + match ctx.type_().name() { + #[cfg(all(target_os = "linux", feature = "x11egl"))] + "GdkX11GLContextEGL" => { + self.initialize_x11egl(display, &mut display_ctx_guard, &mut app_ctx_guard)?; + } + #[cfg(all(target_os = "linux", feature = "x11glx"))] + "GdkX11GLContextGLX" => { + self.initialize_x11glx(display, &mut display_ctx_guard, &mut app_ctx_guard)?; + } + #[cfg(all(target_os = "linux", feature = "wayland"))] + "GdkWaylandGLContext" => { + self.initialize_waylandegl(display, &mut display_ctx_guard, &mut app_ctx_guard)?; + } + _ => { + gst::error!( + CAT, + imp: self, + "Unsupported GDK display {} for GL", + &display, + ); + return Err(glib::Error::new( + gst::ResourceError::Failed, + &format!("Unsupported GDK display {display} for GL"), + )); + } + }; + + // This should have been initialized once we are done with the platform checks + assert!(app_ctx_guard.is_some()); + + match app_ctx_guard.as_ref().unwrap().activate(true) { + Ok(_) => gst::info!(CAT, imp: self, "Successfully activated GL Context."), + Err(_) => { + gst::error!(CAT, imp: self, "Failed to activate GL context",); + return Err(glib::Error::new( + gst::ResourceError::Failed, + "Failed to activate GL context", + )); + } + }; + + match app_ctx_guard.as_ref().unwrap().fill_info() { + Ok(_) => { + match app_ctx_guard.as_ref().unwrap().activate(true) { + Ok(_) => gst::info!( + CAT, + imp: self, + "Successfully activated GL Context after fill_info" + ), + Err(_) => { + gst::error!(CAT, imp: self, "Failed to activate GL context",); + return Err(glib::Error::new( + gst::ResourceError::Failed, + "Failed to activate GL context after fill_info", + )); + } + }; + } + Err(err) => { + gst::error!( + CAT, + imp: self, + "Failed to fill info on the GL Context: {}", + &err + ); + return Err(err); + } + }; + + match display_ctx_guard + .as_ref() + .unwrap() + .create_context(app_ctx_guard.as_ref().unwrap()) + { + Ok(gst_context) => { + let mut gst_ctx_guard = self.gst_context.lock().unwrap(); + gst::info!(CAT, imp: self, "Successfully initialized GL Context"); + gst_ctx_guard.replace(gst_context); + Ok(context) + } + Err(err) => { + gst::error!(CAT, imp: self, "Could not create GL context: {}", &err); + Err(err) + } + } + } + + #[cfg(all(target_os = "linux", feature = "x11egl"))] + fn initialize_x11egl( + &self, + display: gdk::Display, + display_ctx_guard: &mut Option, + app_ctx_guard: &mut Option, + ) -> Result<(), glib::Error> { + gst::info!( + CAT, + imp: self, + "Initializing GL with for x11 EGL backend and display." + ); + + let platform = gst_gl::GLPlatform::EGL; + let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform); + let gl_ctx = gst_gl::GLContext::current_gl_context(platform); + + if gl_ctx != 0 { + unsafe { + let d = display.downcast::().unwrap(); + let x11_display = gdk_x11::ffi::gdk_x11_display_get_egl_display(d.to_glib_none().0); + assert!(!x11_display.is_null()); + + let gst_display = + gst_gl_egl::ffi::gst_gl_display_egl_new_with_egl_display(x11_display); + assert!(!gst_display.is_null()); + let gst_display: gst_gl::GLDisplay = + from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay); + + let gst_app_context = + gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api); + + assert!(gst_app_context.is_some()); + + display_ctx_guard.replace(gst_display); + app_ctx_guard.replace(gst_app_context.unwrap()); + + Ok(()) + } + } else { + gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",); + Err(glib::Error::new( + gst::ResourceError::Failed, + "Failed to get handle from GdkGLContext", + )) + } + } + + #[cfg(all(target_os = "linux", feature = "x11glx"))] + fn initialize_x11glx( + &self, + display: gdk::Display, + display_ctx_guard: &mut Option, + app_ctx_guard: &mut Option, + ) -> Result<(), glib::Error> { + gst::info!( + CAT, + imp: self, + "Initializing GL with for x11 GLX backend and display." + ); + + let platform = gst_gl::GLPlatform::GLX; + let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform); + let gl_ctx = gst_gl::GLContext::current_gl_context(platform); + + if gl_ctx != 0 { + unsafe { + let d = display.downcast::().unwrap(); + let x11_display = gdk_x11::ffi::gdk_x11_display_get_xdisplay(d.to_glib_none().0); + assert!(!x11_display.is_null()); + + let gst_display = gst_gl_x11::ffi::gst_gl_display_x11_new_with_display(x11_display); + assert!(!gst_display.is_null()); + let gst_display: gst_gl::GLDisplay = + from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay); + + let gst_app_context = + gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api); + + assert!(gst_app_context.is_some()); + + display_ctx_guard.replace(gst_display); + app_ctx_guard.replace(gst_app_context.unwrap()); + + Ok(()) + } + } else { + gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",); + Err(glib::Error::new( + gst::ResourceError::Failed, + "Failed to get handle from GdkGLContext", + )) + } + } + + #[cfg(all(target_os = "linux", feature = "wayland"))] + fn initialize_waylandegl( + &self, + display: gdk::Display, + display_ctx_guard: &mut Option, + app_ctx_guard: &mut Option, + ) -> Result<(), glib::Error> { + gst::info!( + CAT, + imp: self, + "Initializing GL with for Wayland EGL backend and display." + ); + + let platform = gst_gl::GLPlatform::EGL; + let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform); + let gl_ctx = gst_gl::GLContext::current_gl_context(platform); + + // FIXME: bindings + if gl_ctx != 0 { + unsafe { + // let wayland_display = gdk_wayland::WaylandDisplay::wl_display(display.downcast()); + // get the ptr directly since we are going to use it raw + let d = display.downcast::().unwrap(); + let wayland_display = + gdk_wayland::ffi::gdk_wayland_display_get_wl_display(d.to_glib_none().0); + assert!(!wayland_display.is_null()); + + let gst_display = + gst_gl_wayland::ffi::gst_gl_display_wayland_new_with_display(wayland_display); + assert!(!gst_display.is_null()); + let gst_display: gst_gl::GLDisplay = + from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay); + + let gst_app_context = + gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api); + + assert!(gst_app_context.is_some()); + + display_ctx_guard.replace(gst_display); + app_ctx_guard.replace(gst_app_context.unwrap()); + + Ok(()) + } + } else { + gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",); + Err(glib::Error::new( + gst::ResourceError::Failed, + "Failed to get handle from GdkGLContext", + )) + } + } +} diff --git a/video/gtk4/src/sink/mod.rs b/video/gtk4/src/sink/mod.rs index cb17dba1..0a578bda 100644 --- a/video/gtk4/src/sink/mod.rs +++ b/video/gtk4/src/sink/mod.rs @@ -11,19 +11,11 @@ use gtk::glib; use gtk::glib::prelude::*; -use gtk::subclass::prelude::*; - -use fragile::Fragile; - -use std::sync::{mpsc, MutexGuard}; mod frame; mod imp; mod paintable; -use frame::Frame; -use paintable::Paintable; - enum SinkEvent { FrameChanged, } @@ -35,74 +27,7 @@ glib::wrapper! { impl PaintableSink { pub fn new(name: Option<&str>) -> Self { - glib::Object::new(&[("name", &name)]) - } - - fn pending_frame(&self) -> Option { - let imp = self.imp(); - imp.pending_frame.lock().unwrap().take() - } - - fn initialize_paintable( - &self, - paintable_storage: &mut MutexGuard>>, - ) { - gst::debug!(imp::CAT, obj: self, "Initializing paintable"); - - let context = glib::MainContext::default(); - - // The channel for the SinkEvents - let (sender, receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT); - - // This is an one time channel we send into the closure, so we can block until the paintable has been - // created. - let (send, recv) = mpsc::channel(); - context.invoke(glib::clone!( - @weak self as sink => - move || { - let paintable = Fragile::new(SinkPaintable::new()); - send.send(paintable).expect("Somehow we dropped the receiver"); - - receiver.attach( - None, - glib::clone!( - @weak sink => @default-return glib::Continue(false), - move |action| sink.do_action(action) - ), - ); - } - )); - - let paintable = recv.recv().expect("Somehow we dropped the sender"); - - **paintable_storage = Some(paintable); - - let imp = self.imp(); - *imp.sender.lock().unwrap() = Some(sender); - } - - fn do_action(&self, action: SinkEvent) -> glib::Continue { - let imp = self.imp(); - let paintable = imp.paintable.lock().unwrap().clone(); - let paintable = match paintable { - Some(paintable) => paintable, - None => return glib::Continue(false), - }; - - match action { - SinkEvent::FrameChanged => { - gst::trace!(imp::CAT, obj: self, "Frame changed"); - paintable.get().handle_frame_changed(self.pending_frame()) - } - } - - glib::Continue(true) - } -} - -impl Default for PaintableSink { - fn default() -> Self { - PaintableSink::new(None) + glib::Object::builder().property("name", &name).build() } } diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index ca29fce9..f270709f 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use once_cell::sync::Lazy; -pub(super) static CAT: Lazy = Lazy::new(|| { +static CAT: Lazy = Lazy::new(|| { gst::DebugCategory::new( "gstgtk4paintable", gst::DebugColorFlags::empty(), @@ -28,21 +28,51 @@ pub(super) static CAT: Lazy = Lazy::new(|| { ) }); -#[derive(Default)] +#[derive(Default, Debug)] pub struct Paintable { paintables: RefCell>, cached_textures: RefCell>, + gl_context: RefCell>, } #[glib::object_subclass] impl ObjectSubclass for Paintable { const NAME: &'static str = "GstGtk4Paintable"; type Type = super::Paintable; - type ParentType = glib::Object; type Interfaces = (gdk::Paintable,); } -impl ObjectImpl for Paintable {} +impl ObjectImpl for Paintable { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + glib::ParamSpecObject::builder::("gl-context") + .nick("GL Context") + .blurb("GL context to use for rendering") + .construct_only() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + match pspec.name() { + "gl-context" => self.gl_context.borrow().to_value(), + _ => unimplemented!(), + } + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + match pspec.name() { + "gl-context" => { + *self.gl_context.borrow_mut() = value.get::>().unwrap(); + } + _ => unimplemented!(), + } + } +} impl PaintableImpl for Paintable { fn intrinsic_height(&self) -> i32 { @@ -136,10 +166,12 @@ impl PaintableImpl for Paintable { impl Paintable { pub(super) fn handle_frame_changed(&self, frame: Option) { + let context = self.gl_context.borrow(); if let Some(frame) = frame { gst::trace!(CAT, imp: self, "Received new frame"); - let new_paintables = frame.into_textures(&mut self.cached_textures.borrow_mut()); + let new_paintables = + frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()); let new_size = new_paintables .first() .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32)) diff --git a/video/gtk4/src/sink/paintable/mod.rs b/video/gtk4/src/sink/paintable/mod.rs index 4101f63e..56489a37 100644 --- a/video/gtk4/src/sink/paintable/mod.rs +++ b/video/gtk4/src/sink/paintable/mod.rs @@ -22,20 +22,15 @@ glib::wrapper! { } impl Paintable { - pub fn new() -> Self { - glib::Object::new(&[]) - } -} - -impl Default for Paintable { - fn default() -> Self { - Self::new() + pub fn new(context: Option) -> Self { + glib::Object::builder() + .property("gl-context", context) + .build() } } impl Paintable { pub(crate) fn handle_frame_changed(&self, frame: Option) { - let imp = self.imp(); - imp.handle_frame_changed(frame); + self.imp().handle_frame_changed(frame); } } diff --git a/video/gtk4/src/utils.rs b/video/gtk4/src/utils.rs new file mode 100644 index 00000000..293427d2 --- /dev/null +++ b/video/gtk4/src/utils.rs @@ -0,0 +1,16 @@ +use gtk::glib; +use std::sync::mpsc; + +pub(crate) fn invoke_on_main_thread(func: F) -> T +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + let context = glib::MainContext::default(); + + let (send, recv) = mpsc::channel(); + context.invoke(move || { + send.send(func()).expect("Somehow we dropped the receiver"); + }); + recv.recv().expect("Somehow we dropped the sender") +}