video/gtk4: Implement support for GLTextures when possible.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/588>
This commit is contained in:
Jordan Petridis 2021-10-19 09:45:07 +03:00
parent 51c34267a9
commit 975f0141be
14 changed files with 803 additions and 163 deletions

View file

@ -8,7 +8,7 @@ from utils import iterate_plugins
# the csound version used on ci does not ship a .pc file
# threadshare we skip in meson static build as well
IGNORE = ['csound', 'threadshare']
IGNORE = ['csound', 'threadshare', 'gtk4']
outdir = sys.argv[1]

View file

@ -17,11 +17,18 @@ function Run-Tests {
param (
$Features
)
$local_exclude = $exclude_crates;
# In this case the plugin will pull x11/wayland features
# which will fail to build on windows.
if (($Features -eq '--all-features') -or ($Features -eq '')) {
$local_exclude += @("--exclude", "gst-plugin-gtk4")
}
Write-Host "Features: $Features"
Write-Host "Exclude string: $exclude_crates"
Write-Host "Exclude string: $local_exclude"
cargo build --color=always --workspace $exclude_crates --all-targets $Features
cargo build --color=always --workspace $local_exclude --all-targets $Features
if (!$?) {
Write-Host "Build failed"
@ -29,7 +36,7 @@ function Run-Tests {
}
$env:G_DEBUG="fatal_warnings"
cargo test --no-fail-fast --color=always --workspace $exclude_crates --all-targets $Features
cargo test --no-fail-fast --color=always --workspace $local_exclude --all-targets $Features
if (!$?) {
Write-Host "Tests failed"

View file

@ -1940,7 +1940,7 @@
"long-name": "GTK 4 Paintable Sink",
"pad-templates": {
"sink": {
"caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"caps": "video/x-raw:\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n",
"direction": "sink",
"presence": "always"
}

View file

@ -140,7 +140,7 @@ else
message('csound not found, disabling its plugin')
endif
if dependency('gtk4', required : get_option('gtk4')).found()
if dependency('gtk4', version: '>= 4.6.0', required : get_option('gtk4')).found()
plugins += {'gst-plugin-gtk4' : 'libgstgtk4',}
endif
@ -278,7 +278,7 @@ foreach plugin : plugins
)
meson.override_dependency(plugin_name, dep)
if static_build and plugin_name in ['gstcsound', 'gstthreadshare']
if static_build and plugin_name in ['gstcsound', 'gstthreadshare', 'gstgtk4']
warning('Static plugin @0@ is known to fail. It will not be included in libgstreamer-full.'.format(plugin_name))
else
gst_plugins += dep

View file

@ -9,11 +9,18 @@ rust-version = "1.63"
description = "GStreamer GTK 4 Sink element and Paintable widget"
[dependencies]
gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs" }
gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_6"] }
gdk_wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_4"], optional = true}
gdk_x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", features = ["v4_4"], optional = true}
gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] }
gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
gst_video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
gst_gl = { package = "gstreamer-gl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] }
gst_gl_wayland = { package = "gstreamer-gl-wayland", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true }
gst_gl_x11 = { package = "gstreamer-gl-x11", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true }
gst_gl_egl = { package = "gstreamer-gl-egl", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"], optional = true }
once_cell = "1.0"
fragile = "2"
@ -27,7 +34,11 @@ path = "src/lib.rs"
gst-plugin-version-helper = { path="../../version-helper" }
[features]
default = []
static = []
wayland = ["gdk_wayland", "gst_gl_wayland"]
x11glx = ["gdk_x11", "gst_gl_x11"]
x11egl = ["gdk_x11", "gst_gl_egl"]
capi = []
doc = ["gst/v1_18"]

View file

@ -2,3 +2,6 @@
GTK 4 provides `gtk::Video` & `gtk::Picture` for rendering media such as videos. As the default `gtk::Video` widget doesn't
offer the possibility to use a custom `gst::Pipeline`. The plugin provides a `gst_video::VideoSink` along with a `gdk::Paintable` that's capable of rendering the sink's frames.
The Sink can generate GL Textures if the system is capable of it, but it needs to be compiled
with either `wayland`, `x11glx` or `x11egl` cargo features.

View file

@ -6,30 +6,6 @@ use gtk::{gdk, gio, glib};
use std::cell::RefCell;
fn create_ui(app: &gtk::Application) {
let pipeline = gst::Pipeline::default();
let src = gst::ElementFactory::make("videotestsrc").build().unwrap();
let overlay = gst::ElementFactory::make("clockoverlay")
.property("font-desc", "Monospace 42")
.build()
.unwrap();
let sink = gst::ElementFactory::make("gtk4paintablesink")
.build()
.unwrap();
let paintable = sink.property::<gdk::Paintable>("paintable");
pipeline.add_many(&[&src, &overlay, &sink]).unwrap();
src.link_filtered(
&overlay,
&gst_video::VideoCapsBuilder::new()
.width(640)
.height(480)
.build(),
)
.unwrap();
overlay.link(&sink).unwrap();
let window = gtk::ApplicationWindow::new(app);
window.set_default_size(640, 480);
@ -37,6 +13,48 @@ fn create_ui(app: &gtk::Application) {
let picture = gtk::Picture::new();
let label = gtk::Label::new(Some("Position: 00:00:00"));
let pipeline = gst::Pipeline::new(None);
let overlay = gst::ElementFactory::make("clockoverlay")
.property("font-desc", "Monospace 42")
.build()
.unwrap();
let gtksink = gst::ElementFactory::make("gtk4paintablesink")
.build()
.unwrap();
// Need to set state to Ready to get a GL context
gtksink.set_state(gst::State::Ready).unwrap();
let paintable = gtksink.property::<gdk::Paintable>("paintable");
// TODO: future plans to provide a bin-like element that works with less setup
let (src, sink) = if paintable
.property::<Option<gdk::GLContext>>("gl-context")
.is_some()
{
let src = gst::ElementFactory::make("gltestsrc").build().unwrap();
let sink = gst::ElementFactory::make("glsinkbin")
.property("sink", &gtksink)
.build()
.unwrap();
(src, sink)
} else {
let src = gst::ElementFactory::make("videotestsrc").build().unwrap();
(src, gtksink)
};
pipeline.add_many(&[&src, &overlay, &sink]).unwrap();
let caps = gst_video::VideoCapsBuilder::new()
.width(640)
.height(480)
.any_features()
.build();
src.link_filtered(&overlay, &caps).unwrap();
overlay.link(&sink).unwrap();
picture.set_paintable(Some(&paintable));
vbox.append(&picture);
vbox.append(&label);
@ -115,12 +133,10 @@ fn main() {
gstgtk4::plugin_register_static().expect("Failed to register gstgtk4 plugin");
{
let app = gtk::Application::new(None, gio::ApplicationFlags::FLAGS_NONE);
app.connect_activate(create_ui);
app.run();
}
unsafe {
gst::deinit();

View file

@ -18,6 +18,7 @@
use gst::glib;
mod sink;
mod utils;
pub use sink::PaintableSink;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {

View file

@ -9,7 +9,7 @@
//
// SPDX-License-Identifier: MPL-2.0
use gtk::prelude::*;
use gst_gl::prelude::*;
use gtk::{gdk, glib};
use std::collections::{HashMap, HashSet};
@ -17,6 +17,7 @@ use std::collections::{HashMap, HashSet};
pub(crate) struct Frame {
frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
overlays: Vec<Overlay>,
gst_context: Option<gst_gl::GLContext>,
}
#[derive(Debug)]
@ -31,7 +32,6 @@ struct Overlay {
#[derive(Debug)]
pub(crate) struct Texture {
//FIXME: create getters instead of having the fields public
pub texture: gdk::Texture,
pub x: f32,
pub y: f32,
@ -90,15 +90,63 @@ fn video_frame_to_memory_texture(
(texture, pixel_aspect_ratio)
}
fn video_frame_to_gl_texture(
frame: &gst_video::VideoFrame<gst_video::video_frame::Readable>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
used_textures: &mut HashSet<usize>,
gdk_context: &gdk::GLContext,
gst_context: &gst_gl::GLContext,
) -> (gdk::Texture, f64) {
let texture_id = frame.texture_id(0).expect("Invalid texture id") as usize;
let pixel_aspect_ratio =
(frame.info().par().numer() as f64) / (frame.info().par().denom() as f64);
if let Some(texture) = cached_textures.get(&(texture_id)) {
used_textures.insert(texture_id);
return (texture.clone(), pixel_aspect_ratio);
}
let width = frame.width();
let height = frame.height();
let sync_meta = frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
sync_meta.wait(gst_context);
let texture = unsafe {
gdk::GLTexture::new(gdk_context, texture_id as u32, width as i32, height as i32)
.upcast::<gdk::Texture>()
};
cached_textures.insert(texture_id, texture.clone());
used_textures.insert(texture_id);
(texture, pixel_aspect_ratio)
}
impl Frame {
pub fn into_textures(self, cached_textures: &mut HashMap<usize, gdk::Texture>) -> Vec<Texture> {
pub(crate) fn into_textures(
self,
gdk_context: Option<&gdk::GLContext>,
cached_textures: &mut HashMap<usize, gdk::Texture>,
) -> Vec<Texture> {
let mut textures = Vec::with_capacity(1 + self.overlays.len());
let mut used_textures = HashSet::with_capacity(1 + self.overlays.len());
let width = self.frame.width();
let height = self.frame.height();
let (texture, pixel_aspect_ratio) =
video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures);
if let (Some(gdk_ctx), Some(gst_ctx)) = (gdk_context, self.gst_context.as_ref()) {
video_frame_to_gl_texture(
&self.frame,
cached_textures,
&mut used_textures,
gdk_ctx,
gst_ctx,
)
} else {
video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures)
};
textures.push(Texture {
texture,
@ -131,9 +179,48 @@ impl Frame {
}
impl Frame {
pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Result<Self, gst::FlowError> {
let frame = gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?;
pub(crate) fn new(
buffer: &gst::Buffer,
info: &gst_video::VideoInfo,
have_gl_context: bool,
) -> Result<Self, gst::FlowError> {
let mut gst_context = None;
// Empty buffers get filtered out in show_frame
debug_assert!(buffer.n_memory() > 0);
let is_buffer_gl = buffer
.peek_memory(0)
.downcast_memory_ref::<gst_gl::GLBaseMemory>()
.is_some();
let frame = if !is_buffer_gl || !have_gl_context {
gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?
} else {
let gst_ctx = buffer
.peek_memory(0)
.downcast_memory_ref::<gst_gl::GLBaseMemory>()
.map(|m| m.context())
.expect("Failed to retrieve the GstGL Context.");
gst_context = Some(gst_ctx.clone());
if let Some(meta) = buffer.meta::<gst_gl::GLSyncMeta>() {
meta.set_sync_point(gst_ctx);
gst_video::VideoFrame::from_buffer_readable_gl(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?
} else {
let mut buffer = buffer.clone();
{
let buffer = buffer.make_mut();
let meta = gst_gl::GLSyncMeta::add(buffer, gst_ctx);
meta.set_sync_point(gst_ctx);
}
gst_video::VideoFrame::from_buffer_readable_gl(buffer, info)
.map_err(|_| gst::FlowError::Error)?
}
};
let overlays = frame
.buffer()
@ -171,6 +258,10 @@ impl Frame {
})
.collect();
Ok(Self { frame, overlays })
Ok(Self {
frame,
overlays,
gst_context,
})
}
}

View file

@ -13,21 +13,26 @@ use super::SinkEvent;
use crate::sink::frame::Frame;
use crate::sink::paintable::Paintable;
use glib::prelude::*;
use glib::translate::*;
use glib::Sender;
use gtk::prelude::*;
use gtk::{gdk, glib};
use gst::prelude::*;
use gst::subclass::prelude::*;
use gst_base::subclass::prelude::*;
use gst_gl::prelude::GLContextExt as GstGLContextExt;
use gst_gl::prelude::*;
use gst_video::subclass::prelude::*;
use gtk::glib;
use once_cell::sync::Lazy;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, MutexGuard};
use crate::utils;
use fragile::Fragile;
pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"gstgtk4paintablesink",
gst::DebugColorFlags::empty(),
@ -35,25 +40,24 @@ pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
)
});
#[derive(Default)]
#[derive(Default, Debug)]
pub struct PaintableSink {
pub(super) paintable: Mutex<Option<Fragile<Paintable>>>,
paintable: Mutex<Option<Fragile<Paintable>>>,
info: Mutex<Option<gst_video::VideoInfo>>,
pub(super) sender: Mutex<Option<Sender<SinkEvent>>>,
pub(super) pending_frame: Mutex<Option<Frame>>,
sender: Mutex<Option<Sender<SinkEvent>>>,
pending_frame: Mutex<Option<Frame>>,
gst_display: Mutex<Option<gst_gl::GLDisplay>>,
gst_app_context: Mutex<Option<gst_gl::GLContext>>,
gst_context: Mutex<Option<gst_gl::GLContext>>,
cached_caps: Mutex<Option<gst::Caps>>,
have_gl_context: AtomicBool,
}
impl Drop for PaintableSink {
fn drop(&mut self) {
let mut paintable = self.paintable.lock().unwrap();
// Drop the paintable from the main thread
if let Some(paintable) = paintable.take() {
let context = glib::MainContext::default();
context.invoke(move || {
drop(paintable);
});
glib::MainContext::default().invoke(|| drop(paintable))
}
}
}
@ -69,7 +73,7 @@ impl ObjectImpl for PaintableSink {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![
glib::ParamSpecObject::builder::<gtk::gdk::Paintable>("paintable")
glib::ParamSpecObject::builder::<gdk::Paintable>("paintable")
.nick("Paintable")
.blurb("The Paintable the sink renders to")
.read_only()
@ -85,14 +89,14 @@ impl ObjectImpl for PaintableSink {
"paintable" => {
let mut paintable = self.paintable.lock().unwrap();
if paintable.is_none() {
self.obj().initialize_paintable(&mut paintable);
self.create_paintable(&mut paintable);
}
let paintable = match &*paintable {
Some(ref paintable) => paintable,
None => {
gst::error!(CAT, imp: self, "Failed to create paintable");
return None::<&gtk::gdk::Paintable>.to_value();
return None::<&gdk::Paintable>.to_value();
}
};
@ -105,7 +109,7 @@ impl ObjectImpl for PaintableSink {
imp: self,
"Can't retrieve Paintable from non-main thread"
);
None::<&gtk::gdk::Paintable>.to_value()
None::<&gdk::Paintable>.to_value()
}
}
}
@ -139,6 +143,8 @@ impl ElementImpl for PaintableSink {
for features in [
None,
Some(&["memory:GLMemory", "meta:GstVideoOverlayComposition"][..]),
Some(&["memory:GLMemory"][..]),
Some(&["memory:SystemMemory", "meta:GstVideoOverlayComposition"][..]),
Some(&["meta:GstVideoOverlayComposition"][..]),
] {
@ -156,6 +162,12 @@ impl ElementImpl for PaintableSink {
c.get_mut()
.unwrap()
.set_features_simple(Some(gst::CapsFeatures::new(features)));
if features.contains(&"memory:GLMemory") {
c.get_mut()
.unwrap()
.set_simple(&[("texture-target", &"2D")])
}
}
caps.append(c);
@ -182,8 +194,9 @@ impl ElementImpl for PaintableSink {
match transition {
gst::StateChange::NullToReady => {
let mut paintable = self.paintable.lock().unwrap();
if paintable.is_none() {
self.obj().initialize_paintable(&mut paintable);
self.create_paintable(&mut paintable);
}
if paintable.is_none() {
@ -209,6 +222,35 @@ impl ElementImpl for PaintableSink {
}
impl BaseSinkImpl for PaintableSink {
fn caps(&self, filter: Option<&gst::Caps>) -> Option<gst::Caps> {
let cached_caps = self
.cached_caps
.lock()
.expect("Failed to lock cached caps mutex")
.clone();
let mut tmp_caps = cached_caps.unwrap_or_else(|| {
let templ = Self::pad_templates();
templ[0].caps().clone()
});
gst::debug!(CAT, imp: self, "Advertising our own caps: {:?}", &tmp_caps);
if let Some(filter_caps) = filter {
gst::debug!(
CAT,
imp: self,
"Intersecting with filter caps: {:?}",
&filter_caps
);
tmp_caps = filter_caps.intersect_with_mode(&tmp_caps, gst::CapsIntersectMode::First);
};
gst::debug!(CAT, imp: self, "Returning caps: {:?}", &tmp_caps);
Some(tmp_caps)
}
fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> {
gst::debug!(CAT, imp: self, "Setting caps {:?}", caps);
@ -224,12 +266,113 @@ impl BaseSinkImpl for PaintableSink {
&self,
query: &mut gst::query::Allocation,
) -> Result<(), gst::LoggableError> {
query.add_allocation_meta::<gst_video::VideoMeta>(None);
gst::debug!(CAT, imp: self, "Proposing Allocation query");
self.parent_propose_allocation(query)?;
query.add_allocation_meta::<gst_video::VideoMeta>(None);
// TODO: Provide a preferred "window size" here for higher-resolution rendering
query.add_allocation_meta::<gst_video::VideoOverlayCompositionMeta>(None);
self.parent_propose_allocation(query)
{
// Early return if there is no context initialized
let gst_context_guard = self.gst_context.lock().unwrap();
if gst_context_guard.is_none() {
gst::debug!(
CAT,
imp: self,
"Found no GL Context during propose_allocation."
);
return Ok(());
}
}
// GL specific things
let (caps, need_pool) = query.get_owned();
if caps.is_empty() {
return Err(gst::loggable_error!(CAT, "No caps where specified."));
}
if let Some(f) = caps.features(0) {
if !f.contains("memory:GLMemory") {
gst::debug!(
CAT,
imp: self,
"No 'memory:GLMemory' feature in caps: {}",
caps
)
}
}
let info = gst_video::VideoInfo::from_caps(&caps)
.map_err(|_| gst::loggable_error!(CAT, "Failed to get VideoInfo from caps"))?;
let size = info.size() as u32;
{
let gst_context = { self.gst_context.lock().unwrap().clone().unwrap() };
let buffer_pool = gst_gl::GLBufferPool::new(&gst_context);
if need_pool {
gst::debug!(CAT, imp: self, "Creating new Pool");
let mut config = buffer_pool.config();
config.set_params(Some(&caps), size, 0, 0);
config.add_option("GstBufferPoolOptionGLSyncMeta");
if let Err(err) = buffer_pool.set_config(config) {
return Err(gst::loggable_error!(
CAT,
format!("Failed to set config in the GL BufferPool.: {}", err)
));
}
}
// we need at least 2 buffer because we hold on to the last one
query.add_allocation_pool(Some(&buffer_pool), size, 2, 0);
if gst_context.check_feature("GL_ARB_sync")
|| gst_context.check_feature("GL_EXT_EGL_sync")
{
query.add_allocation_meta::<gst_gl::GLSyncMeta>(None)
}
}
Ok(())
}
fn query(&self, query: &mut gst::QueryRef) -> bool {
gst::log!(CAT, imp: self, "Handling query {:?}", query);
match query.view_mut() {
gst::QueryViewMut::Context(q) => {
// Avoid holding the locks while we respond to the query
// The objects are ref-counted anyway.
let gst_display = { self.gst_display.lock().unwrap().clone() };
if let Some(display) = gst_display {
let (app_ctx, gst_ctx) = {
(
self.gst_app_context.lock().unwrap().clone(),
self.gst_context.lock().unwrap().clone(),
)
};
assert_ne!(app_ctx, None);
assert_ne!(gst_ctx, None);
return gst_gl::functions::gl_handle_context_query(
&*self.instance(),
q,
Some(&display),
gst_ctx.as_ref(),
app_ctx.as_ref(),
);
}
BaseSinkImplExt::parent_query(self, query)
}
_ => BaseSinkImplExt::parent_query(self, query),
}
}
}
@ -237,13 +380,24 @@ impl VideoSinkImpl for PaintableSink {
fn show_frame(&self, buffer: &gst::Buffer) -> Result<gst::FlowSuccess, gst::FlowError> {
gst::trace!(CAT, imp: self, "Rendering buffer {:?}", buffer);
// Empty buffer, nothing to render
if buffer.n_memory() == 0 {
gst::trace!(
CAT,
imp: self,
"Empty buffer, nothing to render. Returning."
);
return Ok(gst::FlowSuccess::Ok);
};
let info = self.info.lock().unwrap();
let info = info.as_ref().ok_or_else(|| {
gst::error!(CAT, imp: self, "Received no caps yet");
gst::FlowError::NotNegotiated
})?;
let frame = Frame::new(buffer, info).map_err(|err| {
let frame = Frame::new(buffer, info, self.have_gl_context.load(Ordering::Relaxed))
.map_err(|err| {
gst::error!(CAT, imp: self, "Failed to map video frame");
err
})?;
@ -263,3 +417,392 @@ impl VideoSinkImpl for PaintableSink {
Ok(gst::FlowSuccess::Ok)
}
}
impl PaintableSink {
fn pending_frame(&self) -> Option<Frame> {
self.pending_frame.lock().unwrap().take()
}
fn do_action(&self, action: SinkEvent) -> glib::Continue {
let paintable = self.paintable.lock().unwrap().clone();
let paintable = match paintable {
Some(paintable) => paintable,
None => return glib::Continue(false),
};
match action {
SinkEvent::FrameChanged => {
gst::trace!(CAT, imp: self, "Frame changed");
paintable.get().handle_frame_changed(self.pending_frame())
}
}
glib::Continue(true)
}
fn configure_caps(&self) {
let mut tmp_caps = Self::pad_templates()[0].caps().clone();
// Filter out GL caps from the template pads if we have no context
if !self.have_gl_context.load(Ordering::Relaxed) {
tmp_caps = tmp_caps
.iter_with_features()
.filter(|(_, features)| !features.contains("memory:GLMemory"))
.map(|(s, c)| (s.to_owned(), c.to_owned()))
.collect::<gst::Caps>();
}
self.cached_caps
.lock()
.expect("Failed to lock Mutex")
.replace(tmp_caps);
}
fn create_paintable(&self, paintable_storage: &mut MutexGuard<Option<Fragile<Paintable>>>) {
let ctx = self.realize_context();
let ctx = if let Some(c) = ctx {
if let Ok(c) = self.initialize_gl_wrapper(c) {
self.have_gl_context.store(true, Ordering::Relaxed);
Some(c)
} else {
None
}
} else {
None
};
self.configure_caps();
self.initialize_paintable(ctx, paintable_storage);
}
fn initialize_paintable(
&self,
gl_context: Option<Fragile<gdk::GLContext>>,
paintable_storage: &mut MutexGuard<Option<Fragile<Paintable>>>,
) {
gst::debug!(CAT, imp: self, "Initializing paintable");
let paintable = utils::invoke_on_main_thread(|| {
// grab the context out of the fragile
let ctx = gl_context.map(|f| f.into_inner());
Fragile::new(Paintable::new(ctx))
});
// The channel for the SinkEvents
let (sender, receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT);
let sink = self.instance();
receiver.attach(
None,
glib::clone!(
@weak sink => @default-return glib::Continue(false),
move |action| sink.imp().do_action(action)
),
);
**paintable_storage = Some(paintable);
*self.sender.lock().unwrap() = Some(sender);
}
fn realize_context(&self) -> Option<Fragile<gdk::GLContext>> {
gst::debug!(CAT, imp: self, "Realizing GDK GL Context");
let weak = self.instance().downgrade();
let cb = move || -> Option<Fragile<gdk::GLContext>> {
let obj = weak
.upgrade()
.expect("Failed to upgrade Weak ref during gl initialization.");
gst::debug!(CAT, obj: &obj, "Realizing GDK GL Context from main context");
// This can return NULL but only happens in 2 situations:
// * If the function is called before gtk_init
// * If the function is called after gdk_display_close(default_display)
// Both of which are treated as programming errors.
//
// However, when we are building the docs, gtk_init doesn't get called
// and this would cause the documentation generation to error.
// Thus its okayish to return None here and fallback to software
// rendering, since this path isn't going to be used by applications
// anyway.
//
// FIXME: add a couple more gtk_init checks across the codebase where
// applicable since this is no longer going to panic.
let display = gdk::Display::default()?;
let ctx = display.create_gl_context();
if let Ok(ctx) = ctx {
gst::info!(CAT, obj: &obj, "Realizing GDK GL Context",);
if ctx.realize().is_ok() {
gst::info!(CAT, obj: &obj, "Successfully realized GDK GL Context",);
return Some(Fragile::new(ctx));
} else {
gst::warning!(CAT, obj: &obj, "Failed to realize GDK GL Context",);
}
} else {
gst::warning!(CAT, obj: &obj, "Failed to create GDK GL Context",);
};
None
};
utils::invoke_on_main_thread(cb)
}
fn initialize_gl_wrapper(
&self,
context: Fragile<gdk::GLContext>,
) -> Result<Fragile<gdk::GLContext>, glib::Error> {
gst::info!(CAT, imp: self, "Initializing GDK GL Context");
let self_ = self.instance().clone();
utils::invoke_on_main_thread(move || self_.imp().initialize_gl(context))
}
fn initialize_gl(
&self,
context: Fragile<gdk::GLContext>,
) -> Result<Fragile<gdk::GLContext>, glib::Error> {
let ctx = context.get();
let display = gtk::prelude::GLContextExt::display(ctx)
.expect("Failed to get GDK Display from GDK Context.");
ctx.make_current();
let mut app_ctx_guard = self.gst_app_context.lock().unwrap();
let mut display_ctx_guard = self.gst_display.lock().unwrap();
match ctx.type_().name() {
#[cfg(all(target_os = "linux", feature = "x11egl"))]
"GdkX11GLContextEGL" => {
self.initialize_x11egl(display, &mut display_ctx_guard, &mut app_ctx_guard)?;
}
#[cfg(all(target_os = "linux", feature = "x11glx"))]
"GdkX11GLContextGLX" => {
self.initialize_x11glx(display, &mut display_ctx_guard, &mut app_ctx_guard)?;
}
#[cfg(all(target_os = "linux", feature = "wayland"))]
"GdkWaylandGLContext" => {
self.initialize_waylandegl(display, &mut display_ctx_guard, &mut app_ctx_guard)?;
}
_ => {
gst::error!(
CAT,
imp: self,
"Unsupported GDK display {} for GL",
&display,
);
return Err(glib::Error::new(
gst::ResourceError::Failed,
&format!("Unsupported GDK display {display} for GL"),
));
}
};
// This should have been initialized once we are done with the platform checks
assert!(app_ctx_guard.is_some());
match app_ctx_guard.as_ref().unwrap().activate(true) {
Ok(_) => gst::info!(CAT, imp: self, "Successfully activated GL Context."),
Err(_) => {
gst::error!(CAT, imp: self, "Failed to activate GL context",);
return Err(glib::Error::new(
gst::ResourceError::Failed,
"Failed to activate GL context",
));
}
};
match app_ctx_guard.as_ref().unwrap().fill_info() {
Ok(_) => {
match app_ctx_guard.as_ref().unwrap().activate(true) {
Ok(_) => gst::info!(
CAT,
imp: self,
"Successfully activated GL Context after fill_info"
),
Err(_) => {
gst::error!(CAT, imp: self, "Failed to activate GL context",);
return Err(glib::Error::new(
gst::ResourceError::Failed,
"Failed to activate GL context after fill_info",
));
}
};
}
Err(err) => {
gst::error!(
CAT,
imp: self,
"Failed to fill info on the GL Context: {}",
&err
);
return Err(err);
}
};
match display_ctx_guard
.as_ref()
.unwrap()
.create_context(app_ctx_guard.as_ref().unwrap())
{
Ok(gst_context) => {
let mut gst_ctx_guard = self.gst_context.lock().unwrap();
gst::info!(CAT, imp: self, "Successfully initialized GL Context");
gst_ctx_guard.replace(gst_context);
Ok(context)
}
Err(err) => {
gst::error!(CAT, imp: self, "Could not create GL context: {}", &err);
Err(err)
}
}
}
#[cfg(all(target_os = "linux", feature = "x11egl"))]
fn initialize_x11egl(
&self,
display: gdk::Display,
display_ctx_guard: &mut Option<gst_gl::GLDisplay>,
app_ctx_guard: &mut Option<gst_gl::GLContext>,
) -> Result<(), glib::Error> {
gst::info!(
CAT,
imp: self,
"Initializing GL with for x11 EGL backend and display."
);
let platform = gst_gl::GLPlatform::EGL;
let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform);
let gl_ctx = gst_gl::GLContext::current_gl_context(platform);
if gl_ctx != 0 {
unsafe {
let d = display.downcast::<gdk_x11::X11Display>().unwrap();
let x11_display = gdk_x11::ffi::gdk_x11_display_get_egl_display(d.to_glib_none().0);
assert!(!x11_display.is_null());
let gst_display =
gst_gl_egl::ffi::gst_gl_display_egl_new_with_egl_display(x11_display);
assert!(!gst_display.is_null());
let gst_display: gst_gl::GLDisplay =
from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay);
let gst_app_context =
gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api);
assert!(gst_app_context.is_some());
display_ctx_guard.replace(gst_display);
app_ctx_guard.replace(gst_app_context.unwrap());
Ok(())
}
} else {
gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",);
Err(glib::Error::new(
gst::ResourceError::Failed,
"Failed to get handle from GdkGLContext",
))
}
}
#[cfg(all(target_os = "linux", feature = "x11glx"))]
fn initialize_x11glx(
&self,
display: gdk::Display,
display_ctx_guard: &mut Option<gst_gl::GLDisplay>,
app_ctx_guard: &mut Option<gst_gl::GLContext>,
) -> Result<(), glib::Error> {
gst::info!(
CAT,
imp: self,
"Initializing GL with for x11 GLX backend and display."
);
let platform = gst_gl::GLPlatform::GLX;
let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform);
let gl_ctx = gst_gl::GLContext::current_gl_context(platform);
if gl_ctx != 0 {
unsafe {
let d = display.downcast::<gdk_x11::X11Display>().unwrap();
let x11_display = gdk_x11::ffi::gdk_x11_display_get_xdisplay(d.to_glib_none().0);
assert!(!x11_display.is_null());
let gst_display = gst_gl_x11::ffi::gst_gl_display_x11_new_with_display(x11_display);
assert!(!gst_display.is_null());
let gst_display: gst_gl::GLDisplay =
from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay);
let gst_app_context =
gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api);
assert!(gst_app_context.is_some());
display_ctx_guard.replace(gst_display);
app_ctx_guard.replace(gst_app_context.unwrap());
Ok(())
}
} else {
gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",);
Err(glib::Error::new(
gst::ResourceError::Failed,
"Failed to get handle from GdkGLContext",
))
}
}
#[cfg(all(target_os = "linux", feature = "wayland"))]
fn initialize_waylandegl(
&self,
display: gdk::Display,
display_ctx_guard: &mut Option<gst_gl::GLDisplay>,
app_ctx_guard: &mut Option<gst_gl::GLContext>,
) -> Result<(), glib::Error> {
gst::info!(
CAT,
imp: self,
"Initializing GL with for Wayland EGL backend and display."
);
let platform = gst_gl::GLPlatform::EGL;
let (gl_api, _, _) = gst_gl::GLContext::current_gl_api(platform);
let gl_ctx = gst_gl::GLContext::current_gl_context(platform);
// FIXME: bindings
if gl_ctx != 0 {
unsafe {
// let wayland_display = gdk_wayland::WaylandDisplay::wl_display(display.downcast());
// get the ptr directly since we are going to use it raw
let d = display.downcast::<gdk_wayland::WaylandDisplay>().unwrap();
let wayland_display =
gdk_wayland::ffi::gdk_wayland_display_get_wl_display(d.to_glib_none().0);
assert!(!wayland_display.is_null());
let gst_display =
gst_gl_wayland::ffi::gst_gl_display_wayland_new_with_display(wayland_display);
assert!(!gst_display.is_null());
let gst_display: gst_gl::GLDisplay =
from_glib_full(gst_display as *mut gst_gl::ffi::GstGLDisplay);
let gst_app_context =
gst_gl::GLContext::new_wrapped(&gst_display, gl_ctx, platform, gl_api);
assert!(gst_app_context.is_some());
display_ctx_guard.replace(gst_display);
app_ctx_guard.replace(gst_app_context.unwrap());
Ok(())
}
} else {
gst::error!(CAT, imp: self, "Failed to get handle from GdkGLContext",);
Err(glib::Error::new(
gst::ResourceError::Failed,
"Failed to get handle from GdkGLContext",
))
}
}
}

View file

@ -11,19 +11,11 @@
use gtk::glib;
use gtk::glib::prelude::*;
use gtk::subclass::prelude::*;
use fragile::Fragile;
use std::sync::{mpsc, MutexGuard};
mod frame;
mod imp;
mod paintable;
use frame::Frame;
use paintable::Paintable;
enum SinkEvent {
FrameChanged,
}
@ -35,74 +27,7 @@ glib::wrapper! {
impl PaintableSink {
pub fn new(name: Option<&str>) -> Self {
glib::Object::new(&[("name", &name)])
}
fn pending_frame(&self) -> Option<Frame> {
let imp = self.imp();
imp.pending_frame.lock().unwrap().take()
}
fn initialize_paintable(
&self,
paintable_storage: &mut MutexGuard<Option<Fragile<SinkPaintable>>>,
) {
gst::debug!(imp::CAT, obj: self, "Initializing paintable");
let context = glib::MainContext::default();
// The channel for the SinkEvents
let (sender, receiver) = glib::MainContext::channel(glib::PRIORITY_DEFAULT);
// This is an one time channel we send into the closure, so we can block until the paintable has been
// created.
let (send, recv) = mpsc::channel();
context.invoke(glib::clone!(
@weak self as sink =>
move || {
let paintable = Fragile::new(SinkPaintable::new());
send.send(paintable).expect("Somehow we dropped the receiver");
receiver.attach(
None,
glib::clone!(
@weak sink => @default-return glib::Continue(false),
move |action| sink.do_action(action)
),
);
}
));
let paintable = recv.recv().expect("Somehow we dropped the sender");
**paintable_storage = Some(paintable);
let imp = self.imp();
*imp.sender.lock().unwrap() = Some(sender);
}
fn do_action(&self, action: SinkEvent) -> glib::Continue {
let imp = self.imp();
let paintable = imp.paintable.lock().unwrap().clone();
let paintable = match paintable {
Some(paintable) => paintable,
None => return glib::Continue(false),
};
match action {
SinkEvent::FrameChanged => {
gst::trace!(imp::CAT, obj: self, "Frame changed");
paintable.get().handle_frame_changed(self.pending_frame())
}
}
glib::Continue(true)
}
}
impl Default for PaintableSink {
fn default() -> Self {
PaintableSink::new(None)
glib::Object::builder().property("name", &name).build()
}
}

View file

@ -20,7 +20,7 @@ use std::collections::HashMap;
use once_cell::sync::Lazy;
pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"gstgtk4paintable",
gst::DebugColorFlags::empty(),
@ -28,21 +28,51 @@ pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
)
});
#[derive(Default)]
#[derive(Default, Debug)]
pub struct Paintable {
paintables: RefCell<Vec<Texture>>,
cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
gl_context: RefCell<Option<gdk::GLContext>>,
}
#[glib::object_subclass]
impl ObjectSubclass for Paintable {
const NAME: &'static str = "GstGtk4Paintable";
type Type = super::Paintable;
type ParentType = glib::Object;
type Interfaces = (gdk::Paintable,);
}
impl ObjectImpl for Paintable {}
impl ObjectImpl for Paintable {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![
glib::ParamSpecObject::builder::<gdk::GLContext>("gl-context")
.nick("GL Context")
.blurb("GL context to use for rendering")
.construct_only()
.build(),
]
});
PROPERTIES.as_ref()
}
fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
match pspec.name() {
"gl-context" => self.gl_context.borrow().to_value(),
_ => unimplemented!(),
}
}
fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) {
match pspec.name() {
"gl-context" => {
*self.gl_context.borrow_mut() = value.get::<Option<gtk::gdk::GLContext>>().unwrap();
}
_ => unimplemented!(),
}
}
}
impl PaintableImpl for Paintable {
fn intrinsic_height(&self) -> i32 {
@ -136,10 +166,12 @@ impl PaintableImpl for Paintable {
impl Paintable {
pub(super) fn handle_frame_changed(&self, frame: Option<Frame>) {
let context = self.gl_context.borrow();
if let Some(frame) = frame {
gst::trace!(CAT, imp: self, "Received new frame");
let new_paintables = frame.into_textures(&mut self.cached_textures.borrow_mut());
let new_paintables =
frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut());
let new_size = new_paintables
.first()
.map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))

View file

@ -22,20 +22,15 @@ glib::wrapper! {
}
impl Paintable {
pub fn new() -> Self {
glib::Object::new(&[])
}
}
impl Default for Paintable {
fn default() -> Self {
Self::new()
pub fn new(context: Option<gdk::GLContext>) -> Self {
glib::Object::builder()
.property("gl-context", context)
.build()
}
}
impl Paintable {
pub(crate) fn handle_frame_changed(&self, frame: Option<Frame>) {
let imp = self.imp();
imp.handle_frame_changed(frame);
self.imp().handle_frame_changed(frame);
}
}

16
video/gtk4/src/utils.rs Normal file
View file

@ -0,0 +1,16 @@
use gtk::glib;
use std::sync::mpsc;
pub(crate) fn invoke_on_main_thread<F, T>(func: F) -> T
where
F: FnOnce() -> T + Send + 'static,
T: Send + 'static,
{
let context = glib::MainContext::default();
let (send, recv) = mpsc::channel();
context.invoke(move || {
send.send(func()).expect("Somehow we dropped the receiver");
});
recv.recv().expect("Somehow we dropped the sender")
}