From f5bafe5a07585918b342db8e742863ac787a5f4a Mon Sep 17 00:00:00 2001 From: Marijn Suijten Date: Thu, 18 Jul 2024 23:37:48 +0200 Subject: [PATCH] examples/glupload: Provide new shared `GLContext` via pad probe On certain GL drivers on Windows (e.g. those for Intel Arc) we see that creation of a shared context based on our wrapped WGL context (from `glutin`) fails with `ERROR_BUSY`: 0:00:00.509113900 29460 000001E07A782CC0 DEBUG glcontext gstglcontext.c:1227:gst_gl_context_create_thread: Creating thread 0:00:00.509352100 29460 000001E07A782CC0 FIXME glcontext gstglcontext.c:2041:gst_gl_wrapped_context_get_config: wrapped context could not retrieve config. The application may be missing a call to gst_gl_context_fill_info() or the specific platform implemention is not implemented for retrieving the config from a wrapped OpenGL context. 0:00:00.543157300 29460 000001E07A782CC0 INFO glcontext gstglcontext_wgl.c:402:gst_gl_context_wgl_choose_format: chosen config gst-gl-context-config, platform=(GstGLPlatform)GST_GL_PLATFORM_WGL, red-size=(int)8, blue-size=(int)8, green-size=(int)8, alpha-size=(int)8, depth-size=(int)24, stencil-size=(int)8, native-visual-id=(uint)5, surface-type=(GstGLConfigSurfaceType)GST_GL_CONFIG_SURFACE_TYPE_WINDOW; 0:00:00.543770000 29460 000001E07A782CC0 INFO glcontext gstglcontext.c:1322:gst_gl_context_create_thread: Attempting to create opengl context. user chosen api(s) (any), compiled api support (opengl opengl3) display api (any) 0:00:00.553201100 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:186:gst_gl_context_wgl_create_context: gl context created: 65537 0:00:00.613589500 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:214:gst_gl_context_wgl_create_context: Available WGL extensions WGL_EXT_depth_float WGL_ARB_buffer_region WGL_ARB_extensions_string WGL_ARB_make_current_read WGL_ARB_pixel_format WGL_ARB_pbuffer WGL_EXT_extensions_string WGL_EXT_swap_control WGL_ARB_multisample WGL_ARB_pixel_format_float WGL_ARB_framebuffer_sRGB WGL_ARB_create_context WGL_ARB_create_context_profile WGL_EXT_pixel_format_packed_float WGL_EXT_create_context_es_profile WGL_EXT_create_context_es2_profile WGL_NV_DX_interop WGL_NV_DX_interop2 WGL_ARB_robustness_application_isolation WGL_ARB_robustness_share_group_isolation WGL_ARB_create_context_robustness WGL_ARB_context_flush_control 0:00:00.614036500 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.5 context 0:00:00.631649700 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.4 context 0:00:00.650484600 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.3 context 0:00:00.669332300 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.2 context 0:00:00.687633700 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.1 context 0:00:00.706444500 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 4.0 context 0:00:00.725763400 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 3.3 context 0:00:00.745413200 29460 000001E07A782CC0 DEBUG glcontext gstglcontext_wgl.c:235:gst_gl_context_wgl_create_context: trying to create a GL 3.2 context 0:00:00.781065300 29460 000001E07A782CC0 WARN glcontext gstglcontext.c:1326:gst_gl_context_create_thread: Failed to create context 0:00:00.781358600 29460 000001E076862100 INFO glcontext gstglcontext.c:1079:gst_gl_context_create: gl thread created 0:00:00.781584200 29460 000001E076862100 DEBUG glcontext gstglcontext.c:746:gst_gl_context_finalize: End of finalize 0:00:00.781787700 29460 000001E076862100 WARN glbasefilter gstglbasefilter.c:608:gst_gl_base_filter_find_gl_context_unlocked: error: failed to share contexts through wglShareLists 0xaa 0:00:00.782145700 29460 000001E076862100 INFO glcontext gstglcontext.c:349:gst_gl_context_new: creating a context for display , user choice:(null) 0:00:00.782381300 29460 000001E076862100 DEBUG glcontext gstglcontext.c:383:gst_gl_context_new: Done creating context for display (user_choice:(null)) 0:00:00.782632900 29460 000001E076862100 DEBUG glcontext gstglcontext.c:1058:gst_gl_context_create: other_context: 0:00:00.782921300 29460 000001E076862100 INFO glwindow gstglwindow.c:295:gst_gl_window_new: creating a window, user choice:(null) 0:00:00.783246300 29460 000001E076862100 DEBUG glcontext gstglcontext.c:956:gst_gl_context_set_window: window: 0:00:00.783815200 29460 000001E07A782D00 DEBUG glcontext gstglcontext.c:1227:gst_gl_context_create_thread: Creating thread 0:00:00.784130000 29460 000001E07A782D00 FIXME glcontext gstglcontext.c:2041:gst_gl_wrapped_context_get_config: wrapped context could not retrieve config. The application may be missing a call to gst_gl_context_fill_info() or the specific platform implemention is not implemented for retrieving the config from a wrapped OpenGL context. thread 'main' panicked at examples\src\bin\..\glupload.rs:755:44: called `Result::unwrap()` on an `Err` value: Received error from /GstPipeline:pipeline0/GstGLSinkBin:glsinkbin0/GstGLUploadElement:gluploadelement0: failed to share contexts through wglShareLists 0xaa (debug: Some("../gst-libs/gst/gl/gstglbasefilter.c(608): gst_gl_base_filter_find_gl_context_unlocked (): /GstPipeline:pipeline0/GstGLSinkBin:glsinkbin0/GstGLUploadElement:gluploadelement0")) This happens because the context is made "current" inside the `winit` loop before asynchronous GL initialization inside GStreamer creates a shared context for contexts replied to `NeedContext("gst.gl.app_context")`. `wglShareLists()` requires neither context to be current on separate threads. As we have a `Surface` on the `glutin` side, we're not uncurrenting it anymore, and instead want to perform GStreamer initialization up-front without a thread, or on a separate `GLContext`. One way to prevent GStreamer from creating yet another context on top of our wrapped context asynchronously, is by creating a new shared GL context based on our wrapped (WGL) context ourselves. By adding that context to `GLDisplay` (that we provide in the relevant `gst.gl.GLDisplay` context query) instead of via the `gst.gl.app_context` `NeedContext` query, the underlying initialization code no longer attempts to create a shared context because it will use the one from `GLDisplay` directly (specifically in e.g. `gst_gl_base_filter_find_gl_context_unlocked()`). The preferred way however is to reply this new shared context to a `Query` for the `gst.gl.local_context` context, which arbitrary applications can achieve by inserting a pad probe on one of the pads in the pipeline. GL elements and functions like the one mentioned above call into `gst_gl_query_local_gl_context()` which performs this query in both directions, before reading contexts from `GLDisplay` or ultimately sending a `NeedContext` message outlined above. With this patch the initialization flow becomes clear from the logs, where the above lines that will call `wglShareLists()` are now called directly when `gl_context.create(wrapped_context)` is called in our Rust code. In theory, since `GLContext` tracks whether it is current and on which thread, the upsteam source should emit an error of sorts when `wglShareLists()` is called on a thread where this `other_context` in `gst_gl_context_create(_thread)` was not yet current, to not make such mistakes go unnoticed as most drivers (i.e. AMD's) seem to not make an issue out of it. Part-of: --- examples/src/glupload.rs | 155 ++++++++++++++++++++------------------- 1 file changed, 78 insertions(+), 77 deletions(-) diff --git a/examples/src/glupload.rs b/examples/src/glupload.rs index 4b35d6c8c..0d39f9ace 100644 --- a/examples/src/glupload.rs +++ b/examples/src/glupload.rs @@ -14,7 +14,6 @@ use std::{ }; use anyhow::{Context, Result}; -use derive_more::derive::{Display, Error}; use glutin::{ config::GetGlConfig as _, context::AsRawContext as _, @@ -22,18 +21,10 @@ use glutin::{ prelude::*, }; use glutin_winit::GlWindow as _; -use gst::element_error; +use gst::{element_error, PadProbeReturn, PadProbeType, QueryViewMut}; use gst_gl::prelude::*; use raw_window_handle::HasRawWindowHandle as _; -#[derive(Debug, Display, Error)] -#[display("Received error from {src}: {error} (debug: {debug:?})")] -struct ErrorMessage { - src: glib::GString, - error: glib::Error, - debug: Option, -} - #[rustfmt::skip] static VERTICES: [f32; 20] = [ 1.0, 1.0, 0.0, 1.0, 0.0, @@ -325,17 +316,16 @@ fn load(gl_display: &impl glutin::display::GlDisplay) -> Gl { #[derive(Debug)] enum Message { Frame(gst_video::VideoInfo, gst::Buffer), - BusEvent, + BusMessage(gst::Message), } pub(crate) struct App { pipeline: gst::Pipeline, appsink: gst_app::AppSink, - bus: gst::Bus, event_loop: winit::event_loop::EventLoop, window: Option, not_current_gl_context: Option, - shared_context: gst_gl::GLContext, + glutin_context: gst_gl::GLContext, } impl App { @@ -471,60 +461,82 @@ impl App { handler => anyhow::bail!("Unsupported platform: {handler:?}."), }; - let shared_context = unsafe { + let glutin_context = unsafe { gst_gl::GLContext::new_wrapped(&gst_gl_display, raw_gl_context, platform, api) } .context("Couldn't wrap GL context")?; - let gl_context = shared_context.clone(); + { + // Make a new context that isn't the wrapped glutin context so that it can be made + // current on a new "gstglcontext" thread (see `gst_gl_context_create_thread()`), while + // the wrapped glutin context is made current on the winit event loop thread (this main + // thread). + let shared_context = gst_gl::GLContext::new(&gst_gl_display); + shared_context + .create(Some(&glutin_context)) + .context("Couldn't share wrapped Glutin GL context with new GL context")?; + + // Return the shared `GLContext` out of a pad probe for "gst.gl.local_context" to + // make the underlying pipeline use it directly, instead of creating a new GL context + // that is *shared* with the resulting context from a context `Query` (among other + // elements) or `NeedContext` bus message for "gst.gl.app_context", as documented for + // `gst_gl_ensure_element_data()`. + // + // On Windows, such context sharing calls `wglShareLists()` which fails on certain + // drivers when one of the contexts is already current on another thread. This would + // happen because the pipeline and specifically the aforementioned "gstglcontext" + // thread would be initialized asynchronously from the winit loop which makes our glutin + // context current. By calling `GLContext::create()` above, context sharing happens + // directly. + // + // An alternative approach would be using `gst_gl::GLDisplay::add_context()` to store + // the context inside `GLDisplay`, but the pad probe takes precedence. + + // While the pad probe could be installed anywhere, it makes logical sense to insert it + // on the appsink where the images are extracted and displayed to a window via the same + // GL contexts. + appsink + .static_pad("sink") + .unwrap() + .add_probe(PadProbeType::QUERY_DOWNSTREAM, move |pad, probe_info| { + if let Some(q) = probe_info.query_mut() { + if let QueryViewMut::Context(cq) = q.view_mut() { + if gst_gl::functions::gl_handle_context_query( + &pad.parent_element().unwrap(), + cq, + Some(&gst_gl_display), + Some(&shared_context), + None::<&gst_gl::GLContext>, + ) { + return PadProbeReturn::Handled; + } + } + } + PadProbeReturn::Ok + }) + .unwrap(); + } + let event_proxy = event_loop.create_proxy(); #[allow(clippy::single_match)] - bus.set_sync_handler(move |_, msg| { - match msg.view() { - gst::MessageView::NeedContext(ctxt) => { - let context_type = ctxt.context_type(); - if context_type == *gst_gl::GL_DISPLAY_CONTEXT_TYPE { - if let Some(el) = - msg.src().map(|s| s.downcast_ref::().unwrap()) - { - let context = gst::Context::new(context_type, true); - context.set_gl_display(&gst_gl_display); - el.set_context(&context); - } - } - if context_type == "gst.gl.app_context" { - if let Some(el) = - msg.src().map(|s| s.downcast_ref::().unwrap()) - { - let mut context = gst::Context::new(context_type, true); - { - let context = context.get_mut().unwrap(); - let s = context.structure_mut(); - s.set("context", &gl_context); - } - el.set_context(&context); - } - } - } - _ => (), - } - - if let Err(e) = event_proxy.send_event(Message::BusEvent) { + bus.set_sync_handler(move |_bus, msg| { + if let Err(e) = event_proxy + // Forward all messages to winit's event loop + .send_event(Message::BusMessage(msg.to_owned())) + { eprintln!("Failed to send BusEvent to event proxy: {e}") } - gst::BusSyncReply::Pass + gst::BusSyncReply::Drop }); - Ok(App { pipeline, appsink, - bus, event_loop, window, not_current_gl_context: Some(not_current_gl_context), - shared_context, + glutin_context, }) } @@ -658,28 +670,20 @@ impl App { } } - fn handle_messages(bus: &gst::Bus) -> Result<()> { + /// Should be called from within the event loop + fn handle_message(msg: gst::Message) { use gst::MessageView; - for msg in bus.iter() { - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(err) => { - return Err(ErrorMessage { - src: msg - .src() - .map(|s| s.path_string()) - .unwrap_or_else(|| glib::GString::from("UNKNOWN")), - error: err.error(), - debug: err.debug(), - } - .into()); - } - _ => (), - } + // Only handle error messages by panicking, to hard-stop the event loop + if let MessageView::Error(err) = msg.view() { + let src = msg + .src() + .map(|s| s.path_string()) + .unwrap_or_else(|| glib::GString::from("UNKNOWN")); + let error = err.error(); + let debug = err.debug(); + panic!("Received error from {src}: {error} (debug: {debug:?})"); } - - Ok(()) } } @@ -688,11 +692,10 @@ pub(crate) fn main_loop(app: App) -> Result<()> { let App { pipeline, - bus, event_loop, mut window, mut not_current_gl_context, - shared_context, + glutin_context, .. } = app; @@ -751,9 +754,7 @@ pub(crate) fn main_loop(app: App) -> Result<()> { } } // Handle all pending messages when we are awaken by set_sync_handler - winit::event::Event::UserEvent(Message::BusEvent) => { - App::handle_messages(&bus).unwrap(); - } + winit::event::Event::UserEvent(Message::BusMessage(msg)) => App::handle_message(msg), winit::event::Event::Resumed => { let not_current_gl_context = not_current_gl_context .take() @@ -781,9 +782,9 @@ pub(crate) fn main_loop(app: App) -> Result<()> { // Tell GStreamer that the context has been made current (for borrowed contexts, // this does not try to make it current again) - shared_context.activate(true).unwrap(); + glutin_context.activate(true).unwrap(); - shared_context + glutin_context .fill_info() .expect("Couldn't fill context info"); @@ -812,7 +813,7 @@ pub(crate) fn main_loop(app: App) -> Result<()> { if let Some((gl, gl_context, gl_surface)) = &running_state { if let Some(frame) = curr_frame.as_ref() { let sync_meta = frame.buffer().meta::().unwrap(); - sync_meta.wait(&shared_context); + sync_meta.wait(&glutin_context); if let Ok(texture) = frame.texture_id(0) { gl.draw_frame(texture as gl::types::GLuint); }