From c506adc95003c65be2fd84765e170e1ef76452a5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 29 May 2020 15:44:55 +1000 Subject: [PATCH] gl/eagl: don't access UIkit objects on the main thread This means we cannot access [view layer] or view.bounds from the OpenGL thread. This also means that we need to call the main thread when setting the window handle. However, we cannot perform that synchronously as that may deadlock with the application performing the set_window_handle() call. We need to defer the actual update and run it asynchronously and wait for the window handle update internally at each point it is needed. Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/issues/372 Part-of: --- gst-libs/gst/gl/eagl/gstglcontext_eagl.h | 7 +- gst-libs/gst/gl/eagl/gstglcontext_eagl.m | 81 +++++------ gst-libs/gst/gl/eagl/gstglios_utils.h | 45 ++++++ gst-libs/gst/gl/eagl/gstglwindow_eagl.h | 5 - gst-libs/gst/gl/eagl/gstglwindow_eagl.m | 173 +++++++++++++++++++++-- gst-libs/gst/gl/meson.build | 4 +- 6 files changed, 245 insertions(+), 70 deletions(-) create mode 100644 gst-libs/gst/gl/eagl/gstglios_utils.h diff --git a/gst-libs/gst/gl/eagl/gstglcontext_eagl.h b/gst-libs/gst/gl/eagl/gstglcontext_eagl.h index fa3244e0ab..662d34bf40 100644 --- a/gst-libs/gst/gl/eagl/gstglcontext_eagl.h +++ b/gst-libs/gst/gl/eagl/gstglcontext_eagl.h @@ -33,11 +33,6 @@ G_BEGIN_DECLS #define GST_IS_GL_CONTEXT_EAGL_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_GL_CONTEXT_EAGL)) #define GST_GL_CONTEXT_EAGL_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_GL_CONTEXT_EAGL, GstGLContextEaglClass)) -#define GST_GL_CONTEXT_EAGL_CONTEXT(obj) \ - ((__bridge EAGLContext *)(obj->priv->eagl_context)) -#define GST_GL_CONTEXT_EAGL_LAYER(obj) \ - ((__bridge CAEAGLLayer *)(obj->priv->eagl_layer)) - typedef struct _GstGLContextEagl GstGLContextEagl; typedef struct _GstGLContextEaglPrivate GstGLContextEaglPrivate; typedef struct _GstGLContextEaglClass GstGLContextEaglClass; @@ -64,7 +59,7 @@ GType gst_gl_context_eagl_get_type (void); GstGLContextEagl * gst_gl_context_eagl_new (GstGLDisplay * display); -void gst_gl_context_eagl_update_layer (GstGLContext * context); +void gst_gl_context_eagl_update_layer (GstGLContext * context, gpointer layer); void gst_gl_context_eagl_resize (GstGLContextEagl * eagl_context); void gst_gl_context_eagl_prepare_draw (GstGLContextEagl * context); void gst_gl_context_eagl_finish_draw (GstGLContextEagl * context); diff --git a/gst-libs/gst/gl/eagl/gstglcontext_eagl.m b/gst-libs/gst/gl/eagl/gstglcontext_eagl.m index 1f56a3a45e..fe9bba97aa 100644 --- a/gst-libs/gst/gl/eagl/gstglcontext_eagl.m +++ b/gst-libs/gst/gl/eagl/gstglcontext_eagl.m @@ -29,6 +29,12 @@ #include "gstglcontext_eagl.h" #include "../gstglcontext_private.h" +#include "gstglios_utils.h" + +#define GST_GL_CONTEXT_EAGL_CONTEXT(obj) \ + ((__bridge EAGLContext *)(obj->priv->eagl_context)) +#define GST_GL_CONTEXT_EAGL_LAYER(obj) \ + ((__bridge CAEAGLLayer *)(obj->priv->eagl_layer)) #define GST_CAT_DEFAULT gst_gl_context_debug @@ -108,7 +114,9 @@ gst_gl_context_eagl_resize (GstGLContextEagl * eagl_context) int width, height; glBindRenderbuffer (GL_RENDERBUFFER, eagl_context->priv->color_renderbuffer); - [GST_GL_CONTEXT_EAGL_CONTEXT(eagl_context) renderbufferStorage:GL_RENDERBUFFER fromDrawable:GST_GL_CONTEXT_EAGL_LAYER(eagl_context)]; + [GST_GL_CONTEXT_EAGL_CONTEXT(eagl_context) + renderbufferStorage:GL_RENDERBUFFER + fromDrawable:GST_GL_CONTEXT_EAGL_LAYER(eagl_context)]; glGetRenderbufferParameteriv (GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &width); glGetRenderbufferParameteriv (GL_RENDERBUFFER, @@ -128,7 +136,9 @@ gst_gl_context_eagl_release_layer (GstGLContext * context) if (context_eagl->priv->eagl_layer) { gst_gl_context_eagl_activate (context, TRUE); - [GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl) renderbufferStorage: GL_RENDERBUFFER fromDrawable:nil]; + [GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl) + renderbufferStorage:GL_RENDERBUFFER + fromDrawable:nil]; glDeleteFramebuffers (1, &context_eagl->priv->framebuffer); context_eagl->priv->framebuffer = 0; @@ -138,13 +148,14 @@ gst_gl_context_eagl_release_layer (GstGLContext * context) glDeleteRenderbuffers (1, &context_eagl->priv->color_renderbuffer); context_eagl->priv->color_renderbuffer = 0; - context_eagl->priv->eagl_layer = nil; + CFRelease (context_eagl->priv->eagl_layer); + context_eagl->priv->eagl_layer = NULL; gst_gl_context_eagl_activate (context, FALSE); } } void -gst_gl_context_eagl_update_layer (GstGLContext * context) +gst_gl_context_eagl_update_layer (GstGLContext * context, gpointer layer) { GLuint framebuffer; GLuint color_renderbuffer; @@ -155,23 +166,17 @@ gst_gl_context_eagl_update_layer (GstGLContext * context) GLenum status; GstGLContextEagl *context_eagl = GST_GL_CONTEXT_EAGL (context); GstGLContextEaglPrivate *priv = context_eagl->priv; - UIView *window_handle = nil; GstGLWindow *window = gst_gl_context_get_window (context); - if (window) - window_handle = (__bridge UIView *)((void *)gst_gl_window_get_window_handle (window)); - if (!window_handle) { + if (!layer || !gst_gl_window_get_window_handle (window)) { GST_INFO_OBJECT (context, "window handle not set yet, not updating layer"); goto out; } - GST_INFO_OBJECT (context, "updating layer, frame %fx%f", - window_handle.frame.size.width, window_handle.frame.size.height); - if (priv->eagl_layer) gst_gl_context_eagl_release_layer (context); - eagl_layer = (CAEAGLLayer *)[window_handle layer]; + eagl_layer = (__bridge CAEAGLLayer *) layer; [EAGLContext setCurrentContext:GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl)]; /* Allocate framebuffer */ @@ -205,7 +210,7 @@ gst_gl_context_eagl_update_layer (GstGLContext * context) glBindRenderbuffer (GL_RENDERBUFFER, 0); glBindFramebuffer (GL_FRAMEBUFFER, 0); - priv->eagl_layer = (__bridge_retained gpointer)eagl_layer; + priv->eagl_layer = (__bridge_retained gpointer) eagl_layer; priv->framebuffer = framebuffer; priv->color_renderbuffer = color_renderbuffer; priv->depth_renderbuffer = depth_renderbuffer; @@ -213,6 +218,8 @@ gst_gl_context_eagl_update_layer (GstGLContext * context) out: if (window) gst_object_unref (window); + if (layer) + CFRelease (layer); } static gboolean @@ -221,6 +228,9 @@ gst_gl_context_eagl_create_context (GstGLContext * context, GstGLAPI gl_api, { GstGLContextEagl *context_eagl = GST_GL_CONTEXT_EAGL (context); GstGLContextEaglPrivate *priv = context_eagl->priv; + GstGLWindow *window; + GstGLWindowEagl *window_eagl; + gpointer layer; EAGLSharegroup *share_group; if (other_context) { @@ -248,7 +258,19 @@ gst_gl_context_eagl_create_context (GstGLContext * context, GstGLAPI gl_api, priv->depth_renderbuffer = 0; GST_INFO_OBJECT (context, "context created, updating layer"); - gst_gl_context_eagl_update_layer (context); + window = gst_gl_context_get_window (context); + if (!window) { + g_set_error_literal (error, GST_GL_CONTEXT_ERROR, + GST_GL_CONTEXT_ERROR_CREATE_CONTEXT, + "No window to render into"); + return FALSE; + } + + window_eagl = GST_GL_WINDOW_EAGL (window); + layer = gst_gl_window_eagl_get_layer (window_eagl); + gst_gl_context_eagl_update_layer (context, layer); + + gst_object_unref (window); return TRUE; } @@ -272,35 +294,6 @@ gst_gl_context_eagl_destroy_context (GstGLContext * context) static gboolean gst_gl_context_eagl_choose_format (GstGLContext * context, GError ** error) { - GstGLContextEagl *context_eagl; - GstGLWindow *window; - UIView *window_handle = nil; - - context_eagl = GST_GL_CONTEXT_EAGL (context); - window = gst_gl_context_get_window (context); - - if (!window) - return TRUE; - - if (window) - window_handle = (__bridge UIView *)(void *)gst_gl_window_get_window_handle (window); - - if (!window_handle) { - gst_object_unref (window); - return TRUE; - } - - CAEAGLLayer *eagl_layer; - NSDictionary * dict =[NSDictionary dictionaryWithObjectsAndKeys: - [NSNumber numberWithBool:NO], kEAGLDrawablePropertyRetainedBacking, - kEAGLColorFormatRGBA8, kEAGLDrawablePropertyColorFormat, nil]; - - eagl_layer = (CAEAGLLayer *)[window_handle layer]; - [eagl_layer setOpaque:YES]; - [eagl_layer setDrawableProperties:dict]; - - gst_object_unref (window); - return TRUE; } @@ -315,7 +308,7 @@ gst_gl_context_eagl_prepare_draw (GstGLContextEagl * context) { if (!context->priv->eagl_layer) return; - + glBindFramebuffer (GL_FRAMEBUFFER, context->priv->framebuffer); glBindRenderbuffer (GL_RENDERBUFFER, context->priv->color_renderbuffer); } diff --git a/gst-libs/gst/gl/eagl/gstglios_utils.h b/gst-libs/gst/gl/eagl/gstglios_utils.h new file mode 100644 index 0000000000..40f414dcc0 --- /dev/null +++ b/gst-libs/gst/gl/eagl/gstglios_utils.h @@ -0,0 +1,45 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GL_IOS_UTILS_H__ +#define __GL_IOS_UTILS_H__ + +#include +#include + +#include "gstglwindow_eagl.h" + +G_BEGIN_DECLS + +@interface GstGLUIView : UIView +- (void) setGstWindow:(GstGLWindowEagl *)window_eagl; +@end + +typedef void (*GstGLWindowEaglFunc) (gpointer data); + +G_GNUC_INTERNAL +void _gl_invoke_on_main (GstGLWindowEaglFunc func, gpointer data, GDestroyNotify notify); + +G_GNUC_INTERNAL +gpointer gst_gl_window_eagl_get_layer (GstGLWindowEagl * window_eagl); + +G_END_DECLS + +#endif diff --git a/gst-libs/gst/gl/eagl/gstglwindow_eagl.h b/gst-libs/gst/gl/eagl/gstglwindow_eagl.h index 561b21d93f..f42296aa1b 100644 --- a/gst-libs/gst/gl/eagl/gstglwindow_eagl.h +++ b/gst-libs/gst/gl/eagl/gstglwindow_eagl.h @@ -33,11 +33,6 @@ G_BEGIN_DECLS #define GST_IS_GL_WINDOW_EAGL_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_GL_WINDOW_EAGL)) #define GST_GL_WINDOW_EAGL_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_GL_WINDOW_EAGL, GstGLWindowEaglClass)) -#define GST_GL_WINDOW_EAGL_VIEW(obj) \ - ((__bridge UIView *)(obj->priv->view)) -#define GST_GL_WINDOW_EAGL_QUEUE(obj) \ - ((__bridge dispatch_queue_t)(obj->priv->gl_queue)) - typedef struct _GstGLWindowEagl GstGLWindowEagl; typedef struct _GstGLWindowEaglPrivate GstGLWindowEaglPrivate; typedef struct _GstGLWindowEaglClass GstGLWindowEaglClass; diff --git a/gst-libs/gst/gl/eagl/gstglwindow_eagl.m b/gst-libs/gst/gl/eagl/gstglwindow_eagl.m index 0a62324b1d..76dd05ea4f 100644 --- a/gst-libs/gst/gl/eagl/gstglwindow_eagl.m +++ b/gst-libs/gst/gl/eagl/gstglwindow_eagl.m @@ -28,10 +28,16 @@ #include "gstglwindow_eagl.h" #include "gstglcontext_eagl.h" +#include "gstglios_utils.h" #define GST_CAT_DEFAULT gst_gl_window_eagl_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +#define GST_GL_WINDOW_EAGL_LAYER(obj) \ + ((__bridge CAEAGLLayer *)(obj->priv->layer)) +#define GST_GL_WINDOW_EAGL_QUEUE(obj) \ + ((__bridge dispatch_queue_t)(obj->priv->gl_queue)) + static void gst_gl_window_eagl_finalize (GObject * object); static guintptr gst_gl_window_eagl_get_display (GstGLWindow * window); @@ -46,10 +52,15 @@ static void gst_gl_window_eagl_send_message_async (GstGLWindow * window, struct _GstGLWindowEaglPrivate { - gpointer view; + gboolean pending_set_window_handle; + gpointer external_view; + gpointer internal_view; + gpointer layer; gint window_width, window_height; gint preferred_width, preferred_height; gpointer gl_queue; + GMutex draw_lock; + GCond cond; }; #define DEBUG_INIT \ @@ -85,13 +96,22 @@ gst_gl_window_eagl_init (GstGLWindowEagl * window) window->priv = gst_gl_window_eagl_get_instance_private (window); window->priv->gl_queue = (__bridge_retained gpointer)dispatch_queue_create ("org.freedesktop.gstreamer.glwindow", NULL); + g_mutex_init (&window->priv->draw_lock); + g_cond_init (&window->priv->cond); } static void gst_gl_window_eagl_finalize (GObject * object) { GstGLWindowEagl *window = GST_GL_WINDOW_EAGL (object); + + if (window->priv->layer) + CFRelease (window->priv->layer); + window->priv->layer = NULL; CFRelease(window->priv->gl_queue); + g_mutex_clear (&window->priv->draw_lock); + g_cond_clear (&window->priv->cond); + G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -117,23 +137,89 @@ gst_gl_window_eagl_get_display (GstGLWindow * window) static guintptr gst_gl_window_eagl_get_window_handle (GstGLWindow * window) { - return (guintptr) GST_GL_WINDOW_EAGL (window)->priv->view; + return (guintptr) GST_GL_WINDOW_EAGL (window)->priv->internal_view; +} + +static void +_create_gl_window (GstGLWindowEagl * window_eagl) +{ + GstGLWindowEaglPrivate *priv = window_eagl->priv; + UIView *external_view; + CGRect rect; + GstGLUIView *view; + + g_mutex_lock (&priv->draw_lock); + + external_view = (__bridge UIView *) priv->external_view; + rect = CGRectMake (0, 0, external_view.frame.size.width, external_view.frame.size.height); + + window_eagl->priv->window_width = rect.size.width; + window_eagl->priv->window_height = rect.size.height; + + view = [[GstGLUIView alloc] initWithFrame:rect]; + [view setGstWindow:window_eagl]; + view.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; + view.contentMode = UIViewContentModeRedraw; + + priv->internal_view = (__bridge_retained gpointer) view; + [external_view addSubview:view]; + priv->internal_view = (__bridge_retained gpointer) view; + priv->layer = (__bridge_retained gpointer) [view layer]; + + NSDictionary * dict = [NSDictionary dictionaryWithObjectsAndKeys: + [NSNumber numberWithBool:NO], kEAGLDrawablePropertyRetainedBacking, + kEAGLColorFormatRGBA8, kEAGLDrawablePropertyColorFormat, nil]; + + [[view layer] setOpaque:YES]; + [(CAEAGLLayer *) [view layer] setDrawableProperties:dict]; + + g_cond_broadcast (&priv->cond); + g_mutex_unlock (&priv->draw_lock); +} + +static void +ensure_window_handle_is_set_unlocked (GstGLWindowEagl * window_eagl) +{ + GstGLContext *context; + + if (!window_eagl->priv->pending_set_window_handle) + return; + + context = gst_gl_window_get_context (GST_GL_WINDOW (window_eagl)); + if (!context) { + g_critical ("Window does not have a GstGLContext attached!. " + "Aborting set window handle."); + g_mutex_unlock (&window_eagl->priv->draw_lock); + return; + } + + while (!window_eagl->priv->internal_view) + g_cond_wait (&window_eagl->priv->cond, &window_eagl->priv->draw_lock); + + GST_INFO_OBJECT (context, "handle set, updating layer"); + gst_gl_context_eagl_update_layer (context, window_eagl->priv->layer); + gst_object_unref (context); + + window_eagl->priv->pending_set_window_handle = FALSE; } static void gst_gl_window_eagl_set_window_handle (GstGLWindow * window, guintptr handle) { GstGLWindowEagl *window_eagl; - GstGLContext *context; window_eagl = GST_GL_WINDOW_EAGL (window); - context = gst_gl_window_get_context (window); - window_eagl->priv->view = (gpointer)handle; - GST_INFO_OBJECT (context, "handle set, updating layer"); - gst_gl_context_eagl_update_layer (context); + g_mutex_lock (&window_eagl->priv->draw_lock); + if (window_eagl->priv->external_view) + CFRelease (window_eagl->priv->external_view); + window_eagl->priv->external_view = (gpointer)handle; + window_eagl->priv->pending_set_window_handle = TRUE; + g_mutex_unlock (&window_eagl->priv->draw_lock); - gst_object_unref (context); + /* XXX: Maybe we need an async set_window_handle? */ + _gl_invoke_on_main ((GstGLWindowEaglFunc) _create_gl_window, + gst_object_ref (window_eagl), gst_object_unref); } static void @@ -180,21 +266,22 @@ draw_cb (gpointer data) GstGLContext *context = gst_gl_window_get_context (window); GstGLContextEagl *eagl_context = GST_GL_CONTEXT_EAGL (context); - if (window_eagl->priv->view) { + g_mutex_lock (&window_eagl->priv->draw_lock); + + ensure_window_handle_is_set_unlocked (window_eagl); + + if (window_eagl->priv->internal_view) { CGSize size; CAEAGLLayer *eagl_layer; - eagl_layer = (CAEAGLLayer *)[GST_GL_WINDOW_EAGL_VIEW(window_eagl) layer]; + eagl_layer = GST_GL_WINDOW_EAGL_LAYER (window_eagl); size = eagl_layer.frame.size; - size = CGSizeMake (size.width * eagl_layer.contentsScale, size.height * eagl_layer.contentsScale); + size = CGSizeMake (size.width * eagl_layer.contentsScale, size.height * eagl_layer.contentsScale); if (window->queue_resize || window_eagl->priv->window_width != size.width || window_eagl->priv->window_height != size.height) { - window_eagl->priv->window_width = size.width; - window_eagl->priv->window_height = size.height; - gst_gl_context_eagl_resize (eagl_context); gst_gl_window_resize (window, window_eagl->priv->window_width, @@ -211,6 +298,8 @@ draw_cb (gpointer data) gst_gl_context_eagl_finish_draw (eagl_context); + g_mutex_unlock (&window_eagl->priv->draw_lock); + gst_object_unref (context); } @@ -219,3 +308,59 @@ gst_gl_window_eagl_draw (GstGLWindow * window) { gst_gl_window_send_message (window, (GstGLWindowCB) draw_cb, window); } + +gpointer +gst_gl_window_eagl_get_layer (GstGLWindowEagl * window_eagl) +{ + gpointer layer; + + g_mutex_lock (&window_eagl->priv->draw_lock); + if (window_eagl->priv->layer) + CFRetain (window_eagl->priv->layer); + layer = window_eagl->priv->layer; + g_mutex_unlock (&window_eagl->priv->draw_lock); + + return layer; +} + +@implementation GstGLUIView { + GstGLWindowEagl * window_eagl; +}; + ++(Class) layerClass +{ + return [CAEAGLLayer class]; +} + +-(void) setGstWindow:(GstGLWindowEagl *) window +{ + window_eagl = window; +} + +-(void) layoutSubViews +{ + g_mutex_lock (&window_eagl->priv->draw_lock); + [super layoutSubviews]; + CGSize rect = self.bounds.size; + self->window_eagl->priv->window_width = rect.width; + self->window_eagl->priv->window_height = rect.height; + g_mutex_unlock (&window_eagl->priv->draw_lock); +} + +@end + +void +_gl_invoke_on_main (GstGLWindowEaglFunc func, gpointer data, GDestroyNotify notify) +{ + if ([NSThread isMainThread]) { + func (data); + if (notify) + notify (data); + } else { + dispatch_async (dispatch_get_main_queue (), ^{ + func (data); + if (notify) + notify (data); + }); + } +} diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build index a752f4d40c..d94b0505d2 100644 --- a/gst-libs/gst/gl/meson.build +++ b/gst-libs/gst/gl/meson.build @@ -809,13 +809,15 @@ if host_system == 'ios' and need_platform_eagl != 'no' and need_win_eagl != 'no' foundation_dep = dependency('appleframeworks', modules : ['Foundation'], required : false) corefoundation_dep = dependency('appleframeworks', modules : ['CoreFoundation'], required : false) coregraphics_dep = dependency('appleframeworks', modules : ['CoreGraphics'], required : false) + quartzcore_dep = dependency('appleframeworks', modules : ['QuartzCore'], required : false) uikit_dep = dependency('appleframeworks', modules : ['UIkit'], required : false) - if foundation_dep.found() and corefoundation_dep.found() and coregraphics_dep.found() and uikit_dep.found() + if foundation_dep.found() and corefoundation_dep.found() and coregraphics_dep.found() and quartzcore_dep.found() and uikit_dep.found() gl_platform_deps += [ corefoundation_dep, foundation_dep, coregraphics_dep, + quartzcore_dep, uikit_dep, ] gl_sources += [