gl/eagl: don't access UIkit objects on the main thread

This means we cannot access [view layer] or view.bounds from the OpenGL
thread.  This also means that we need to call the main thread when
setting the window handle.  However, we cannot perform that
synchronously as that may deadlock with the application performing the
set_window_handle() call.

We need to defer the actual update and run it asynchronously and wait
for the window handle update internally at each point it is needed.

Fixes: https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/issues/372
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/681>

gst-libs/gst/gl/eagl/gstglcontext_eagl.h

@@ -33,11 +33,6 @@ G_BEGIN_DECLS
 #define GST_IS_GL_CONTEXT_EAGL_CLASS(k)  (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_GL_CONTEXT_EAGL))
 #define GST_GL_CONTEXT_EAGL_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_GL_CONTEXT_EAGL, GstGLContextEaglClass))
 
-#define GST_GL_CONTEXT_EAGL_CONTEXT(obj) \
-    ((__bridge EAGLContext *)(obj->priv->eagl_context))
-#define GST_GL_CONTEXT_EAGL_LAYER(obj) \
-    ((__bridge CAEAGLLayer *)(obj->priv->eagl_layer))
-
 typedef struct _GstGLContextEagl        GstGLContextEagl;
 typedef struct _GstGLContextEaglPrivate GstGLContextEaglPrivate;
 typedef struct _GstGLContextEaglClass   GstGLContextEaglClass;
@@ -64,7 +59,7 @@ GType gst_gl_context_eagl_get_type (void);
 
 GstGLContextEagl * gst_gl_context_eagl_new (GstGLDisplay * display);
 
-void gst_gl_context_eagl_update_layer (GstGLContext * context);
+void gst_gl_context_eagl_update_layer (GstGLContext * context, gpointer layer);
 void gst_gl_context_eagl_resize (GstGLContextEagl * eagl_context);
 void gst_gl_context_eagl_prepare_draw (GstGLContextEagl * context);
 void gst_gl_context_eagl_finish_draw (GstGLContextEagl * context);

gst-libs/gst/gl/eagl/gstglcontext_eagl.m

@@ -29,6 +29,12 @@
 
 #include "gstglcontext_eagl.h"
 #include "../gstglcontext_private.h"
+#include "gstglios_utils.h"
+
+#define GST_GL_CONTEXT_EAGL_CONTEXT(obj) \
+    ((__bridge EAGLContext *)(obj->priv->eagl_context))
+#define GST_GL_CONTEXT_EAGL_LAYER(obj) \
+    ((__bridge CAEAGLLayer *)(obj->priv->eagl_layer))
 
 #define GST_CAT_DEFAULT gst_gl_context_debug
 
@@ -108,7 +114,9 @@ gst_gl_context_eagl_resize (GstGLContextEagl * eagl_context)
   int width, height;
 
   glBindRenderbuffer (GL_RENDERBUFFER, eagl_context->priv->color_renderbuffer);
-  [GST_GL_CONTEXT_EAGL_CONTEXT(eagl_context) renderbufferStorage:GL_RENDERBUFFER fromDrawable:GST_GL_CONTEXT_EAGL_LAYER(eagl_context)];
+  [GST_GL_CONTEXT_EAGL_CONTEXT(eagl_context)
+      renderbufferStorage:GL_RENDERBUFFER
+      fromDrawable:GST_GL_CONTEXT_EAGL_LAYER(eagl_context)];
   glGetRenderbufferParameteriv (GL_RENDERBUFFER,
       GL_RENDERBUFFER_WIDTH, &width);
   glGetRenderbufferParameteriv (GL_RENDERBUFFER,
@@ -128,7 +136,9 @@ gst_gl_context_eagl_release_layer (GstGLContext * context)
   if (context_eagl->priv->eagl_layer) {
     gst_gl_context_eagl_activate (context, TRUE);
 
-    [GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl) renderbufferStorage: GL_RENDERBUFFER fromDrawable:nil];
+    [GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl)
+        renderbufferStorage:GL_RENDERBUFFER
+        fromDrawable:nil];
 
     glDeleteFramebuffers (1, &context_eagl->priv->framebuffer);
     context_eagl->priv->framebuffer = 0;
@@ -138,13 +148,14 @@ gst_gl_context_eagl_release_layer (GstGLContext * context)
     glDeleteRenderbuffers (1, &context_eagl->priv->color_renderbuffer);
     context_eagl->priv->color_renderbuffer = 0;
 
-    context_eagl->priv->eagl_layer = nil;
+    CFRelease (context_eagl->priv->eagl_layer);
+    context_eagl->priv->eagl_layer = NULL;
     gst_gl_context_eagl_activate (context, FALSE);
   }
 }
 
 void
-gst_gl_context_eagl_update_layer (GstGLContext * context)
+gst_gl_context_eagl_update_layer (GstGLContext * context, gpointer layer)
 {
   GLuint framebuffer;
   GLuint color_renderbuffer;
@@ -155,23 +166,17 @@ gst_gl_context_eagl_update_layer (GstGLContext * context)
   GLenum status;
   GstGLContextEagl *context_eagl = GST_GL_CONTEXT_EAGL (context);
   GstGLContextEaglPrivate *priv = context_eagl->priv;
-  UIView *window_handle = nil;
   GstGLWindow *window = gst_gl_context_get_window (context);
-  if (window)
-    window_handle = (__bridge UIView *)((void *)gst_gl_window_get_window_handle (window));
 
-  if (!window_handle) {
+  if (!layer || !gst_gl_window_get_window_handle (window)) {
     GST_INFO_OBJECT (context, "window handle not set yet, not updating layer");
     goto out;
   }
 
-  GST_INFO_OBJECT (context, "updating layer, frame %fx%f",
-      window_handle.frame.size.width, window_handle.frame.size.height);
-
   if (priv->eagl_layer)
     gst_gl_context_eagl_release_layer (context);
 
-  eagl_layer = (CAEAGLLayer *)[window_handle layer];
+  eagl_layer = (__bridge CAEAGLLayer *) layer;
   [EAGLContext setCurrentContext:GST_GL_CONTEXT_EAGL_CONTEXT(context_eagl)];
 
   /* Allocate framebuffer */
@@ -205,7 +210,7 @@ gst_gl_context_eagl_update_layer (GstGLContext * context)
   glBindRenderbuffer (GL_RENDERBUFFER, 0);
   glBindFramebuffer (GL_FRAMEBUFFER, 0);
 
-  priv->eagl_layer = (__bridge_retained gpointer)eagl_layer;
+  priv->eagl_layer = (__bridge_retained gpointer) eagl_layer;
   priv->framebuffer = framebuffer;
   priv->color_renderbuffer = color_renderbuffer;
   priv->depth_renderbuffer = depth_renderbuffer;
@@ -213,6 +218,8 @@ gst_gl_context_eagl_update_layer (GstGLContext * context)
 out:
   if (window)
     gst_object_unref (window);
+  if (layer)
+    CFRelease (layer);
 }
 
 static gboolean
@@ -221,6 +228,9 @@ gst_gl_context_eagl_create_context (GstGLContext * context, GstGLAPI gl_api,
 {
   GstGLContextEagl *context_eagl = GST_GL_CONTEXT_EAGL (context);
   GstGLContextEaglPrivate *priv = context_eagl->priv;
+  GstGLWindow *window;
+  GstGLWindowEagl *window_eagl;
+  gpointer layer;
   EAGLSharegroup *share_group;
 
   if (other_context) {
@@ -248,7 +258,19 @@ gst_gl_context_eagl_create_context (GstGLContext * context, GstGLAPI gl_api,
   priv->depth_renderbuffer = 0;
 
   GST_INFO_OBJECT (context, "context created, updating layer");
-  gst_gl_context_eagl_update_layer (context);
+  window = gst_gl_context_get_window (context);
+  if (!window) {
+    g_set_error_literal (error, GST_GL_CONTEXT_ERROR,
+        GST_GL_CONTEXT_ERROR_CREATE_CONTEXT,
+        "No window to render into");
+    return FALSE;
+  }
+
+  window_eagl = GST_GL_WINDOW_EAGL (window);
+  layer = gst_gl_window_eagl_get_layer (window_eagl);
+  gst_gl_context_eagl_update_layer (context, layer);
+
+  gst_object_unref (window);
 
   return TRUE;
 }
@@ -272,35 +294,6 @@ gst_gl_context_eagl_destroy_context (GstGLContext * context)
 static gboolean
 gst_gl_context_eagl_choose_format (GstGLContext * context, GError ** error)
 {
-  GstGLContextEagl *context_eagl;
-  GstGLWindow *window;
-  UIView *window_handle = nil;
-
-  context_eagl = GST_GL_CONTEXT_EAGL (context);
-  window = gst_gl_context_get_window (context);
-
-  if (!window)
-    return TRUE;
-
-  if (window)
-    window_handle = (__bridge UIView *)(void *)gst_gl_window_get_window_handle (window);
-
-  if (!window_handle) {
-    gst_object_unref (window);
-    return TRUE;
-  }
-  
-  CAEAGLLayer *eagl_layer;
-  NSDictionary * dict =[NSDictionary dictionaryWithObjectsAndKeys:
-      [NSNumber numberWithBool:NO], kEAGLDrawablePropertyRetainedBacking,
-      kEAGLColorFormatRGBA8, kEAGLDrawablePropertyColorFormat, nil];
-
-  eagl_layer = (CAEAGLLayer *)[window_handle layer];
-  [eagl_layer setOpaque:YES];
-  [eagl_layer setDrawableProperties:dict];
-
-  gst_object_unref (window);
-
   return TRUE;
 }
 
@@ -315,7 +308,7 @@ gst_gl_context_eagl_prepare_draw (GstGLContextEagl * context)
 {
   if (!context->priv->eagl_layer)
     return;
-  
+
   glBindFramebuffer (GL_FRAMEBUFFER, context->priv->framebuffer);
   glBindRenderbuffer (GL_RENDERBUFFER, context->priv->color_renderbuffer);
 }

gst-libs/gst/gl/eagl/gstglios_utils.h

@@ -0,0 +1,45 @@
+/*
+ * GStreamer
+ * Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GL_IOS_UTILS_H__
+#define __GL_IOS_UTILS_H__
+
+#include <gst/gst.h>
+#include <UIKit/UIKit.h>
+
+#include "gstglwindow_eagl.h"
+
+G_BEGIN_DECLS
+
+@interface GstGLUIView : UIView
+- (void) setGstWindow:(GstGLWindowEagl *)window_eagl;
+@end
+
+typedef void (*GstGLWindowEaglFunc) (gpointer data);
+
+G_GNUC_INTERNAL
+void _gl_invoke_on_main (GstGLWindowEaglFunc func, gpointer data, GDestroyNotify notify);
+
+G_GNUC_INTERNAL
+gpointer gst_gl_window_eagl_get_layer (GstGLWindowEagl * window_eagl);
+
+G_END_DECLS
+
+#endif

gst-libs/gst/gl/eagl/gstglwindow_eagl.h

@@ -33,11 +33,6 @@ G_BEGIN_DECLS
 #define GST_IS_GL_WINDOW_EAGL_CLASS(k)  (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_GL_WINDOW_EAGL))
 #define GST_GL_WINDOW_EAGL_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_GL_WINDOW_EAGL, GstGLWindowEaglClass))
 
-#define GST_GL_WINDOW_EAGL_VIEW(obj) \
-    ((__bridge UIView *)(obj->priv->view))
-#define GST_GL_WINDOW_EAGL_QUEUE(obj) \
-    ((__bridge dispatch_queue_t)(obj->priv->gl_queue))
-
 typedef struct _GstGLWindowEagl        GstGLWindowEagl;
 typedef struct _GstGLWindowEaglPrivate GstGLWindowEaglPrivate;
 typedef struct _GstGLWindowEaglClass   GstGLWindowEaglClass;

gst-libs/gst/gl/eagl/gstglwindow_eagl.m

@@ -28,10 +28,16 @@
 
 #include "gstglwindow_eagl.h"
 #include "gstglcontext_eagl.h"
+#include "gstglios_utils.h"
 
 #define GST_CAT_DEFAULT gst_gl_window_eagl_debug
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
 
+#define GST_GL_WINDOW_EAGL_LAYER(obj) \
+    ((__bridge CAEAGLLayer *)(obj->priv->layer))
+#define GST_GL_WINDOW_EAGL_QUEUE(obj) \
+    ((__bridge dispatch_queue_t)(obj->priv->gl_queue))
+
 static void gst_gl_window_eagl_finalize (GObject * object);
 
 static guintptr gst_gl_window_eagl_get_display (GstGLWindow * window);
@@ -46,10 +52,15 @@ static void gst_gl_window_eagl_send_message_async (GstGLWindow * window,
 
 struct _GstGLWindowEaglPrivate
 {
-  gpointer view;
+  gboolean pending_set_window_handle;
+  gpointer external_view;
+  gpointer internal_view;
+  gpointer layer;
   gint window_width, window_height;
   gint preferred_width, preferred_height;
   gpointer gl_queue;
+  GMutex draw_lock;
+  GCond cond;
 };
 
 #define DEBUG_INIT \
@@ -85,13 +96,22 @@ gst_gl_window_eagl_init (GstGLWindowEagl * window)
   window->priv = gst_gl_window_eagl_get_instance_private (window);
   window->priv->gl_queue =
       (__bridge_retained gpointer)dispatch_queue_create ("org.freedesktop.gstreamer.glwindow", NULL);
+  g_mutex_init (&window->priv->draw_lock);
+  g_cond_init (&window->priv->cond);
 }
 
 static void
 gst_gl_window_eagl_finalize (GObject * object)
 {
   GstGLWindowEagl *window = GST_GL_WINDOW_EAGL (object);
+
+  if (window->priv->layer)
+    CFRelease (window->priv->layer);
+  window->priv->layer = NULL;
   CFRelease(window->priv->gl_queue);
+  g_mutex_clear (&window->priv->draw_lock);
+  g_cond_clear (&window->priv->cond);
+
   G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
@@ -117,23 +137,89 @@ gst_gl_window_eagl_get_display (GstGLWindow * window)
 static guintptr
 gst_gl_window_eagl_get_window_handle (GstGLWindow * window)
 {
-  return (guintptr) GST_GL_WINDOW_EAGL (window)->priv->view;
+  return (guintptr) GST_GL_WINDOW_EAGL (window)->priv->internal_view;
+}
+
+static void
+_create_gl_window (GstGLWindowEagl * window_eagl)
+{
+  GstGLWindowEaglPrivate *priv = window_eagl->priv;
+  UIView *external_view;
+  CGRect rect;
+  GstGLUIView *view;
+
+  g_mutex_lock (&priv->draw_lock);
+
+  external_view = (__bridge UIView *) priv->external_view;
+  rect = CGRectMake (0, 0, external_view.frame.size.width, external_view.frame.size.height);
+
+  window_eagl->priv->window_width = rect.size.width;
+  window_eagl->priv->window_height = rect.size.height;
+
+  view = [[GstGLUIView alloc] initWithFrame:rect];
+  [view setGstWindow:window_eagl];
+  view.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+  view.contentMode = UIViewContentModeRedraw;
+
+  priv->internal_view = (__bridge_retained gpointer) view;
+  [external_view addSubview:view];
+  priv->internal_view = (__bridge_retained gpointer) view;
+  priv->layer = (__bridge_retained gpointer) [view layer];
+
+  NSDictionary * dict = [NSDictionary dictionaryWithObjectsAndKeys:
+      [NSNumber numberWithBool:NO], kEAGLDrawablePropertyRetainedBacking,
+      kEAGLColorFormatRGBA8, kEAGLDrawablePropertyColorFormat, nil];
+
+  [[view layer] setOpaque:YES];
+  [(CAEAGLLayer *) [view layer] setDrawableProperties:dict];
+
+  g_cond_broadcast (&priv->cond);
+  g_mutex_unlock (&priv->draw_lock);
+}
+
+static void
+ensure_window_handle_is_set_unlocked (GstGLWindowEagl * window_eagl)
+{
+  GstGLContext *context;
+
+  if (!window_eagl->priv->pending_set_window_handle)
+    return;
+
+  context = gst_gl_window_get_context (GST_GL_WINDOW (window_eagl));
+  if (!context) {
+    g_critical ("Window does not have a GstGLContext attached!. "
+        "Aborting set window handle.");
+    g_mutex_unlock (&window_eagl->priv->draw_lock);
+    return;
+  }
+
+  while (!window_eagl->priv->internal_view)
+    g_cond_wait (&window_eagl->priv->cond, &window_eagl->priv->draw_lock);
+
+  GST_INFO_OBJECT (context, "handle set, updating layer");
+  gst_gl_context_eagl_update_layer (context, window_eagl->priv->layer);
+  gst_object_unref (context);
+
+  window_eagl->priv->pending_set_window_handle = FALSE;
 }
 
 static void
 gst_gl_window_eagl_set_window_handle (GstGLWindow * window, guintptr handle)
 {
   GstGLWindowEagl *window_eagl;
-  GstGLContext *context;
 
   window_eagl = GST_GL_WINDOW_EAGL (window);
-  context = gst_gl_window_get_context (window);
 
-  window_eagl->priv->view = (gpointer)handle;
-  GST_INFO_OBJECT (context, "handle set, updating layer");
-  gst_gl_context_eagl_update_layer (context);
+  g_mutex_lock (&window_eagl->priv->draw_lock);
+  if (window_eagl->priv->external_view)
+    CFRelease (window_eagl->priv->external_view);
+  window_eagl->priv->external_view = (gpointer)handle;
+  window_eagl->priv->pending_set_window_handle = TRUE;
+  g_mutex_unlock (&window_eagl->priv->draw_lock);
 
-  gst_object_unref (context);
+  /* XXX: Maybe we need an async set_window_handle? */
+  _gl_invoke_on_main ((GstGLWindowEaglFunc) _create_gl_window,
+      gst_object_ref (window_eagl), gst_object_unref);
 }
 
 static void
@@ -180,21 +266,22 @@ draw_cb (gpointer data)
   GstGLContext *context = gst_gl_window_get_context (window);
   GstGLContextEagl *eagl_context = GST_GL_CONTEXT_EAGL (context);
 
-  if (window_eagl->priv->view) {
+  g_mutex_lock (&window_eagl->priv->draw_lock);
+
+  ensure_window_handle_is_set_unlocked (window_eagl);
+
+  if (window_eagl->priv->internal_view) {
     CGSize size;
     CAEAGLLayer *eagl_layer;
 
-    eagl_layer = (CAEAGLLayer *)[GST_GL_WINDOW_EAGL_VIEW(window_eagl) layer];
+    eagl_layer = GST_GL_WINDOW_EAGL_LAYER (window_eagl);
     size = eagl_layer.frame.size;
 
-    size = CGSizeMake (size.width * eagl_layer.contentsScale,  size.height * eagl_layer.contentsScale);
+    size = CGSizeMake (size.width * eagl_layer.contentsScale, size.height * eagl_layer.contentsScale);
 
     if (window->queue_resize || window_eagl->priv->window_width != size.width ||
         window_eagl->priv->window_height != size.height) {
 
-      window_eagl->priv->window_width = size.width;
-      window_eagl->priv->window_height = size.height;
-
       gst_gl_context_eagl_resize (eagl_context);
 
       gst_gl_window_resize (window, window_eagl->priv->window_width,
@@ -211,6 +298,8 @@ draw_cb (gpointer data)
 
   gst_gl_context_eagl_finish_draw (eagl_context);
 
+  g_mutex_unlock (&window_eagl->priv->draw_lock);
+
   gst_object_unref (context);
 }
 
@@ -219,3 +308,59 @@ gst_gl_window_eagl_draw (GstGLWindow * window)
 {
   gst_gl_window_send_message (window, (GstGLWindowCB) draw_cb, window);
 }
+
+gpointer
+gst_gl_window_eagl_get_layer (GstGLWindowEagl * window_eagl)
+{
+  gpointer layer;
+
+  g_mutex_lock (&window_eagl->priv->draw_lock);
+  if (window_eagl->priv->layer)
+    CFRetain (window_eagl->priv->layer);
+  layer = window_eagl->priv->layer;
+  g_mutex_unlock (&window_eagl->priv->draw_lock);
+
+  return layer;
+}
+
+@implementation GstGLUIView {
+    GstGLWindowEagl * window_eagl;
+};
+
++(Class) layerClass
+{
+  return [CAEAGLLayer class];
+}
+
+-(void) setGstWindow:(GstGLWindowEagl *) window
+{
+  window_eagl = window;
+}
+
+-(void) layoutSubViews
+{
+  g_mutex_lock (&window_eagl->priv->draw_lock);
+  [super layoutSubviews];
+  CGSize rect = self.bounds.size;
+  self->window_eagl->priv->window_width = rect.width;
+  self->window_eagl->priv->window_height = rect.height;
+  g_mutex_unlock (&window_eagl->priv->draw_lock);
+}
+
+@end
+
+void
+_gl_invoke_on_main (GstGLWindowEaglFunc func, gpointer data, GDestroyNotify notify)
+{
+  if ([NSThread isMainThread]) {
+    func (data);
+    if (notify)
+      notify (data);
+  } else {
+    dispatch_async (dispatch_get_main_queue (), ^{
+      func (data);
+      if (notify)
+        notify (data);
+    });
+  }
+}

gst-libs/gst/gl/meson.build

@@ -809,13 +809,15 @@ if host_system == 'ios' and need_platform_eagl != 'no' and need_win_eagl != 'no'
   foundation_dep = dependency('appleframeworks', modules : ['Foundation'], required : false)
   corefoundation_dep = dependency('appleframeworks', modules : ['CoreFoundation'], required : false)
   coregraphics_dep = dependency('appleframeworks', modules : ['CoreGraphics'], required : false)
+  quartzcore_dep = dependency('appleframeworks', modules : ['QuartzCore'], required : false)
   uikit_dep = dependency('appleframeworks', modules : ['UIkit'], required : false)
 
-  if foundation_dep.found() and corefoundation_dep.found() and coregraphics_dep.found() and uikit_dep.found()
+  if foundation_dep.found() and corefoundation_dep.found() and coregraphics_dep.found() and quartzcore_dep.found() and uikit_dep.found()
     gl_platform_deps += [
       corefoundation_dep,
       foundation_dep,
       coregraphics_dep,
+      quartzcore_dep,
       uikit_dep,
     ]
     gl_sources += [