From 57029ba098fcd2c26df6cc116ffb02171ac36917 Mon Sep 17 00:00:00 2001
From: Alexander Vandenbulcke <av@telecom-it.be>
Date: Fri, 8 Jan 2021 11:51:36 +0100
Subject: [PATCH] gl/dispmanx: fix deadlock triggered by `set_render_rectangle`

When the gstglimagesink is started with the option `glimagesink
render-rectangle="<0,0,1920,1080>"`, the pipeline reaches a deadlock.
The reason the deadlock occurs is that the
`gst_gl_window_set_render_rectangle` takes locks on the window, in
addition it calls `window_class->set_render_rectangle(...)` which
executes the `_on_resize` function. Since the `_on_resize` function also
takes locks on the window the deadlock is achieved.

By scheduling the adjustment of the render rectangle through an async
message for `gst_gl_window_dispmanx_set_render_rectangle`, the actual
resize happens in another context and therefore doesn't suffers from the
lock taken in `gst_gl_window_set_render_rectangle`.

This solution follows the same approach as gl/wayland. The problem was
introduced by b887db1. For the full discussion check #849.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/1030>
---
 .../gl/dispmanx/gstglwindow_dispmanx_egl.c    | 47 ++++++++++++++++---
 1 file changed, 41 insertions(+), 6 deletions(-)

diff --git a/gst-libs/gst/gl/dispmanx/gstglwindow_dispmanx_egl.c b/gst-libs/gst/gl/dispmanx/gstglwindow_dispmanx_egl.c
index b749eff379..094888e327 100644
--- a/gst-libs/gst/gl/dispmanx/gstglwindow_dispmanx_egl.c
+++ b/gst-libs/gst/gl/dispmanx/gstglwindow_dispmanx_egl.c
@@ -286,18 +286,53 @@ window_resize (GstGLWindowDispmanxEGL * window_egl, guint width, guint height,
   window_egl->native.height = height;
 }
 
+struct SetRenderRectangle
+{
+  GstGLWindowDispmanxEGL *window_egl;
+  GstVideoRectangle rect;
+};
+
+static void
+_free_set_render_rectangle (struct SetRenderRectangle *render)
+{
+  if (render) {
+    if (render->window_egl)
+      gst_object_unref (render->window_egl);
+    g_free (render);
+  }
+}
+
+static void
+_set_render_rectangle (gpointer data)
+{
+  struct SetRenderRectangle *render = data;
+
+  GST_LOG_OBJECT (render->window_egl, "setting render rectangle %i,%i+%ix%i",
+      render->rect.x, render->rect.y, render->rect.w, render->rect.h);
+
+  window_resize (render->window_egl, render->rect.w, render->rect.h, TRUE);
+
+  render->window_egl->render_rect = render->rect;
+}
+
 static gboolean
 gst_gl_window_dispmanx_egl_set_render_rectangle (GstGLWindow * window,
     gint x, gint y, gint width, gint height)
 {
   GstGLWindowDispmanxEGL *window_egl = GST_GL_WINDOW_DISPMANX_EGL (window);
-  window_egl->render_rect.x = x;
-  window_egl->render_rect.y = y;
-  window_egl->render_rect.w = width;
-  window_egl->render_rect.h = height;
+  struct SetRenderRectangle *render;
+
+  render = g_new0 (struct SetRenderRectangle, 1);
+  render->window_egl = gst_object_ref (window_egl);
+  render->rect.x = x;
+  render->rect.y = y;
+  render->rect.w = width;
+  render->rect.h = height;
+
+  gst_gl_window_send_message_async (window,
+      (GstGLWindowCB) _set_render_rectangle, render,
+      (GDestroyNotify) _free_set_render_rectangle);
 
-  window_resize (window_egl, window_egl->render_rect.w,
-      window_egl->render_rect.h, TRUE);
   return TRUE;
 }