d3d11videosink: Fix deadlock when parent window is busy

Deadlock sequence:
* From a streaming thread, d3d11videosink sends synchronous message
  to the parent window, so that internal (child) window can be
  constructed on the parent window's thread
* App thread (parent window thread) is waiting for pipeline's
  state change (to GST_STATE_NULL) but streaming thread is
  blocked and waiting for app thread

To avoid the deadlock, GstD3D11WindowWin32 should send message
to the parent window asynchronously.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3570>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp

@@ -707,7 +707,7 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
   return TRUE;
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
 {
   gint video_width, video_height;
@@ -716,6 +716,8 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
   guint num, den;
   GError *error = NULL;
   GstStructure *config;
+  GstD3D11Window *window;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   GST_DEBUG_OBJECT (self, "Updating window with caps %" GST_PTR_FORMAT, caps);
 
@@ -728,14 +730,14 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
     GST_ELEMENT_ERROR (self, RESOURCE, NOT_FOUND, (nullptr),
         ("Failed to open window."));
 
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (!gst_video_info_from_caps (&self->info, caps)) {
     GST_DEBUG_OBJECT (self,
         "Could not locate image format from caps %" GST_PTR_FORMAT, caps);
     LeaveCriticalSection (&self->lock);
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   video_width = GST_VIDEO_INFO_WIDTH (&self->info);
@@ -754,7 +756,7 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
 
     GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (nullptr),
         ("Error calculating the output display ratio of the video."));
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   GST_DEBUG_OBJECT (self,
@@ -797,7 +799,7 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
 
     GST_ELEMENT_ERROR (self, CORE, NEGOTIATION, (nullptr),
         ("Error calculating the output display ratio of the video."));
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (self->pending_render_rect) {
@@ -813,30 +815,41 @@ gst_d3d11_video_sink_update_window (GstD3D11VideoSink * self, GstCaps * caps)
       GST_D3D11_CONVERTER_OPT_PRIMARIES_MODE,
       GST_TYPE_VIDEO_PRIMARIES_MODE, self->primaries_mode, nullptr);
 
-  if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
+  window = (GstD3D11Window *) gst_object_ref (self->window);
-          GST_VIDEO_SINK_HEIGHT (self), caps, config, self->display_format,
+  LeaveCriticalSection (&self->lock);
-          &error)) {
+
+  ret = gst_d3d11_window_prepare (window, GST_VIDEO_SINK_WIDTH (self),
+      GST_VIDEO_SINK_HEIGHT (self), caps, config, self->display_format, &error);
+  if (ret != GST_FLOW_OK) {
     GstMessage *error_msg;
 
-    LeaveCriticalSection (&self->lock);
+    if (ret == GST_FLOW_FLUSHING) {
+      GstD3D11CSLockGuard lk (&self->lock);
+      GST_WARNING_OBJECT (self, "Couldn't prepare window but we are flushing");
+      gst_clear_object (&self->window);
+      gst_object_unref (window);
+
+      return GST_FLOW_FLUSHING;
+    }
 
     GST_ERROR_OBJECT (self, "cannot create swapchain");
     error_msg = gst_message_new_error (GST_OBJECT_CAST (self),
         error, "Failed to prepare d3d11window");
     g_clear_error (&error);
     gst_element_post_message (GST_ELEMENT (self), error_msg);
+    gst_object_unref (window);
 
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (self->title) {
-    gst_d3d11_window_set_title (self->window, self->title);
+    gst_d3d11_window_set_title (window, self->title);
     g_clear_pointer (&self->title, g_free);
   }
 
-  LeaveCriticalSection (&self->lock);
+  gst_object_unref (window);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void
@@ -997,6 +1010,9 @@ done:
   g_signal_connect (self->window, "present",
       G_CALLBACK (gst_d3d11_video_sink_present), self);
 
+  GST_DEBUG_OBJECT (self,
+      "Have prepared window %" GST_PTR_FORMAT, self->window);
+
   return TRUE;
 }
 
@@ -1299,17 +1315,16 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
 
   if (self->caps_updated || !self->window) {
     GstCaps *caps = gst_pad_get_current_caps (GST_BASE_SINK_PAD (sink));
-    gboolean update_ret;
 
     /* shouldn't happen */
     if (!caps)
       return GST_FLOW_NOT_NEGOTIATED;
 
-    update_ret = gst_d3d11_video_sink_update_window (self, caps);
+    ret = gst_d3d11_video_sink_update_window (self, caps);
     gst_caps_unref (caps);
 
-    if (!update_ret)
+    if (ret != GST_FLOW_OK)
-      return GST_FLOW_NOT_NEGOTIATED;
+      return ret;
   }
 
   gst_d3d11_window_show (self->window);

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp

@@ -108,7 +108,7 @@ static GstFlowReturn gst_d3d111_window_present (GstD3D11Window * self,
     GstBuffer * buffer, GstBuffer * render_target);
 static void gst_d3d11_window_on_resize_default (GstD3D11Window * window,
     guint width, guint height);
-static gboolean gst_d3d11_window_prepare_default (GstD3D11Window * window,
+static GstFlowReturn gst_d3d11_window_prepare_default (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
     GstStructure * config, DXGI_FORMAT display_format, GError ** error);
 
@@ -517,14 +517,14 @@ typedef struct
   gboolean supported;
 } GstD3D11WindowDisplayFormat;
 
-gboolean
+GstFlowReturn
 gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width,
     guint display_height, GstCaps * caps, GstStructure * config,
     DXGI_FORMAT display_format, GError ** error)
 {
   GstD3D11WindowClass *klass;
 
-  g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
+  g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), GST_FLOW_ERROR);
 
   klass = GST_D3D11_WINDOW_GET_CLASS (window);
   g_assert (klass->prepare != NULL);
@@ -536,7 +536,7 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width,
       display_format, error);
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     guint display_height, GstCaps * caps, GstStructure * config,
     DXGI_FORMAT display_format, GError ** error)
@@ -600,7 +600,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     if (config)
       gst_structure_free (config);
 
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (display_format != DXGI_FORMAT_UNKNOWN) {
@@ -621,7 +621,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
       if (config)
         gst_structure_free (config);
 
-      return FALSE;
+      return GST_FLOW_ERROR;
     }
   } else {
     for (guint i = 0; i < GST_VIDEO_INFO_N_COMPONENTS (&window->info); i++) {
@@ -694,7 +694,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     if (config)
       gst_structure_free (config);
 
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   /* this rect struct will be used to calculate render area */
@@ -782,7 +782,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     GST_ERROR_OBJECT (window, "Cannot create converter");
     g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
         "Cannot create converter");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   if (have_hdr10_meta) {
@@ -800,7 +800,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
     GST_ERROR_OBJECT (window, "Cannot create overlay compositor");
     g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
         "Cannot create overlay compositor");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   /* call resize to allocated resources */
@@ -811,7 +811,7 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width,
 
   GST_DEBUG_OBJECT (window, "New swap chain 0x%p created", window->swap_chain);
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 void

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h

@@ -145,7 +145,7 @@ struct _GstD3D11WindowClass
                                            guint width,
                                            guint height);
 
-  gboolean      (*prepare)                (GstD3D11Window * window,
+  GstFlowReturn (*prepare)                (GstD3D11Window * window,
                                            guint display_width,
                                            guint display_height,
                                            GstCaps * caps,
@@ -181,7 +181,7 @@ void          gst_d3d11_window_set_title            (GstD3D11Window * window,
 void          gst_d3d11_window_set_orientation      (GstD3D11Window * window,
                                                      GstVideoOrientationMethod method);
 
-gboolean      gst_d3d11_window_prepare              (GstD3D11Window * window,
+GstFlowReturn gst_d3d11_window_prepare              (GstD3D11Window * window,
                                                      guint display_width,
                                                      guint display_height,
                                                      GstCaps * caps,

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_dummy.cpp

@@ -50,7 +50,7 @@ G_DEFINE_TYPE (GstD3D11WindowDummy, gst_d3d11_window_dummy,
 
 static void gst_d3d11_window_dummy_on_resize (GstD3D11Window * window,
     guint width, guint height);
-static gboolean gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
+static GstFlowReturn gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
     GstStructure * config, DXGI_FORMAT display_format, GError ** error);
 static void gst_d3d11_window_dummy_unprepare (GstD3D11Window * window);
@@ -82,7 +82,7 @@ gst_d3d11_window_dummy_init (GstD3D11WindowDummy * self)
 {
 }
 
-static gboolean
+static GstFlowReturn
 gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     guint display_width, guint display_height, GstCaps * caps,
     GstStructure * config, DXGI_FORMAT display_format, GError ** error)
@@ -131,7 +131,7 @@ gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     GST_ERROR_OBJECT (window, "Cannot create converter");
     g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
         "Cannot create converter");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
   window->compositor =
@@ -140,10 +140,10 @@ gst_d3d11_window_dummy_prepare (GstD3D11Window * window,
     GST_ERROR_OBJECT (window, "Cannot create overlay compositor");
     g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
         "Cannot create overlay compositor");
-    return FALSE;
+    return GST_FLOW_ERROR;
   }
 
-  return TRUE;
+  return GST_FLOW_OK;
 }
 
 static void

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window_win32.cpp

@@ -92,6 +92,9 @@ struct _GstD3D11WindowWin32
 
   /* Handle set_render_rectangle */
   GstVideoRectangle render_rect;
+
+  gboolean flushing;
+  gboolean setup_external_hwnd;
 };
 
 #define gst_d3d11_window_win32_parent_class parent_class
@@ -118,17 +121,21 @@ gst_d3d11_window_win32_create_internal_window (GstD3D11WindowWin32 * self);
 static void gst_d3d11_window_win32_destroy_internal_window (HWND hwnd);
 static void gst_d3d11_window_win32_release_external_handle (HWND hwnd);
 static void
-gst_d3d11_window_win32_set_window_handle (GstD3D11WindowWin32 * self,
-    guintptr handle);
-static void
 gst_d3d11_window_win32_on_resize (GstD3D11Window * window,
     guint width, guint height);
+static GstFlowReturn gst_d3d11_window_win32_prepare (GstD3D11Window * window,
+    guint display_width, guint display_height, GstCaps * caps,
+    GstStructure * config, DXGI_FORMAT display_format, GError ** error);
 static void gst_d3d11_window_win32_unprepare (GstD3D11Window * window);
 static void
 gst_d3d11_window_win32_set_render_rectangle (GstD3D11Window * window,
     const GstVideoRectangle * rect);
 static void gst_d3d11_window_win32_set_title (GstD3D11Window * window,
     const gchar * title);
+static gboolean gst_d3d11_window_win32_unlock (GstD3D11Window * window);
+static gboolean gst_d3d11_window_win32_unlock_stop (GstD3D11Window * window);
+static GstFlowReturn
+gst_d3d11_window_win32_set_external_handle (GstD3D11WindowWin32 * self);
 
 static void
 gst_d3d11_window_win32_class_init (GstD3D11WindowWin32Class * klass)
@@ -149,12 +156,16 @@ gst_d3d11_window_win32_class_init (GstD3D11WindowWin32Class * klass)
   window_class->present = GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_present);
   window_class->on_resize =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_on_resize);
+  window_class->prepare = GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_prepare);
   window_class->unprepare =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_unprepare);
   window_class->set_render_rectangle =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_set_render_rectangle);
   window_class->set_title =
       GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_set_title);
+  window_class->unlock = GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_unlock);
+  window_class->unlock_stop =
+      GST_DEBUG_FUNCPTR (gst_d3d11_window_win32_unlock_stop);
 }
 
 static void
@@ -170,7 +181,9 @@ gst_d3d11_window_win32_constructed (GObject * object)
   GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (object);
 
   if (window->external_handle) {
-    gst_d3d11_window_win32_set_window_handle (self, window->external_handle);
+    /* Will setup internal child window on ::prepare() */
+    self->setup_external_hwnd = TRUE;
+    window->initialized = TRUE;
     goto done;
   }
 
@@ -194,6 +207,51 @@ gst_d3d11_window_win32_dispose (GObject * object)
   G_OBJECT_CLASS (parent_class)->dispose (object);
 }
 
+static GstFlowReturn
+gst_d3d11_window_win32_prepare (GstD3D11Window * window, guint display_width,
+    guint display_height, GstCaps * caps, GstStructure * config,
+    DXGI_FORMAT display_format, GError ** error)
+{
+  GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
+  HWND hwnd;
+  GstFlowReturn ret;
+
+  if (!self->setup_external_hwnd)
+    goto done;
+
+  hwnd = (HWND) window->external_handle;
+  if (!IsWindow (hwnd)) {
+    GST_ERROR_OBJECT (self, "Invalid window handle");
+    g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
+        "Invalid window handle");
+    return GST_FLOW_ERROR;
+  }
+
+  self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_NONE;
+  self->external_hwnd = hwnd;
+
+  GST_DEBUG_OBJECT (self, "Preparing external handle");
+  ret = gst_d3d11_window_win32_set_external_handle (self);
+  if (ret != GST_FLOW_OK) {
+    if (ret == GST_FLOW_FLUSHING) {
+      GST_WARNING_OBJECT (self, "Flushing");
+      return GST_FLOW_FLUSHING;
+    }
+
+    GST_ERROR_OBJECT (self, "Couldn't configure internal window");
+    g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
+        "Window handle configuration failed");
+    return GST_FLOW_ERROR;
+  }
+
+  GST_DEBUG_OBJECT (self, "External handle got prepared");
+  self->setup_external_hwnd = FALSE;
+
+done:
+  return GST_D3D11_WINDOW_CLASS (parent_class)->prepare (window, display_width,
+      display_height, caps, config, display_format, error);
+}
+
 static void
 gst_d3d11_window_win32_unprepare (GstD3D11Window * window)
 {
@@ -293,6 +351,34 @@ gst_d3d11_window_win32_set_title (GstD3D11Window * window, const gchar * title)
   }
 }
 
+static gboolean
+gst_d3d11_window_win32_unlock (GstD3D11Window * window)
+{
+  GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
+  GstD3D11SRWLockGuard lk (&self->lock);
+
+  GST_DEBUG_OBJECT (self, "Unlock");
+
+  self->flushing = TRUE;
+  WakeAllConditionVariable (&self->cond);
+
+  return TRUE;
+}
+
+static gboolean
+gst_d3d11_window_win32_unlock_stop (GstD3D11Window * window)
+{
+  GstD3D11WindowWin32 *self = GST_D3D11_WINDOW_WIN32 (window);
+  GstD3D11SRWLockGuard lk (&self->lock);
+
+  GST_DEBUG_OBJECT (self, "Unlock stop");
+
+  self->flushing = FALSE;
+  WakeAllConditionVariable (&self->cond);
+
+  return TRUE;
+}
+
 static gboolean
 running_cb (gpointer user_data)
 {
@@ -384,10 +470,11 @@ gst_d3d11_window_win32_destroy_internal_window (HWND hwnd)
         ", 0x%x", (guintptr) hwnd, (guint) GetLastError ());
 }
 
-static void
+static GstFlowReturn
 gst_d3d11_window_win32_set_external_handle (GstD3D11WindowWin32 * self)
 {
   WNDPROC external_window_proc;
+  GstFlowReturn ret = GST_FLOW_OK;
 
   external_window_proc =
       (WNDPROC) GetWindowLongPtrA (self->external_hwnd, GWLP_WNDPROC);
@@ -402,9 +489,27 @@ gst_d3d11_window_win32_set_external_handle (GstD3D11WindowWin32 * self)
   SetWindowLongPtrA (self->external_hwnd, GWLP_WNDPROC,
       (LONG_PTR) sub_class_proc);
 
-  /* Will create our internal window on parent window's thread */
+  /* SendMessage() may cause deadlock if parent window thread is busy
-  SendMessageA (self->external_hwnd, WM_GST_D3D11_CONSTRUCT_INTERNAL_WINDOW,
+   * for changing pipeline's state. Post our message instead, and wait for
+   * the parent window's thread or flushing */
+  PostMessageA (self->external_hwnd, WM_GST_D3D11_CONSTRUCT_INTERNAL_WINDOW,
       0, 0);
+
+  GstD3D11SRWLockGuard lk (&self->lock);
+  while (self->external_hwnd &&
+      self->overlay_state == GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_NONE &&
+      !self->flushing) {
+    SleepConditionVariableSRW (&self->cond, &self->lock, INFINITE, 0);
+  }
+
+  if (self->overlay_state != GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_OPENED) {
+    if (self->flushing)
+      ret = GST_FLOW_FLUSHING;
+    else
+      ret = GST_FLOW_ERROR;
+  }
+
+  return ret;
 }
 
 static void
@@ -809,6 +914,10 @@ sub_class_proc (HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
     MoveWindow (self->internal_hwnd, rect.left, rect.top, rect.right,
         rect.bottom, FALSE);
 
+    GstD3D11SRWLockGuard lk (&self->lock);
+    self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_OPENED;
+    WakeAllConditionVariable (&self->cond);
+
     /* don't need to be chained up to parent window procedure,
      * as this is our custom message */
     return 0;
@@ -822,13 +931,16 @@ sub_class_proc (HWND hWnd, UINT uMsg, WPARAM wParam, LPARAM lParam)
       gst_d3d11_window_win32_release_external_handle (self->external_hwnd);
       self->external_hwnd = NULL;
 
-      RemovePropA (self->internal_hwnd, D3D11_WINDOW_PROP_NAME);
+      if (self->internal_hwnd) {
-      ShowWindow (self->internal_hwnd, SW_HIDE);
+        RemovePropA (self->internal_hwnd, D3D11_WINDOW_PROP_NAME);
-      gst_d3d11_window_win32_destroy_internal_window (self->internal_hwnd);
+        ShowWindow (self->internal_hwnd, SW_HIDE);
+        gst_d3d11_window_win32_destroy_internal_window (self->internal_hwnd);
+      }
       self->internal_hwnd = NULL;
       self->internal_hwnd_thread = NULL;
 
       self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_CLOSED;
+      WakeAllConditionVariable (&self->cond);
     } else {
       gst_d3d11_window_win32_handle_window_proc (self, hWnd, uMsg, wParam,
           lParam);
@@ -996,18 +1108,6 @@ gst_d3d11_window_win32_create_swap_chain (GstD3D11Window * window,
   return TRUE;
 }
 
-static void
-gst_d3d11_window_win32_set_window_handle (GstD3D11WindowWin32 * self,
-    guintptr handle)
-{
-  self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_NONE;
-
-  self->external_hwnd = (HWND) handle;
-  gst_d3d11_window_win32_set_external_handle (self);
-
-  self->overlay_state = GST_D3D11_WINDOW_WIN32_OVERLAY_STATE_OPENED;
-}
-
 static void
 gst_d3d11_window_win32_show (GstD3D11Window * window)
 {