From ac11ccd4fff0bcff78be2f7b29e4806b3a9fdb22 Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Wed, 15 Nov 2023 01:11:24 +0900
Subject: [PATCH] dwrite: Protect entire draw operation with D3D11 lock

d2d runtime seems to execute pending GPU command list
when DXGI ID2D1RenderTarget is being released, and it will invoke
d3d11 immediate context APIs. Should protect all rendering operations
and DXGI resources with lock.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5659>
---
 .../sys/dwrite/gstdwriteoverlayobject.cpp     | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp
index 589abf8ecc..fe42315317 100644
--- a/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp
+++ b/subprojects/gst-plugins-bad/sys/dwrite/gstdwriteoverlayobject.cpp
@@ -1128,10 +1128,8 @@ gst_dwrite_overlay_object_draw_layout (GstDWriteOverlayObject * self,
       return FALSE;
     }
 
-    gst_d3d11_device_lock (priv->device);
     if (!gst_dwrite_overlay_object_get_target_from_d3d11 (self, mem, &target)) {
       GST_ERROR_OBJECT (self, "Couldn't get target from texture");
-      gst_d3d11_device_unlock (priv->device);
       gst_memory_unmap (mem, &info);
       gst_clear_buffer (&priv->layout_buf);
       return FALSE;
@@ -1144,10 +1142,13 @@ gst_dwrite_overlay_object_draw_layout (GstDWriteOverlayObject * self,
       D2D1::Rect (0, 0, width, height), layout, target.Get ());
   target->EndDraw ();
 
-  if (!priv->use_bitmap) {
-    gst_d3d11_device_unlock (priv->device);
+  /* Release render target before unmapping. Otherwise pending GPU operations
+   * can be executed after releasing keyed-mutex, if texture was allocated with
+   * keyed-mutex enabled */
+  target = nullptr;
+
+  if (!priv->use_bitmap)
     gst_memory_unmap (mem, &info);
-  }
 
   priv->overlay_rect = gst_video_overlay_rectangle_new_raw (priv->layout_buf,
       x, y, width, height, GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
@@ -1245,7 +1246,6 @@ gst_dwrite_overlay_mode_convert (GstDWriteOverlayObject * self,
     return FALSE;
   }
 
-  gst_d3d11_device_lock (priv->device);
   if (!gst_d3d11_converter_convert_buffer_unlocked (priv->pre_conv,
           buffer, pre_buf)) {
     GST_ERROR_OBJECT (self, "pre-convert failed");
@@ -1264,13 +1264,11 @@ gst_dwrite_overlay_mode_convert (GstDWriteOverlayObject * self,
     goto error;
   }
 
-  gst_d3d11_device_unlock (priv->device);
   gst_buffer_unref (pre_buf);
 
   return TRUE;
 
 error:
-  gst_d3d11_device_unlock (priv->device);
   gst_clear_buffer (&pre_buf);
   return FALSE;
 }
@@ -1282,8 +1280,11 @@ gst_dwrite_overlay_object_draw (GstDWriteOverlayObject * object,
   GstDWriteOverlayObjectPrivate *priv = object->priv;
   gboolean ret = FALSE;
 
+  if (priv->device)
+    gst_d3d11_device_lock (priv->device);
+
   if (!gst_dwrite_overlay_object_draw_layout (object, layout, x, y))
-    return FALSE;
+    goto out;
 
   switch (priv->blend_mode) {
     case GstDWriteBlendMode::ATTACH_TEXTURE:
@@ -1302,8 +1303,11 @@ gst_dwrite_overlay_object_draw (GstDWriteOverlayObject * object,
       break;
     default:
       g_assert_not_reached ();
-      return FALSE;
+      break;
   }
 
+out:
+  if (priv->device)
+    gst_d3d11_device_unlock (priv->device);
   return ret;
 }