d3d11videosink: Fixing unexpected overlay composition meta drops

A buffer can hold multiple GstVideoOverlayCompositionMeta objects.
Should scan all metas on a buffer and render all overlay rectangles
if any.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4862>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11overlaycompositor.cpp

@@ -24,6 +24,9 @@
 #include "gstd3d11overlaycompositor.h"
 #include "gstd3d11pluginutils.h"
 #include <wrl.h>
+#include <memory>
+#include <vector>
+#include <algorithm>
 
 GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_overlay_compositor_debug);
 #define GST_CAT_DEFAULT gst_d3d11_overlay_compositor_debug
@@ -95,7 +98,23 @@ static const gchar templ_vertex_shader[] =
     "{\n"
     "  return input;\n"
     "}\n";
-/* *INDENT-ON* */
+
+struct GstD3D11CompositionOverlay
+{
+  ~GstD3D11CompositionOverlay ()
+  {
+    if (overlay_rect)
+      gst_video_overlay_rectangle_unref (overlay_rect);
+  }
+
+  GstVideoOverlayRectangle *overlay_rect = nullptr;
+  ComPtr<ID3D11Texture2D> texture;
+  ComPtr<ID3D11ShaderResourceView> srv;
+  ComPtr<ID3D11Buffer> vertex_buffer;
+  gboolean premul_alpha = FALSE;
+};
+
+typedef std::shared_ptr<GstD3D11CompositionOverlay> GstD3D11CompositionOverlayPtr;
 
 struct _GstD3D11OverlayCompositorPrivate
 {
@@ -103,33 +122,22 @@ struct _GstD3D11OverlayCompositorPrivate
 
   D3D11_VIEWPORT viewport;
 
-  ID3D11PixelShader *ps;
+  ComPtr<ID3D11PixelShader> ps;
-  ID3D11PixelShader *premul_ps;
+  ComPtr<ID3D11PixelShader> premul_ps;
-  ID3D11VertexShader *vs;
+  ComPtr<ID3D11VertexShader> vs;
-  ID3D11InputLayout *layout;
+  ComPtr<ID3D11InputLayout> layout;
-  ID3D11SamplerState *sampler;
+  ComPtr<ID3D11SamplerState> sampler;
-  ID3D11BlendState *blend;
+  ComPtr<ID3D11BlendState> blend;
-  ID3D11Buffer *index_buffer;
+  ComPtr<ID3D11Buffer> index_buffer;
 
-  /* GstD3D11CompositionOverlay */
+  std::vector<GstD3D11CompositionOverlayPtr> overlays;
-  GList *overlays;
 };
+/* *INDENT-ON* */
 
-typedef struct
+static void gst_d3d11_overlay_compositor_finalize (GObject * object);
-{
-  GstVideoOverlayRectangle *overlay_rect;
-  ID3D11Texture2D *texture;
-  ID3D11ShaderResourceView *srv;
-  ID3D11Buffer *vertex_buffer;
-  gboolean premul_alpha;
-} GstD3D11CompositionOverlay;
-
-static void gst_d3d11_overlay_compositor_dispose (GObject * object);
-static void
-gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * self);
 
 #define gst_d3d11_overlay_compositor_parent_class parent_class
-G_DEFINE_TYPE_WITH_PRIVATE (GstD3D11OverlayCompositor,
+G_DEFINE_TYPE (GstD3D11OverlayCompositor,
     gst_d3d11_overlay_compositor, GST_TYPE_OBJECT);
 
 static void
@@ -137,43 +145,32 @@ gst_d3d11_overlay_compositor_class_init (GstD3D11OverlayCompositorClass * klass)
 {
   GObjectClass *object_class = G_OBJECT_CLASS (klass);
 
-  object_class->dispose = gst_d3d11_overlay_compositor_dispose;
+  object_class->finalize = gst_d3d11_overlay_compositor_finalize;
 }
 
 static void
 gst_d3d11_overlay_compositor_init (GstD3D11OverlayCompositor * self)
 {
-  self->priv = (GstD3D11OverlayCompositorPrivate *)
+  self->priv = new GstD3D11OverlayCompositorPrivate ();
-      gst_d3d11_overlay_compositor_get_instance_private (self);
 }
 
 static void
-gst_d3d11_overlay_compositor_dispose (GObject * object)
+gst_d3d11_overlay_compositor_finalize (GObject * object)
 {
   GstD3D11OverlayCompositor *self = GST_D3D11_OVERLAY_COMPOSITOR (object);
-  GstD3D11OverlayCompositorPrivate *priv = self->priv;
 
-  gst_d3d11_overlay_compositor_free_overlays (self);
+  delete self->priv;
-
-  GST_D3D11_CLEAR_COM (priv->ps);
-  GST_D3D11_CLEAR_COM (priv->premul_ps);
-  GST_D3D11_CLEAR_COM (priv->vs);
-  GST_D3D11_CLEAR_COM (priv->layout);
-  GST_D3D11_CLEAR_COM (priv->sampler);
-  GST_D3D11_CLEAR_COM (priv->blend);
-  GST_D3D11_CLEAR_COM (priv->index_buffer);
 
   gst_clear_object (&self->device);
 
-  G_OBJECT_CLASS (parent_class)->dispose (object);
+  G_OBJECT_CLASS (parent_class)->finalize (object);
 }
 
-static GstD3D11CompositionOverlay *
+static GstD3D11CompositionOverlayPtr
 gst_d3d11_composition_overlay_new (GstD3D11OverlayCompositor * self,
     GstVideoOverlayRectangle * overlay_rect)
 {
   GstD3D11OverlayCompositorPrivate *priv = self->priv;
-  GstD3D11CompositionOverlay *overlay = nullptr;
   gint x, y;
   guint width, height;
   D3D11_SUBRESOURCE_DATA subresource_data;
@@ -343,32 +340,16 @@ gst_d3d11_composition_overlay_new (GstD3D11OverlayCompositor * self,
 
   context_handle->Unmap (vertex_buffer.Get (), 0);
 
-  overlay = g_new0 (GstD3D11CompositionOverlay, 1);
+  auto overlay = std::make_shared < GstD3D11CompositionOverlay > ();
   overlay->overlay_rect = gst_video_overlay_rectangle_ref (overlay_rect);
-  overlay->texture = texture.Detach ();
+  overlay->texture = texture;
-  overlay->srv = srv.Detach ();
+  overlay->srv = srv;
-  overlay->vertex_buffer = vertex_buffer.Detach ();
+  overlay->vertex_buffer = vertex_buffer;
   overlay->premul_alpha = premul_alpha;
 
   return overlay;
 }
 
-static void
-gst_d3d11_composition_overlay_free (GstD3D11CompositionOverlay * overlay)
-{
-  if (!overlay)
-    return;
-
-  if (overlay->overlay_rect)
-    gst_video_overlay_rectangle_unref (overlay->overlay_rect);
-
-  GST_D3D11_CLEAR_COM (overlay->srv);
-  GST_D3D11_CLEAR_COM (overlay->texture);
-  GST_D3D11_CLEAR_COM (overlay->vertex_buffer);
-
-  g_free (overlay);
-}
-
 static gboolean
 gst_d3d11_overlay_compositor_setup_shader (GstD3D11OverlayCompositor * self)
 {
@@ -550,39 +531,27 @@ gst_d3d11_overlay_compositor_new (GstD3D11Device * device,
   return self;
 }
 
-static void
-gst_d3d11_overlay_compositor_free_overlays (GstD3D11OverlayCompositor * self)
-{
-  GstD3D11OverlayCompositorPrivate *priv = self->priv;
-
-  if (priv->overlays) {
-    g_list_free_full (priv->overlays,
-        (GDestroyNotify) gst_d3d11_composition_overlay_free);
-
-    priv->overlays = nullptr;
-  }
-}
-
-static gint
-find_in_compositor (const GstD3D11CompositionOverlay * overlay,
-    const GstVideoOverlayRectangle * rect)
-{
-  return !(overlay->overlay_rect == rect);
-}
-
 static gboolean
-is_in_video_overlay_composition (GstVideoOverlayComposition * voc,
+gst_d3d11_overlay_compositor_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
-    GstD3D11CompositionOverlay * overlay)
+    std::vector < GstVideoOverlayRectangle * >*overlay_rect)
 {
-  guint i;
+  GstVideoOverlayCompositionMeta *cmeta;
+  guint num_rect;
 
-  for (i = 0; i < gst_video_overlay_composition_n_rectangles (voc); i++) {
+  if ((*meta)->info->api != GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE)
-    GstVideoOverlayRectangle *rectangle =
+    return TRUE;
-        gst_video_overlay_composition_get_rectangle (voc, i);
+
-    if (overlay->overlay_rect == rectangle)
+  cmeta = (GstVideoOverlayCompositionMeta *) (*meta);
-      return TRUE;
+  if (!cmeta->overlay)
+    return TRUE;
+
+  num_rect = gst_video_overlay_composition_n_rectangles (cmeta->overlay);
+  for (guint i = 0; i < num_rect; i++) {
+    auto rect = gst_video_overlay_composition_get_rectangle (cmeta->overlay, i);
+    overlay_rect->push_back (rect);
   }
-  return FALSE;
+
+  return TRUE;
 }
 
 gboolean
@@ -590,61 +559,64 @@ gst_d3d11_overlay_compositor_upload (GstD3D11OverlayCompositor * compositor,
     GstBuffer * buf)
 {
   GstD3D11OverlayCompositorPrivate *priv;
-  GstVideoOverlayCompositionMeta *meta;
+  std::vector < GstVideoOverlayRectangle * >new_overlay_rect;
-  gint i, num_overlays;
-  GList *iter;
 
   g_return_val_if_fail (compositor != nullptr, FALSE);
   g_return_val_if_fail (GST_IS_BUFFER (buf), FALSE);
 
   priv = compositor->priv;
 
-  meta = gst_buffer_get_video_overlay_composition_meta (buf);
+  gst_buffer_foreach_meta (buf,
-  if (!meta) {
+      (GstBufferForeachMetaFunc) gst_d3d11_overlay_compositor_foreach_meta,
-    gst_d3d11_overlay_compositor_free_overlays (compositor);
+      &new_overlay_rect);
+
+  if (new_overlay_rect.empty ()) {
+    priv->overlays.clear ();
     return TRUE;
   }
 
-  num_overlays = gst_video_overlay_composition_n_rectangles (meta->overlay);
+  GST_LOG_OBJECT (compositor, "Found %" G_GSIZE_FORMAT
-  if (!num_overlays) {
+      " overlay rectangles, %" G_GSIZE_FORMAT " in current queue",
-    gst_d3d11_overlay_compositor_free_overlays (compositor);
+      new_overlay_rect.size (), priv->overlays.size ());
-    return TRUE;
-  }
 
-  GST_LOG_OBJECT (compositor, "Upload %d overlay rectangles", num_overlays);
+  /* *INDENT-OFF* */
-
+  for (auto it : new_overlay_rect) {
-  /* Upload new overlay */
+    if (std::find_if (priv->overlays.begin (), priv->overlays.end (),
-  for (i = 0; i < num_overlays; i++) {
+          [&] (const auto & overlay) -> bool {
-    GstVideoOverlayRectangle *rectangle =
+            return overlay->overlay_rect == it;
-        gst_video_overlay_composition_get_rectangle (meta->overlay, i);
+           }) == priv->overlays.end ()) {
-
+      auto new_overlay = gst_d3d11_composition_overlay_new (compositor, it);
-    if (!g_list_find_custom (priv->overlays,
+      if (!new_overlay)
-            rectangle, (GCompareFunc) find_in_compositor)) {
-      GstD3D11CompositionOverlay *overlay = nullptr;
-
-      overlay = gst_d3d11_composition_overlay_new (compositor, rectangle);
-
-      if (!overlay)
         return FALSE;
 
-      priv->overlays = g_list_append (priv->overlays, overlay);
+      priv->overlays.push_back (new_overlay);
     }
   }
+  /* *INDENT-ON* */
+
+  GST_LOG_OBJECT (compositor, "Overlay rectangles in queue after uploaded %"
+      G_GSIZE_FORMAT, priv->overlays.size ());
 
   /* Remove old overlay */
-  iter = priv->overlays;
+  /* *INDENT-OFF* */
-  while (iter) {
+  auto it = priv->overlays.begin ();
-    GstD3D11CompositionOverlay *overlay =
+  while (it != priv->overlays.end ()) {
-        (GstD3D11CompositionOverlay *) iter->data;
+    auto old_overlay = *it;
-    GList *next = iter->next;
+    if (std::find_if (new_overlay_rect.begin (), new_overlay_rect.end (),
-
+          [&] (const auto & overlay) -> bool {
-    if (!is_in_video_overlay_composition (meta->overlay, overlay)) {
+            return overlay == old_overlay->overlay_rect;
-      priv->overlays = g_list_delete_link (priv->overlays, iter);
+          }) == new_overlay_rect.end ()) {
-      gst_d3d11_composition_overlay_free (overlay);
+      GST_LOG_OBJECT (compositor, "Removing %p from queue",
+          old_overlay->overlay_rect);
+      it = priv->overlays.erase (it);
+    } else {
+      it++;
     }
-
-    iter = next;
   }
+  /* *INDENT-ON* */
+
+  GST_LOG_OBJECT (compositor, "Final queue size %" G_GSIZE_FORMAT,
+      priv->overlays.size ());
 
   return TRUE;
 }
@@ -677,45 +649,49 @@ gst_d3d11_overlay_compositor_draw_unlocked (GstD3D11OverlayCompositor *
     compositor, ID3D11RenderTargetView * rtv[GST_VIDEO_MAX_PLANES])
 {
   GstD3D11OverlayCompositorPrivate *priv;
-  GList *iter;
   ID3D11DeviceContext *context;
   ID3D11ShaderResourceView *clear_view[GST_VIDEO_MAX_PLANES] = { nullptr, };
   UINT strides = sizeof (VertexData);
   UINT offsets = 0;
+  ID3D11SamplerState *samplers[1];
 
   g_return_val_if_fail (compositor != nullptr, FALSE);
   g_return_val_if_fail (rtv != nullptr, FALSE);
 
   priv = compositor->priv;
 
-  if (!priv->overlays)
+  if (priv->overlays.empty ())
     return TRUE;
 
+  samplers[0] = priv->sampler.Get ();
+
   context = gst_d3d11_device_get_device_context_handle (compositor->device);
   context->IASetPrimitiveTopology (D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
-  context->IASetInputLayout (priv->layout);
+  context->IASetInputLayout (priv->layout.Get ());
-  context->IASetIndexBuffer (priv->index_buffer, DXGI_FORMAT_R16_UINT, 0);
+  context->IASetIndexBuffer (priv->index_buffer.Get (),
-  context->PSSetSamplers (0, 1, &priv->sampler);
+      DXGI_FORMAT_R16_UINT, 0);
-  context->VSSetShader (priv->vs, nullptr, 0);
+  context->PSSetSamplers (0, 1, samplers);
+  context->VSSetShader (priv->vs.Get (), nullptr, 0);
   context->RSSetViewports (1, &priv->viewport);
   context->OMSetRenderTargets (1, rtv, nullptr);
-  context->OMSetBlendState (priv->blend, nullptr, 0xffffffff);
+  context->OMSetBlendState (priv->blend.Get (), nullptr, 0xffffffff);
 
-  for (iter = priv->overlays; iter; iter = g_list_next (iter)) {
+  /* *INDENT-OFF* */
-    GstD3D11CompositionOverlay *overlay =
+  for (auto overlay : priv->overlays) {
-        (GstD3D11CompositionOverlay *) iter->data;
+    ID3D11ShaderResourceView *srv[] = { overlay->srv.Get () };
+    ID3D11Buffer *vertex_buf[] = { overlay->vertex_buffer.Get () };
 
     if (overlay->premul_alpha)
-      context->PSSetShader (priv->premul_ps, nullptr, 0);
+      context->PSSetShader (priv->premul_ps.Get (), nullptr, 0);
     else
-      context->PSSetShader (priv->ps, nullptr, 0);
+      context->PSSetShader (priv->ps.Get (), nullptr, 0);
 
-    context->PSSetShaderResources (0, 1, &overlay->srv);
+    context->PSSetShaderResources (0, 1, srv);
-    context->IASetVertexBuffers (0,
+    context->IASetVertexBuffers (0, 1, vertex_buf, &strides, &offsets);
-        1, &overlay->vertex_buffer, &strides, &offsets);
 
     context->DrawIndexed (6, 0, 0);
   }
+  /* *INDENT-ON* */
 
   context->PSSetShaderResources (0, 1, clear_view);
   context->OMSetRenderTargets (0, nullptr, nullptr);

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp

@@ -1358,6 +1358,27 @@ gst_d3d11_video_sink_check_device_update (GstD3D11VideoSink * self,
   self->device = (GstD3D11Device *) gst_object_ref (dmem->device);
 }
 
+static gboolean
+gst_d3d11_video_sink_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
+    GstBuffer * uploaded)
+{
+  GstVideoOverlayCompositionMeta *cmeta;
+
+  if ((*meta)->info->api != GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE)
+    return TRUE;
+
+  cmeta = (GstVideoOverlayCompositionMeta *) (*meta);
+  if (!cmeta->overlay)
+    return TRUE;
+
+  if (gst_video_overlay_composition_n_rectangles (cmeta->overlay) == 0)
+    return TRUE;
+
+  gst_buffer_add_video_overlay_composition_meta (uploaded, cmeta->overlay);
+
+  return TRUE;
+}
+
 static GstFlowReturn
 gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
 {
@@ -1382,8 +1403,6 @@ gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
   }
 
   if (!gst_is_d3d11_buffer (buffer)) {
-    GstVideoOverlayCompositionMeta *overlay_meta;
-
     ret = gst_buffer_pool_acquire_buffer (self->pool, &self->prepared_buffer,
         nullptr);
     if (ret != GST_FLOW_OK)
@@ -1404,11 +1423,9 @@ gst_d3d11_video_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
       gst_memory_unmap (mem, &info);
     }
 
-    overlay_meta = gst_buffer_get_video_overlay_composition_meta (buffer);
+    gst_buffer_foreach_meta (buffer,
-    if (overlay_meta) {
+        (GstBufferForeachMetaFunc) gst_d3d11_video_sink_foreach_meta,
-      gst_buffer_add_video_overlay_composition_meta (self->prepared_buffer,
+        self->prepared_buffer);
-          overlay_meta->overlay);
-    }
   } else {
     self->prepared_buffer = gst_buffer_ref (buffer);
   }