d3d12: Pass target rectangle to ClearRenderTargetView()

Some drivers seem to be crashing if ClearRenderTargetView() is called
for P010/P016 texture's second subresource (UV plane) without
specified target rectangle.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5952>

subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.cpp

@@ -2178,6 +2178,7 @@ gst_d3d12_compositor_draw_background (GstD3D12Compositor * self)
   ClearColor *color = &priv->clear_color[0];
   auto bg_render = priv->bg_render.get ();
   auto & rtv_handles = priv->rtv_handles;
+  std::vector < D3D12_RECT > rtv_rects;
 
   rtv_handles.clear ();
   for (guint i = 0; i < gst_buffer_n_memory (priv->generated_output_buf); i++) {
@@ -2196,6 +2197,9 @@ gst_d3d12_compositor_draw_background (GstD3D12Compositor * self)
         (rtv_heap->GetCPUDescriptorHandleForHeapStart ());
 
     for (guint plane = 0; plane < num_planes; plane++) {
+      D3D12_RECT rect = { };
+      gst_d3d12_memory_get_plane_rectangle (mem, plane, &rect);
+      rtv_rects.push_back (rect);
       rtv_handles.push_back (cpu_handle);
       cpu_handle.Offset (bg_render->rtv_inc_size);
     }
@@ -2263,8 +2267,10 @@ gst_d3d12_compositor_draw_background (GstD3D12Compositor * self)
     cl->DrawIndexedInstanced (6, 1, 0, 0, 0);
 
     /* clear U and V components if needed */
-    for (size_t i = 1; i < rtv_handles.size (); i++)
+    for (size_t i = 1; i < rtv_handles.size (); i++) {
-      cl->ClearRenderTargetView (rtv_handles[i], color->color[i], 0, nullptr);
+      cl->ClearRenderTargetView (rtv_handles[i], color->color[i], 1,
+          &rtv_rects[i]);
+    }
   } else {
     switch (priv->background) {
       case GST_D3D12_COMPOSITOR_BACKGROUND_BLACK:
@@ -2282,8 +2288,8 @@ gst_d3d12_compositor_draw_background (GstD3D12Compositor * self)
     }
 
     for (size_t i = 0; i < priv->rtv_handles.size (); i++) {
-      cl->ClearRenderTargetView (priv->rtv_handles[i], color->color[i],
+      cl->ClearRenderTargetView (rtv_handles[i], color->color[i], 1,
-          0, nullptr);
+          &rtv_rects[i]);
     }
   }
 

subprojects/gst-plugins-bad/sys/d3d12/gstd3d12converter.cpp

@@ -1866,6 +1866,7 @@ gst_d3d12_converter_execute (GstD3D12Converter * self,
 
   barriers.clear ();
   rtv_handles.clear ();
+  std::vector < D3D12_RECT > rtv_rects;
 
   auto upload_data = priv->upload_data;
 
@@ -1995,6 +1996,9 @@ gst_d3d12_converter_execute (GstD3D12Converter * self,
         (rtv_heap->GetCPUDescriptorHandleForHeapStart ());
 
     for (guint plane = 0; plane < num_planes; plane++) {
+      D3D12_RECT rect = { };
+      gst_d3d12_memory_get_plane_rectangle (mem, plane, &rect);
+      rtv_rects.push_back (rect);
       rtv_handles.push_back (cpu_handle);
       cpu_handle.Offset (priv->rtv_inc_size);
     }
@@ -2006,7 +2010,7 @@ gst_d3d12_converter_execute (GstD3D12Converter * self,
   if (priv->clear_background) {
     for (size_t i = 0; i < rtv_handles.size (); i++) {
       cl->ClearRenderTargetView (rtv_handles[i],
-          priv->clear_color[i], 0, nullptr);
+          priv->clear_color[i], 1, &rtv_rects[i]);
     }
   }
 

subprojects/gst-plugins-bad/sys/d3d12/gstd3d12device.cpp

@@ -1133,6 +1133,10 @@ gst_d3d12_device_clear_yuv_texture (GstD3D12Device * device, GstMemory * mem)
   if (!heap)
     return;
 
+  D3D12_RECT rect = { };
+  if (!gst_d3d12_memory_get_plane_rectangle (dmem, 1, &rect))
+    return;
+
   GstD3D12CommandAllocator *gst_ca = nullptr;
   gst_d3d12_command_allocator_pool_acquire (priv->direct_ca_pool, &gst_ca);
   if (!gst_ca)
@@ -1161,7 +1165,7 @@ gst_d3d12_device_clear_yuv_texture (GstD3D12Device * device, GstMemory * mem)
       priv->rtv_inc_size);
 
   const FLOAT clear_color[4] = { 0.5f, 0.5f, 0.5f, 1.0f };
-  cl->ClearRenderTargetView (rtv_handle, clear_color, 0, nullptr);
+  cl->ClearRenderTargetView (rtv_handle, clear_color, 1, &rect);
 
   auto hr = cl->Close ();
   if (!gst_d3d12_result (hr, device)) {

subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.cpp

@@ -224,6 +224,7 @@ struct _GstD3D12MemoryPrivate
   D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout[GST_VIDEO_MAX_PLANES];
   guint64 size;
   guint num_subresources;
+  D3D12_RECT subresource_rect[GST_VIDEO_MAX_PLANES];
   guint subresource_index[GST_VIDEO_MAX_PLANES];
   DXGI_FORMAT resource_formats[GST_VIDEO_MAX_PLANES];
   guint srv_inc_size;
@@ -501,6 +502,21 @@ gst_d3d12_memory_get_plane_count (GstD3D12Memory * mem)
   return mem->priv->num_subresources;
 }
 
+gboolean
+gst_d3d12_memory_get_plane_rectangle (GstD3D12Memory * mem, guint plane,
+    D3D12_RECT * rect)
+{
+  g_return_val_if_fail (gst_is_d3d12_memory (GST_MEMORY_CAST (mem)), FALSE);
+  g_return_val_if_fail (rect, FALSE);
+
+  if (plane >= mem->priv->num_subresources)
+    return FALSE;
+
+  *rect = mem->priv->subresource_rect[plane];
+
+  return TRUE;
+}
+
 gboolean
 gst_d3d12_memory_get_shader_resource_view_heap (GstD3D12Memory * mem,
     ID3D12DescriptorHeap ** heap)
@@ -718,7 +734,7 @@ gst_d3d12_allocator_alloc_wrapped (GstD3D12Allocator * allocator,
   mem->device = (GstD3D12Device *) gst_object_ref (device);
 
   mem->priv->size = 0;
-  for (guint i = 0; i < mem->priv->num_subresources; i++) {
+  for (guint i = 0; i < num_subresources; i++) {
     UINT64 size;
 
     /* One notable difference between D3D12/D3D11 is that, D3D12 introduced
@@ -746,6 +762,30 @@ gst_d3d12_allocator_alloc_wrapped (GstD3D12Allocator * allocator,
     priv->size += size;
   }
 
+  priv->subresource_rect[0].left = 0;
+  priv->subresource_rect[0].top = 0;
+  priv->subresource_rect[0].right = (LONG) desc.Width;
+  priv->subresource_rect[0].bottom = (LONG) desc.Height;
+
+  for (guint i = 1; i < num_subresources; i++) {
+    priv->subresource_rect[i].left = 0;
+    priv->subresource_rect[i].top = 0;
+    switch (desc.Format) {
+      case DXGI_FORMAT_NV12:
+      case DXGI_FORMAT_P010:
+      case DXGI_FORMAT_P016:
+        priv->subresource_rect[i].right = (LONG) desc.Width / 2;
+        priv->subresource_rect[i].bottom = (LONG) desc.Height / 2;
+        break;
+      default:
+        GST_WARNING_OBJECT (allocator, "Unexpected multi-plane format %d",
+            desc.Format);
+        priv->subresource_rect[i].right = (LONG) desc.Width / 2;
+        priv->subresource_rect[i].bottom = (LONG) desc.Height / 2;
+        break;
+    }
+  }
+
   gst_memory_init (GST_MEMORY_CAST (mem),
       (GstMemoryFlags) 0, GST_ALLOCATOR_CAST (allocator), nullptr,
       mem->priv->size, 0, 0, mem->priv->size);

subprojects/gst-plugins-bad/sys/d3d12/gstd3d12memory.h

@@ -139,6 +139,10 @@ gboolean          gst_d3d12_memory_get_subresource_index (GstD3D12Memory * mem,
 
 guint             gst_d3d12_memory_get_plane_count       (GstD3D12Memory * mem);
 
+gboolean          gst_d3d12_memory_get_plane_rectangle   (GstD3D12Memory * mem,
+                                                          guint plane,
+                                                          D3D12_RECT * rect);
+
 gboolean          gst_d3d12_memory_get_shader_resource_view_heap (GstD3D12Memory * mem,
                                                                   ID3D12DescriptorHeap ** heap);