d3d11decoder: Fix deadlock when DPB texture pool is full

Unlike other stateless decoder implementations (e.g., VA),
our DPB pool cannot be grown since we are using
texture array (pre-allocated, fixed-size d3d11 texture pool).
So, if there's no more available texture to use,
there's no way other than copying it to downstream's
d3d11 buffer pool. Otherwise deadlock will happen.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2003>

sys/d3d11/gstd3d11decoder.c

@@ -1577,15 +1577,49 @@ gst_d3d11_decoder_decide_allocation (GstVideoDecoder * decoder,
 }
 
 gboolean
-gst_d3d11_decoder_supports_direct_rendering (GstD3D11Decoder * decoder)
+gst_d3d11_decoder_can_direct_render (GstD3D11Decoder * decoder,
+    GstBuffer * view_buffer, GstMiniObject * picture)
 {
   GstD3D11DecoderPrivate *priv;
+  GstMemory *mem;
+  GstD3D11Allocator *alloc;
+  guint array_size, num_texture_in_use;
 
   g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE);
+  g_return_val_if_fail (GST_IS_BUFFER (view_buffer), FALSE);
+  g_return_val_if_fail (picture != NULL, FALSE);
 
   priv = decoder->priv;
 
-  return priv->can_direct_rendering;
+  if (!priv->can_direct_rendering)
+    return FALSE;
+
+  /* XXX: Not a thread-safe way, but should not be a problem.
+   * This object must be protected by videodecoder stream lock
+   * and codec base classes are working on upstream streaming thread
+   * (i.g., single threaded) */
+
+  /* Baseclass is not holding this picture. So we can wait for this memory
+   * to be consumed by downstream as it will be relased once it's processed
+   * by downstream */
+  if (GST_MINI_OBJECT_REFCOUNT (picture) == 1)
+    return TRUE;
+
+  mem = gst_buffer_peek_memory (view_buffer, 0);
+  alloc = GST_D3D11_ALLOCATOR_CAST (mem->allocator);
+
+  /* something went wrong */
+  if (!gst_d3d11_allocator_get_texture_array_size (alloc, &array_size,
+          &num_texture_in_use)) {
+    GST_ERROR_OBJECT (decoder, "Couldn't query size of texture array");
+    return FALSE;
+  }
+
+  /* DPB pool is full now */
+  if (num_texture_in_use >= array_size)
+    return FALSE;
+
+  return TRUE;
 }
 
 /* Keep sync with chromium and keep in sorted order.

sys/d3d11/gstd3d11decoder.h

@@ -128,7 +128,10 @@ gboolean          gst_d3d11_decoder_decide_allocation   (GstVideoDecoder * decod
                                                          GstD3D11Codec codec,
                                                          gboolean use_d3d11_pool);
 
-gboolean          gst_d3d11_decoder_supports_direct_rendering (GstD3D11Decoder * decoder);
+gboolean          gst_d3d11_decoder_can_direct_render   (GstD3D11Decoder * decoder,
+                                                         GstBuffer * view_buffer,
+                                                         GstMiniObject * picture);
+
 
 /* Utils for class registration */
 gboolean          gst_d3d11_decoder_util_is_legacy_device (GstD3D11Device * device);

sys/d3d11/gstd3d11h264dec.c

@@ -812,6 +812,7 @@ gst_d3d11_h264_dec_output_picture (GstH264Decoder * decoder,
   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstBuffer *output_buffer = NULL;
   GstBuffer *view_buffer;
+  gboolean direct_rendering = FALSE;
 
   GST_LOG_OBJECT (self,
       "Outputting picture %p (poc %d)", picture, picture->pic_order_cnt);
@@ -827,11 +828,15 @@ gst_d3d11_h264_dec_output_picture (GstH264Decoder * decoder,
    * expose our decoder view without copy. In case of reverse playback, however,
    * we cannot do that since baseclass will store the decoded buffer
    * up to gop size but our dpb pool cannot be increased */
-  if (self->use_d3d11_output &&
+  if (self->use_d3d11_output
-      gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) &&
+      && GST_VIDEO_DECODER (self)->input_segment.rate > 0
-      GST_VIDEO_DECODER (self)->input_segment.rate > 0) {
+      && gst_d3d11_decoder_can_direct_render (self->d3d11_decoder, view_buffer,
-    GstMemory *mem;
+          GST_MINI_OBJECT_CAST (picture))) {
+    direct_rendering = TRUE;
+  }
 
+  if (direct_rendering) {
+    GstMemory *mem;
     output_buffer = gst_buffer_ref (view_buffer);
     mem = gst_buffer_peek_memory (output_buffer, 0);
     GST_MINI_OBJECT_FLAG_SET (mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);

sys/d3d11/gstd3d11h265dec.c

@@ -773,6 +773,7 @@ gst_d3d11_h265_dec_output_picture (GstH265Decoder * decoder,
   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstBuffer *output_buffer = NULL;
   GstBuffer *view_buffer;
+  gboolean direct_rendering = FALSE;
 
   GST_LOG_OBJECT (self,
       "Outputting picture %p, poc %d", picture, picture->pic_order_cnt);
@@ -788,9 +789,14 @@ gst_d3d11_h265_dec_output_picture (GstH265Decoder * decoder,
    * expose our decoder view without copy. In case of reverse playback, however,
    * we cannot do that since baseclass will store the decoded buffer
    * up to gop size but our dpb pool cannot be increased */
-  if (self->use_d3d11_output &&
+  if (self->use_d3d11_output
-      gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) &&
+      && GST_VIDEO_DECODER (self)->input_segment.rate > 0
-      GST_VIDEO_DECODER (self)->input_segment.rate > 0) {
+      && gst_d3d11_decoder_can_direct_render (self->d3d11_decoder, view_buffer,
+          GST_MINI_OBJECT_CAST (picture))) {
+    direct_rendering = TRUE;
+  }
+
+  if (direct_rendering) {
     GstMemory *mem;
 
     output_buffer = gst_buffer_ref (view_buffer);

sys/d3d11/gstd3d11mpeg2dec.c

@@ -861,6 +861,7 @@ gst_d3d11_mpeg2_dec_output_picture (GstMpeg2Decoder * decoder,
   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstBuffer *output_buffer = NULL;
   GstBuffer *view_buffer;
+  gboolean direct_rendering = FALSE;
 
   GST_LOG_OBJECT (self, "Outputting picture %p", picture);
 
@@ -875,9 +876,14 @@ gst_d3d11_mpeg2_dec_output_picture (GstMpeg2Decoder * decoder,
    * expose our decoder view without copy. In case of reverse playback, however,
    * we cannot do that since baseclass will store the decoded buffer
    * up to gop size but our dpb pool cannot be increased */
-  if (self->use_d3d11_output &&
+  if (self->use_d3d11_output
-      gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) &&
+      && GST_VIDEO_DECODER (self)->input_segment.rate > 0
-      vdec->input_segment.rate > 0) {
+      && gst_d3d11_decoder_can_direct_render (self->d3d11_decoder, view_buffer,
+          GST_MINI_OBJECT_CAST (picture))) {
+    direct_rendering = TRUE;
+  }
+
+  if (direct_rendering) {
     GstMemory *mem;
 
     output_buffer = gst_buffer_ref (view_buffer);

sys/d3d11/gstd3d11vp8dec.c

@@ -364,6 +364,7 @@ gst_d3d11_vp8_dec_output_picture (GstVp8Decoder * decoder,
   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstBuffer *output_buffer = NULL;
   GstBuffer *view_buffer;
+  gboolean direct_rendering = FALSE;
 
   g_assert (picture->frame_hdr.show_frame);
 
@@ -380,9 +381,14 @@ gst_d3d11_vp8_dec_output_picture (GstVp8Decoder * decoder,
    * expose our decoder view without copy. In case of reverse playback, however,
    * we cannot do that since baseclass will store the decoded buffer
    * up to gop size but our dpb pool cannot be increased */
-  if (self->use_d3d11_output &&
+  if (self->use_d3d11_output
-      gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) &&
+      && GST_VIDEO_DECODER (self)->input_segment.rate > 0
-      vdec->input_segment.rate > 0) {
+      && gst_d3d11_decoder_can_direct_render (self->d3d11_decoder, view_buffer,
+          GST_MINI_OBJECT_CAST (picture))) {
+    direct_rendering = TRUE;
+  }
+
+  if (direct_rendering) {
     GstMemory *mem;
 
     output_buffer = gst_buffer_ref (view_buffer);

sys/d3d11/gstd3d11vp9dec.c

@@ -463,6 +463,7 @@ gst_d3d11_vp9_dec_output_picture (GstVp9Decoder * decoder,
   GstVideoDecoder *vdec = GST_VIDEO_DECODER (decoder);
   GstBuffer *output_buffer = NULL;
   GstBuffer *view_buffer;
+  gboolean direct_rendering = FALSE;
 
   g_assert (picture->frame_hdr.show_frame);
 
@@ -479,13 +480,18 @@ gst_d3d11_vp9_dec_output_picture (GstVp9Decoder * decoder,
    * expose our decoder view without copy. In case of reverse playback, however,
    * we cannot do that since baseclass will store the decoded buffer
    * up to gop size but our dpb pool cannot be increased */
-  if (self->use_d3d11_output &&
+  if (self->use_d3d11_output
-      gst_d3d11_decoder_supports_direct_rendering (self->d3d11_decoder) &&
+      && GST_VIDEO_DECODER (self)->input_segment.rate > 0
-      GST_VIDEO_DECODER (self)->input_segment.rate > 0 &&
+      && GST_VIDEO_INFO_WIDTH (&self->output_state->info) ==
-      GST_VIDEO_INFO_WIDTH (&self->output_state->info) ==
       picture->frame_hdr.width
       && GST_VIDEO_INFO_HEIGHT (&self->output_state->info) ==
-      picture->frame_hdr.height) {
+      picture->frame_hdr.height
+      && gst_d3d11_decoder_can_direct_render (self->d3d11_decoder, view_buffer,
+          GST_MINI_OBJECT_CAST (picture))) {
+    direct_rendering = TRUE;
+  }
+
+  if (direct_rendering) {
     GstMemory *mem;
 
     output_buffer = gst_buffer_ref (view_buffer);