From a4c976dd20a9634028b8e68747b16c6ae630db6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Cerveau?= <scerveau@igalia.com>
Date: Tue, 28 May 2024 09:55:05 +0200
Subject: [PATCH] vkdecoder: support layered and non dedicated DPB

As NVIDIA Amperium. In this case the each output buffer is also a DPB,
but using a different view layer.

Still pending a validation layer issue:

VUID-VkVideoBeginCodingInfoKHR-flags-07244

Co-authored-by: Victor Jaquez <vjaquez@igalia.com>
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6954>
---
 .../gst-plugins-bad/ext/vulkan/vkh264dec.c      |  2 +-
 .../gst-plugins-bad/ext/vulkan/vkh265dec.c      |  2 +-
 .../gst-libs/gst/vulkan/gstvkdecoder-private.c  | 17 ++++-------------
 3 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/subprojects/gst-plugins-bad/ext/vulkan/vkh264dec.c b/subprojects/gst-plugins-bad/ext/vulkan/vkh264dec.c
index 81fb8b363a..726671080b 100644
--- a/subprojects/gst-plugins-bad/ext/vulkan/vkh264dec.c
+++ b/subprojects/gst-plugins-bad/ext/vulkan/vkh264dec.c
@@ -1081,7 +1081,7 @@ _fill_ref_slot (GstVulkanH264Decoder * self, GstH264Picture * picture,
       .width = self->coded_width,
       .height = self->coded_height,
     },
-   .baseArrayLayer = self->decoder->layered_dpb ? pic->slot_idx : 0,
+   .baseArrayLayer = (self->decoder->layered_dpb && self->decoder->dedicated_dpb) ? pic->slot_idx : 0,
    .imageViewBinding = pic->base.img_view_ref->view,
   };
 
diff --git a/subprojects/gst-plugins-bad/ext/vulkan/vkh265dec.c b/subprojects/gst-plugins-bad/ext/vulkan/vkh265dec.c
index a5d5d1de27..c6e2f54b50 100644
--- a/subprojects/gst-plugins-bad/ext/vulkan/vkh265dec.c
+++ b/subprojects/gst-plugins-bad/ext/vulkan/vkh265dec.c
@@ -1375,7 +1375,7 @@ _fill_ref_slot (GstVulkanH265Decoder * self, GstH265Picture * picture,
     .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
     .codedOffset = { self->x, self->y },
     .codedExtent = { self->coded_width, self->coded_height },
-    .baseArrayLayer = self->decoder->layered_dpb ? pic->slot_idx : 0,
+    .baseArrayLayer = (self->decoder->layered_dpb && self->decoder->dedicated_dpb) ? pic->slot_idx : 0,
     .imageViewBinding = pic->base.img_view_ref->view,
   };
 
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkdecoder-private.c b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkdecoder-private.c
index 1a19b7c66f..d505a3f6f0 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkdecoder-private.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkdecoder-private.c
@@ -298,15 +298,6 @@ gst_vulkan_decoder_start (GstVulkanDecoder * self,
   self->layered_dpb = ((priv->caps.caps.flags &
           VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR) == 0);
 
-  if (self->layered_dpb && !self->dedicated_dpb) {
-    g_set_error (error, GST_VULKAN_ERROR, VK_ERROR_INCOMPATIBLE_DRIVER,
-        "Buggy driver: "
-        "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR set but "
-        "VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR is unset!");
-
-    goto failed;
-  }
-
   priv->caps.caps.pNext = NULL;
 
   /* Get output format */
@@ -697,7 +688,7 @@ gst_vulkan_decoder_decode (GstVulkanDecoder * self,
     return FALSE;
   }
 
-  new_layout = (self->layered_dpb || pic->dpb) ?
+  new_layout = ((self->layered_dpb && self->dedicated_dpb) || pic->dpb) ?
       VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR :
       VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
   gst_vulkan_operation_add_frame_barrier (priv->exec, pic->out,
@@ -713,7 +704,7 @@ gst_vulkan_decoder_decode (GstVulkanDecoder * self,
     }
   }
 
-  if (!self->layered_dpb) {
+  if (!self->layered_buffer) {
     /* All references (apart from the current) for non-layered refs */
 
     for (i = 0; i < pic->decode_info.referenceSlotCount; i++) {
@@ -1148,7 +1139,7 @@ gst_vulkan_decoder_picture_init (GstVulkanDecoder * self,
 
   priv = gst_vulkan_decoder_get_instance_private (self);
 
-  if (self->layered_dpb)
+  if (self->layered_dpb && self->dedicated_dpb)
     g_return_val_if_fail (GST_IS_BUFFER (self->layered_buffer), FALSE);
   else if (self->dedicated_dpb)
     g_return_val_if_fail (GST_IS_BUFFER_POOL (priv->dpb_pool), FALSE);
@@ -1161,7 +1152,7 @@ gst_vulkan_decoder_picture_init (GstVulkanDecoder * self,
   pic->dpb = NULL;
   pic->img_view_ref = NULL;
 
-  if (self->layered_dpb) {
+  if (self->layered_dpb && self->dedicated_dpb) {
     pic->img_view_ref =
         gst_vulkan_decoder_picture_create_view (self, self->layered_buffer,
         FALSE);