From 984956a6fecca3894a3da3a45181f3a75e84ee1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Tue, 27 Aug 2024 18:32:42 +0200
Subject: [PATCH] vkencoder-private: fix parameters overrides

First remove validations since they will fail if there isn't a write operation.
It's valid to pass data without write operations.

Finally, it should check for hasOverride in feedback info. Nonetheless, there's
a NVIDIA bug returning always FALSE for hasOverride, that's why we currently
force it to TRUE.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7993>
---
 .../gst-libs/gst/vulkan/gstvkencoder-private.c      | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
index f5cbbb42e5..f3a9892340 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
@@ -844,6 +844,7 @@ gst_vulkan_encoder_video_session_parameters_overrides (GstVulkanEncoder * self,
   GstVulkanEncoderPrivate *priv;
   gsize size;
   gpointer param_data;
+  gboolean write;
 
   g_return_val_if_fail (GST_IS_VULKAN_ENCODER (self), FALSE);
   g_return_val_if_fail (params != NULL && feedback != NULL, FALSE);
@@ -854,29 +855,27 @@ gst_vulkan_encoder_video_session_parameters_overrides (GstVulkanEncoder * self,
 
   switch (self->codec) {
     case VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR:
-      g_return_val_if_fail ((params->h264.writeStdPPS
-              || params->h264.writeStdSPS) && data, FALSE);
       if (params->h264.sType !=
           VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_SESSION_PARAMETERS_GET_INFO_KHR) {
         gst_vulkan_error_to_g_error (GST_VULKAN_ERROR, error,
             "Invalid parameter for H.264");
         return FALSE;
       }
+      write = params->h264.writeStdPPS || params->h264.writeStdSPS;
       if (feedback) {
         feedback->h264.sType =
             VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_SESSION_PARAMETERS_FEEDBACK_INFO_KHR;
       }
       break;
     case VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR:
-      g_return_val_if_fail ((params->h265.writeStdPPS
-              || params->h265.writeStdSPS || params->h265.writeStdVPS)
-          && data, FALSE);
       if (params->h265.sType !=
           VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_SESSION_PARAMETERS_GET_INFO_KHR) {
         gst_vulkan_error_to_g_error (GST_VULKAN_ERROR, error,
             "Invalid parameter for H.265");
         return FALSE;
       }
+      write = params->h265.writeStdPPS || params->h265.writeStdSPS
+          || params->h265.writeStdVPS;
       if (feedback) {
         feedback->h265.sType =
             VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_SESSION_PARAMETERS_FEEDBACK_INFO_KHR;
@@ -905,7 +904,9 @@ gst_vulkan_encoder_video_session_parameters_overrides (GstVulkanEncoder * self,
           "vGetEncodedVideoSessionParametersKHR") != VK_SUCCESS)
     return FALSE;
 
-  if (!data)
+  /* FIXME: forcing because a bug in NVIDIA driver */
+  feedback_info.hasOverrides = 1;
+  if (!feedback_info.hasOverrides || !data || !write)
     return TRUE;
 
   GST_DEBUG_OBJECT (self, "allocating for bitstream parameters %"