qsvh265dec: Add support for GBR decoding

Use GBR equivalent output formats if RGB colorspace is detected

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5706>

subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator.cpp

@@ -149,6 +149,8 @@ gst_qsv_allocator_init (GstQsvAllocator * self)
 {
   GstQsvAllocatorPrivate *priv;
 
+  self->is_gbr = FALSE;
+
   priv = self->priv = (GstQsvAllocatorPrivate *)
       gst_qsv_allocator_get_instance_private (self);
 
@@ -199,7 +201,7 @@ gst_qsv_allocator_alloc_default (GstQsvAllocator * self, gboolean dummy_alloc,
     return MFX_ERR_UNSUPPORTED;
   }
 
-  format = gst_qsv_frame_info_format_to_gst (&request->Info);
+  format = gst_qsv_frame_info_format_to_gst (&request->Info, self->is_gbr);
   if (format == GST_VIDEO_FORMAT_UNKNOWN) {
     GST_ERROR_OBJECT (self, "Unknown MFX format fourcc %" GST_FOURCC_FORMAT,
         GST_FOURCC_ARGS (request->Info.FourCC));
@@ -406,15 +408,18 @@ gst_qsv_allocator_lock (mfxHDL pthis, mfxMemId mid, mfxFrameData * ptr)
       ptr->V16 = ptr->Y16 + 3;
       break;
     case GST_VIDEO_FORMAT_VUYA:
+    case GST_VIDEO_FORMAT_RBGA:
       ptr->V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame->frame, 0);
       ptr->U = ptr->V + 1;
       ptr->Y = ptr->V + 2;
       ptr->A = ptr->V + 3;
       break;
     case GST_VIDEO_FORMAT_Y410:
+    case GST_VIDEO_FORMAT_BGR10A2_LE:
       ptr->Y410 = (mfxY410 *) GST_VIDEO_FRAME_PLANE_DATA (&frame->frame, 0);
       break;
     case GST_VIDEO_FORMAT_Y412_LE:
+    case GST_VIDEO_FORMAT_BGRA64_LE:
       ptr->U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame->frame, 0);
       ptr->Y = ptr->Y + 2;
       ptr->V = ptr->Y + 4;

subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator.h

@@ -83,6 +83,7 @@ typedef enum
 struct _GstQsvAllocator
 {
   GstObject parent;
+  gboolean is_gbr;
 
   GstQsvAllocatorPrivate *priv;
 };

subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_d3d11.cpp

@@ -99,7 +99,7 @@ gst_qsv_d3d11_allocator_alloc (GstQsvAllocator * allocator,
     return MFX_ERR_UNSUPPORTED;
   }
 
-  format = gst_qsv_frame_info_format_to_gst (&request->Info);
+  format = gst_qsv_frame_info_format_to_gst (&request->Info, allocator->is_gbr);
 
   if (format == GST_VIDEO_FORMAT_UNKNOWN &&
       request->Info.FourCC != MFX_FOURCC_P8) {

subprojects/gst-plugins-bad/sys/qsv/gstqsvdecoder.cpp

@@ -81,6 +81,8 @@ struct _GstQsvDecoderPrivate
 
   mfxSession session;
   mfxVideoParam video_param;
+  mfxExtVideoSignalInfo signal_info;
+  mfxExtBuffer *video_param_ext[1];
 
   /* holding allocated GstQsvFrame, should be cleared via
    * mfxFrameAllocator::Free() */
@@ -888,6 +890,10 @@ gst_qsv_decoder_set_format (GstVideoDecoder * decoder,
 
   memset (&priv->video_param, 0, sizeof (mfxVideoParam));
   priv->video_param.mfx.CodecId = klass->codec_id;
+  priv->signal_info.Header.BufferId = MFX_EXTBUFF_VIDEO_SIGNAL_INFO;
+  priv->signal_info.Header.BufferSz = sizeof (mfxExtVideoSignalInfo);
+  priv->video_param_ext[0] = (mfxExtBuffer *) & priv->signal_info;
+  priv->video_param.ExtParam = priv->video_param_ext;
 
   /* If upstream is live, we will use single async-depth for low-latency
    * decoding */
@@ -1199,6 +1205,7 @@ gst_qsv_decoder_negotiate (GstVideoDecoder * decoder)
   mfxFrameInfo *frame_info = &param->mfx.FrameInfo;
   GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
   GstVideoInterlaceMode interlace_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE;
+  gboolean is_gbr = FALSE;
 
   width = coded_width = frame_info->Width;
   height = coded_height = frame_info->Height;
@@ -1208,6 +1215,16 @@ gst_qsv_decoder_negotiate (GstVideoDecoder * decoder)
     height = frame_info->CropH;
   }
 
+  if (klass->codec_id == MFX_CODEC_HEVC &&
+      priv->signal_info.ColourDescriptionPresent &&
+      gst_video_color_matrix_from_iso (priv->signal_info.MatrixCoefficients) ==
+      GST_VIDEO_COLOR_MATRIX_RGB) {
+    is_gbr = TRUE;
+  }
+
+  if (priv->allocator)
+    priv->allocator->is_gbr = is_gbr;
+
   if (klass->codec_id == MFX_CODEC_JPEG) {
     if (param->mfx.JPEGChromaFormat == MFX_CHROMAFORMAT_YUV422) {
       format = GST_VIDEO_FORMAT_YUY2;
@@ -1219,7 +1236,7 @@ gst_qsv_decoder_negotiate (GstVideoDecoder * decoder)
       frame_info->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
     }
   } else {
-    format = gst_qsv_frame_info_format_to_gst (frame_info);
+    format = gst_qsv_frame_info_format_to_gst (frame_info, is_gbr);
   }
 
   if (format == GST_VIDEO_FORMAT_UNKNOWN) {
@@ -1452,8 +1469,11 @@ gst_qsv_decoder_handle_frame (GstVideoDecoder * decoder,
 
 new_sequence:
   if (!priv->decoder) {
+    if (klass->codec_id == MFX_CODEC_HEVC)
+      priv->video_param.NumExtParam = 1;
     status = MFXVideoDECODE_DecodeHeader (priv->session,
         &bs, &priv->video_param);
+    priv->video_param.NumExtParam = 0;
 
     if (status != MFX_ERR_NONE) {
       if (status == MFX_ERR_MORE_DATA) {

subprojects/gst-plugins-bad/sys/qsv/gstqsvh265dec.cpp

@@ -558,6 +558,7 @@ gst_qsv_h265_dec_register (GstPlugin * plugin, guint rank, guint impl_index,
   if (MFXVideoDECODE_Query (session, &param, &param) == MFX_ERR_NONE) {
     supported_profiles.push_back ("main-444");
     supported_formats.push_back ("VUYA");
+    supported_formats.push_back ("RBGA");
   }
 
   /* main-444-10 */
@@ -566,6 +567,7 @@ gst_qsv_h265_dec_register (GstPlugin * plugin, guint rank, guint impl_index,
   if (MFXVideoDECODE_Query (session, &param, &param) == MFX_ERR_NONE) {
     supported_profiles.push_back ("main-444-10");
     supported_formats.push_back ("Y410");
+    supported_formats.push_back ("BGR10A2_LE");
   }
 
   /* main-444-12 */
@@ -574,6 +576,7 @@ gst_qsv_h265_dec_register (GstPlugin * plugin, guint rank, guint impl_index,
   if (MFXVideoDECODE_Query (session, &param, &param) == MFX_ERR_NONE) {
     supported_profiles.push_back ("main-444-12");
     supported_formats.push_back ("Y412_LE");
+    supported_formats.push_back ("BGRA64_LE");
   }
 
   /* To cover both landscape and portrait,

subprojects/gst-plugins-bad/sys/qsv/gstqsvutils.cpp

@@ -200,7 +200,7 @@ gst_qsv_status_to_string (mfxStatus status)
 }
 
 GstVideoFormat
-gst_qsv_frame_info_format_to_gst (const mfxFrameInfo * info)
+gst_qsv_frame_info_format_to_gst (const mfxFrameInfo * info, gboolean is_gbr)
 {
   GstVideoFormat format = GST_VIDEO_FORMAT_UNKNOWN;
 
@@ -228,12 +228,21 @@ gst_qsv_frame_info_format_to_gst (const mfxFrameInfo * info)
       break;
       break;
     case MFX_FOURCC_AYUV:
+      if (is_gbr)
+        format = GST_VIDEO_FORMAT_RBGA;
+      else
         format = GST_VIDEO_FORMAT_VUYA;
       break;
     case MFX_FOURCC_Y410:
+      if (is_gbr)
+        format = GST_VIDEO_FORMAT_BGR10A2_LE;
+      else
         format = GST_VIDEO_FORMAT_Y410;
       break;
     case MFX_FOURCC_Y416:
+      if (is_gbr)
+        format = GST_VIDEO_FORMAT_BGRA64_LE;
+      else
         format = GST_VIDEO_FORMAT_Y412_LE;
       break;
     case MFX_FOURCC_RGB4:
@@ -300,6 +309,7 @@ gst_qsv_frame_info_set_format (mfxFrameInfo * info, GstVideoFormat format)
       info->Shift = 1;
       break;
     case GST_VIDEO_FORMAT_VUYA:
+    case GST_VIDEO_FORMAT_RBGA:
       info->FourCC = MFX_FOURCC_AYUV;
       info->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
       info->BitDepthLuma = 8;
@@ -307,6 +317,7 @@ gst_qsv_frame_info_set_format (mfxFrameInfo * info, GstVideoFormat format)
       info->Shift = 0;
       break;
     case GST_VIDEO_FORMAT_Y410:
+    case GST_VIDEO_FORMAT_BGR10A2_LE:
       info->FourCC = MFX_FOURCC_Y410;
       info->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
       info->BitDepthLuma = 10;
@@ -314,6 +325,7 @@ gst_qsv_frame_info_set_format (mfxFrameInfo * info, GstVideoFormat format)
       info->Shift = 0;
       break;
     case GST_VIDEO_FORMAT_Y412_LE:
+    case GST_VIDEO_FORMAT_BGRA64_LE:
       info->FourCC = MFX_FOURCC_Y416;
       info->ChromaFormat = MFX_CHROMAFORMAT_YUV444;
       info->BitDepthLuma = 12;

subprojects/gst-plugins-bad/sys/qsv/gstqsvutils.h

@@ -76,7 +76,8 @@ static const GstQsvResolution gst_qsv_resolutions[] = {
   {7680, 4320}, {8192, 4320}, {15360, 8640}, {16384, 8640}
 };
 
-GstVideoFormat gst_qsv_frame_info_format_to_gst (const mfxFrameInfo * info);
+GstVideoFormat gst_qsv_frame_info_format_to_gst (const mfxFrameInfo * info,
+                                                 gboolean is_gbr);
 
 gboolean       gst_qsv_frame_info_set_format (mfxFrameInfo * info,
                                               GstVideoFormat format);