va: allocator: get a surface format from a image format

For the allocator to create surfaces with the correct chroma an
fourcc, it should use a surface format, not necessarily the negotiated
format.

Instead of the previous arbitrary extra formats list, the decoder
extracts the valid pixel formats from the given VA config, and pass
that list to the allocator which stores it (full transfer).

Then, when the allocator allocates a new surface, it looks for a
surface color format chroma-compatible with the negotiated image color
format.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1483>

sys/va/gstvaallocator.c

@@ -558,6 +558,7 @@ struct _GstVaAllocator
 
   GstVaDisplay *display;
   gboolean use_derived;
+  GArray *surface_formats;
 };
 
 typedef struct _GstVaMemory GstVaMemory;
@@ -588,6 +589,7 @@ gst_va_allocator_dispose (GObject * object)
   GstVaAllocator *self = GST_VA_ALLOCATOR (object);
 
   gst_clear_object (&self->display);
+  g_clear_pointer (&self->surface_formats, g_array_unref);
 
   G_OBJECT_CLASS (gst_va_allocator_parent_class)->dispose (object);
 }
@@ -864,15 +866,20 @@ gst_va_allocator_alloc (GstAllocator * allocator,
 
   self = GST_VA_ALLOCATOR (allocator);
 
-  format = GST_VIDEO_INFO_FORMAT (&params->info);
-  if (gst_va_video_format_is_extra (format))
-    format = GST_VIDEO_FORMAT_NV12;
+  format =
+      gst_va_video_surface_format_from_image_format (GST_VIDEO_INFO_FORMAT
+      (&params->info), self->surface_formats);
+  if (format == GST_VIDEO_FORMAT_UNKNOWN) {
+    GST_ERROR_OBJECT (allocator, "Unsupported format: %s",
+        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&params->info)));
+    return NULL;
+  }
 
   fourcc = gst_va_fourcc_from_video_format (format);
   rt_format = gst_va_chroma_from_video_format (format);
   if (fourcc == 0 || rt_format == 0) {
     GST_ERROR_OBJECT (allocator, "Unsupported format: %s",
-        gst_video_format_to_string (format));
+        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&params->info)));
     return NULL;
   }
 
@@ -902,7 +909,7 @@ gst_va_allocator_alloc (GstAllocator * allocator,
 }
 
 GstAllocator *
-gst_va_allocator_new (GstVaDisplay * display)
+gst_va_allocator_new (GstVaDisplay * display, GArray * surface_formats)
 {
   GstVaAllocator *self;
 
@@ -910,6 +917,7 @@ gst_va_allocator_new (GstVaDisplay * display)
 
   self = g_object_new (GST_TYPE_VA_ALLOCATOR, NULL);
   self->display = gst_object_ref (display);
+  self->surface_formats = surface_formats;
   gst_object_ref_sink (self);
 
   return GST_ALLOCATOR (self);

sys/va/gstvaallocator.h

@@ -50,7 +50,8 @@ G_DECLARE_FINAL_TYPE (GstVaAllocator, gst_va_allocator, GST, VA_ALLOCATOR, GstAl
 
 #define GST_ALLOCATOR_VASURFACE   "VAMemory"
 
-GstAllocator *        gst_va_allocator_new                (GstVaDisplay * display);
+GstAllocator *        gst_va_allocator_new                (GstVaDisplay * display,
+                                                           GArray * surface_formats);
 GstMemory *           gst_va_allocator_alloc              (GstAllocator * allocator,
                                                            GstVaAllocationParams * params);
 gboolean              gst_va_allocator_try                (GstAllocator * allocator,

sys/va/gstvacaps.c

@@ -140,25 +140,6 @@ _gst_caps_set_format_array (GstCaps * caps, GArray * formats)
   return TRUE;
 }
 
-/* extra formats to add to raw caps bacause *in theory* all drivers
- * could create images from surface's native format (NV12) to
- * these. */
-static const GstVideoFormat extra_formats[] = {
-  GST_VIDEO_FORMAT_I420, GST_VIDEO_FORMAT_YV12
-};
-
-gboolean
-gst_va_video_format_is_extra (guint format)
-{
-  guint i;
-
-  for (i = 0; i < G_N_ELEMENTS (extra_formats); i++) {
-    if (extra_formats[i] == format)
-      return TRUE;
-  }
-  return FALSE;
-}
-
 GstCaps *
 gst_va_create_raw_caps_from_config (GstVaDisplay * display, VAConfigID config)
 {

sys/va/gstvacaps.h

@@ -37,7 +37,5 @@ VASurfaceAttrib *     gst_va_get_surface_attribs          (GstVaDisplay * displa
 GstCaps *             gst_va_create_raw_caps_from_config  (GstVaDisplay * display,
                                                            VAConfigID config);
 
-gboolean              gst_va_video_format_is_extra        (guint format);
-
 G_END_DECLS
 

sys/va/gstvadecoder.c

@@ -26,6 +26,7 @@
 
 #include "gstvacaps.h"
 #include "gstvadisplay_wrapped.h"
+#include "gstvavideoformat.h"
 
 struct _GstVaDecoder
 {
@@ -447,6 +448,50 @@ gst_va_decoder_get_mem_types (GstVaDecoder * self)
   return ret;
 }
 
+GArray *
+gst_va_decoder_get_surface_formats (GstVaDecoder * self)
+{
+  GArray *formats;
+  GstVideoFormat format;
+  VASurfaceAttrib *attribs;
+  guint i, attrib_count;
+
+  g_return_val_if_fail (GST_IS_VA_DECODER (self), NULL);
+
+  if (!gst_va_decoder_is_open (self))
+    return NULL;
+
+  attribs = gst_va_get_surface_attribs (self->display, self->config,
+      &attrib_count);
+  if (!attribs)
+    return NULL;
+
+  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+
+  for (i = 0; i < attrib_count; i++) {
+    if (attribs[i].value.type != VAGenericValueTypeInteger)
+      continue;
+    switch (attribs[i].type) {
+      case VASurfaceAttribPixelFormat:
+        format = gst_va_video_format_from_va_fourcc (attribs[i].value.value.i);
+        if (format != GST_VIDEO_FORMAT_UNKNOWN)
+          g_array_append_val (formats, format);
+        break;
+      default:
+        break;
+    }
+  }
+
+  g_free (attribs);
+
+  if (formats->len == 0) {
+    g_array_unref (formats);
+    return NULL;
+  }
+
+  return formats;
+}
+
 gboolean
 gst_va_decoder_add_param_buffer (GstVaDecoder * self, GstVaDecodePicture * pic,
     gint type, gpointer data, gsize size)

sys/va/gstvadecoder.h

@@ -51,6 +51,7 @@ GstCaps *             gst_va_decoder_get_sinkpad_caps     (GstVaDecoder * self);
 gboolean              gst_va_decoder_has_profile          (GstVaDecoder * self,
 							   VAProfile profile);
 gint                  gst_va_decoder_get_mem_types        (GstVaDecoder * self);
+GArray *              gst_va_decoder_get_surface_formats  (GstVaDecoder * self);
 
 gboolean              gst_va_decoder_add_param_buffer     (GstVaDecoder * self,
 							   GstVaDecodePicture * pic,

sys/va/gstvah264dec.c

@@ -1104,8 +1104,11 @@ _create_allocator (GstVaH264Dec * self, GstCaps * caps, guint * size)
 
   if (_caps_is_dmabuf (self, caps))
     allocator = gst_va_dmabuf_allocator_new (display);
-  else
-    allocator = gst_va_allocator_new (display);
+  else {
+    GArray *surface_formats =
+        gst_va_decoder_get_surface_formats (self->decoder);
+    allocator = gst_va_allocator_new (display, surface_formats);
+  }
 
   gst_object_unref (display);
 

sys/va/gstvavideoformat.c

@@ -204,3 +204,35 @@ gst_va_video_format_from_va_image_format (const VAImageFormat * va_format)
 
   return map ? map->format : GST_VIDEO_FORMAT_UNKNOWN;
 }
+
+GstVideoFormat
+gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
+    GArray * surface_formats)
+{
+  GstVideoFormat surface_format;
+  guint i, image_chroma, surface_chroma;
+
+  if (image_format == GST_VIDEO_FORMAT_UNKNOWN)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  if (!surface_formats || surface_formats->len == 0)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  image_chroma = gst_va_chroma_from_video_format (image_format);
+  if (image_chroma == 0)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  for (i = 0; i < surface_formats->len; i++) {
+    surface_format = g_array_index (surface_formats, GstVideoFormat, i);
+
+    if (surface_format == image_format)
+      return surface_format;
+
+    surface_chroma = gst_va_chroma_from_video_format (surface_format);
+
+    if (surface_chroma == image_chroma)
+      return surface_format;
+  }
+
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}

sys/va/gstvavideoformat.h

@@ -30,5 +30,7 @@ guint                 gst_va_fourcc_from_video_format     (GstVideoFormat format
 guint                 gst_va_chroma_from_video_format     (GstVideoFormat format);
 const VAImageFormat * gst_va_image_format_from_video_format (GstVideoFormat format);
 GstVideoFormat        gst_va_video_format_from_va_image_format (const VAImageFormat * va_format);
+GstVideoFormat        gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
+								     GArray * surface_formats);
 
 G_END_DECLS