va: videoformats: Fix RGB32 mapping between VA and GStreamer.

Different VA drives might have different definitions for RGB32 color
formats because different bit interpretation. Sadly the specification
doesn't clarify these interpretations. So VA users have to figure out
what's the correct mapping with it's rendering color format
definition.

This patch aims to fix the static map structure after the
VAImageFormats are queried. There is another static map with the
different interpretations of the RGB32 formats, and compare them with
the given VAImageFormat, then with the GStreamer color format, update
the mapping table.

Finally, some RGB32 color formats were added.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2129>

sys/va/gstvadisplay.c

@@ -418,6 +418,8 @@ gst_va_display_get_image_formats (GstVaDisplay * self)
   status = vaQueryImageFormats (dpy, va_formats, &num);
   gst_va_display_unlock (self);
 
+  gst_va_video_format_fix_map (va_formats, num);
+
   if (status != VA_STATUS_SUCCESS) {
     GST_ERROR ("vaQueryImageFormats: %s", vaErrorStr (status));
     goto bail;

sys/va/gstvavideoformat.c

@@ -24,11 +24,12 @@
 
 #include "gstvavideoformat.h"
 
+GST_DEBUG_CATEGORY_STATIC (gstva_debug);
+
 #define VA_NSB_FIRST 0          /* No Significant Bit  */
 
-/* XXX(victor): add RGB fourcc runtime checkups for screwed drivers */
 /* *INDENT-OFF* */
-static const struct FormatMap
+static struct FormatMap
 {
   GstVideoFormat format;
   guint va_rtformat;
@@ -46,15 +47,20 @@ static const struct FormatMap
   G (VUYA, ('A', 'Y', 'U', 'V'), YUV444, LSB, 32),
   F (RGBA, ('R', 'G', 'B', 'A'), RGB32, LSB, 32, 32, 0x000000ff,
       0x0000ff00, 0x00ff0000, 0xff000000),
-  /* F (????, RGBX), */
+  F (RGBx, ('R', 'G', 'B', 'X'), RGB32, LSB, 32, 24, 0x000000ff,
+      0x0000ff00, 0x00ff0000, 0x00000000),
   F (BGRA, ('B', 'G', 'R', 'A'), RGB32, LSB, 32, 32, 0x00ff0000,
       0x0000ff00, 0x000000ff, 0xff000000),
   F (ARGB, ('A', 'R', 'G', 'B'), RGB32, LSB, 32, 32, 0x0000ff00,
       0x00ff0000, 0xff000000, 0x000000ff),
-  /* F (????, XRGB), */
+  F (xRGB, ('X', 'R', 'G', 'B'), RGB32, LSB, 32, 24, 0x0000ff00,
+      0x00ff0000, 0xff000000, 0x00000000),
   F (ABGR, ('A', 'B', 'G', 'R'), RGB32, LSB, 32, 32, 0xff000000,
       0x00ff0000, 0x0000ff00, 0x000000ff),
-  /* F (????, XBGR), */
+  F (xBGR, ('X', 'B', 'G', 'R'), RGB32, LSB, 32, 24, 0xff000000,
+      0x00ff0000, 0x0000ff00, 0x00000000),
+  F (BGRx, ('B', 'G', 'R', 'X'), RGB32, LSB, 32, 24, 0x00ff0000,
+      0x0000ff00, 0x000000ff, 0x00000000),
   G (UYVY, ('U', 'Y', 'V', 'Y'), YUV422, NSB, 16),
   G (YUY2, ('Y', 'U', 'Y', '2'), YUV422, NSB, 16),
   G (AYUV, ('A', 'Y', 'U', 'V'), YUV444, LSB, 32),
@@ -97,6 +103,52 @@ static const struct FormatMap
 #undef F
 #undef G
 };
+
+static const struct RBG32FormatMap
+{
+  GstVideoFormat format;
+  VAImageFormat va_format[2];
+} rgb32_format_map[] = {
+#define  F(fourcc, order, bpp, depth, r, g, b, a)                       \
+  {  VA_FOURCC fourcc, G_PASTE (G_PASTE (VA_, order), _FIRST), bpp, depth, r, g, b, a }
+#define  A(fourcc, order, r, g, b, a) F (fourcc, order, 32, 32, r, g, b, a)
+#define  X(fourcc, order, r, g, b) F (fourcc, order, 32, 24, r, g, b, 0x0)
+  { GST_VIDEO_FORMAT_ARGB, {
+      A (('B', 'G', 'R', 'A'), LSB, 0x0000ff00, 0x00ff0000, 0xff000000, 0x000000ff),
+      A (('A', 'R', 'G', 'B'), MSB, 0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000),
+    } },
+  { GST_VIDEO_FORMAT_RGBA, {
+      A (('A', 'B', 'G', 'R'), LSB, 0x000000ff, 0x0000ff00, 0x00ff0000, 0xff000000),
+      A (('R', 'G', 'B', 'A'), MSB, 0xff000000, 0x00ff0000, 0x0000ff00, 0x000000ff),
+    } },
+  { GST_VIDEO_FORMAT_ABGR, {
+      A (('R', 'G', 'B', 'A'), LSB, 0xff000000, 0x00ff0000, 0x0000ff00, 0x000000ff),
+      A (('A', 'B', 'G', 'R'), MSB, 0x000000ff, 0x0000ff00, 0x00ff0000, 0xff000000),
+    } },
+  { GST_VIDEO_FORMAT_BGRA, {
+      A (('A', 'R', 'G', 'B'), LSB, 0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000),
+      A (('B', 'G', 'R', 'A'), MSB, 0x0000ff00, 0x00ff0000, 0xff000000, 0x000000ff),
+    } },
+  { GST_VIDEO_FORMAT_xRGB, {
+      X (('B', 'G', 'R', 'X'), LSB, 0x0000ff00, 0x00ff0000, 0xff000000),
+      X (('X', 'R', 'G', 'B'), MSB, 0x00ff0000, 0x0000ff00, 0x000000ff),
+    } },
+  { GST_VIDEO_FORMAT_RGBx, {
+      X (('X', 'B', 'G', 'R'), LSB, 0x000000ff, 0x0000ff00, 0x00ff0000),
+      X (('R', 'G', 'B', 'X'), MSB, 0xff000000, 0x00ff0000, 0x0000ff00),
+    } },
+  { GST_VIDEO_FORMAT_xBGR, {
+      X (('R', 'G', 'B', 'X'), LSB, 0xff000000, 0x00ff0000, 0x0000ff00),
+      X (('X', 'B', 'G', 'R'), MSB, 0x000000ff, 0x0000ff00, 0x00ff0000),
+    } },
+  { GST_VIDEO_FORMAT_BGRx, {
+      X (('X', 'R', 'G', 'B'), LSB, 0x00ff0000, 0x0000ff00, 0x000000ff),
+      X (('B', 'G', 'R', 'X'), MSB, 0x0000ff00, 0x00ff0000, 0xff000000),
+    } },
+#undef X
+#undef A
+#undef F
+};
 /* *INDENT-ON* */
 
 static const struct FormatMap *
@@ -112,7 +164,7 @@ get_format_map_from_va_fourcc (guint va_fourcc)
   return NULL;
 }
 
-static const struct FormatMap *
+static struct FormatMap *
 get_format_map_from_video_format (GstVideoFormat format)
 {
   int i;
@@ -231,3 +283,98 @@ gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
 
   return GST_VIDEO_FORMAT_UNKNOWN;
 }
+
+static GstVideoFormat
+find_gst_video_format_in_rgb32_map (VAImageFormat * image_format)
+{
+  guint i, j;
+
+  for (i = 0; i < G_N_ELEMENTS (rgb32_format_map); i++) {
+    for (j = 0; j < G_N_ELEMENTS (rgb32_format_map[i].va_format); j++) {
+      if (va_format_is_same (&rgb32_format_map[i].va_format[j], image_format))
+        return rgb32_format_map[i].format;
+    }
+  }
+
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}
+
+struct ImageFormatArray
+{
+  VAImageFormat *image_formats;
+  gint len;
+};
+
+static gpointer
+fix_map (gpointer data)
+{
+  struct ImageFormatArray *args = data;
+  GstVideoFormat format;
+  VAImageFormat *image_format;
+  struct FormatMap *map;
+  guint i;
+
+  GST_DEBUG_CATEGORY_GET (gstva_debug, "va");
+
+  for (i = 0; i < args->len; i++) {
+    image_format = &args->image_formats[i];
+    if (!va_format_is_rgb (image_format))
+      continue;
+    format = find_gst_video_format_in_rgb32_map (image_format);
+    if (format == GST_VIDEO_FORMAT_UNKNOWN)
+      continue;
+    map = get_format_map_from_video_format (format);
+    if (!map)
+      continue;
+    if (va_format_is_same (&map->va_format, image_format))
+      continue;
+
+    map->va_format = *image_format;
+
+    GST_CAT_INFO (gstva_debug, "GST_VIDEO_FORMAT_%s => { fourcc %"
+        GST_FOURCC_FORMAT ", %s, bpp %d, depth %d, R %#010x, G %#010x, "
+        "B %#010x, A %#010x }", gst_video_format_to_string (map->format),
+        GST_FOURCC_ARGS (map->va_format.fourcc),
+        (map->va_format.byte_order == 1) ? "LSB" : "MSB",
+        map->va_format.bits_per_pixel, map->va_format.depth,
+        map->va_format.red_mask, map->va_format.green_mask,
+        map->va_format.blue_mask, map->va_format.alpha_mask);
+  }
+
+  return NULL;
+}
+
+/* XXX: RGB32 LSB VAImageFormats don't map statically with GStreamer
+ * color formats. Each driver does what they want.
+ *
+ * For MSB, there is no ambiguity: same order in define, memory and
+ * CPU. For example,
+ *
+ *  RGBA is RGBA in memory and RGBA with channel mask R:0xFF0000
+ *  G:0x00FF0000 B:0x0000FF00 A:0x000000FF in CPU.
+ *
+ * For LSB, CPU's perspective and memory's perspective are
+ * different. For example,
+ *
+ *  From CPU's perspective, it's RGBA order in memory, but when it is
+ *  stored in memory, because CPU's little endianness, it will be
+ *  re-ordered, with mask R:0x000000FF G:0x0000FF00 B:0x00FF0000
+ *  A:0xFF000000.
+ *
+ *  In other words, from memory's perspective, RGBA LSB is equal as
+ *  ABGR MSB.
+ *
+ * These definitions are mixed used all over the media system and we
+ * need to correct the mapping form VA video format to GStreamer
+ * video format in both manners.
+ *
+ * https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/123
+ */
+void
+gst_va_video_format_fix_map (VAImageFormat * image_formats, gint num)
+{
+  static GOnce once = G_ONCE_INIT;
+  struct ImageFormatArray args = { image_formats, num };
+
+  g_once (&once, fix_map, &args);
+}

sys/va/gstvavideoformat.h

@@ -32,5 +32,7 @@ const VAImageFormat * gst_va_image_format_from_video_format (GstVideoFormat form
 GstVideoFormat        gst_va_video_format_from_va_image_format (const VAImageFormat * va_format);
 GstVideoFormat        gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
 								     GArray * surface_formats);
+void                  gst_va_video_format_fix_map         (VAImageFormat * image_formats,
+							   gint num);
 
 G_END_DECLS