From 544829382c23779368ab8ead95fb2e4e56e453cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Mon, 28 Oct 2024 15:28:53 +0100
Subject: [PATCH] vaapipluginutil: use a static map format-chroma

Instead of registering the whole list of formats associated to a chroma, our
experience with GstVA tells that entry points only handles one color format per
supported chroma, and they are reflected in the static table.

This avoids exposing unsupported color formats for negotiation.

Fixes: #3914
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7708>
---
 .../gst-libs/gst/vaapi/video-format.c         | 33 --------------
 .../gst-libs/gst/vaapi/video-format.h         |  3 --
 .../gst/vaapi/gstvaapipluginutil.c            | 44 ++++++++++++++-----
 3 files changed, 33 insertions(+), 47 deletions(-)

diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.c b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.c
index 916f66a67a..7a1e827400 100644
--- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.c
+++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.c
@@ -504,39 +504,6 @@ gst_vaapi_video_format_get_best_native (GstVideoFormat format)
   return gst_vaapi_video_format_from_chroma (chroma_type);
 }
 
-/**
- * gst_vaapi_video_format_get_formats_by_chroma:
- * @chroma: a #GstVaapiChromaType
- *
- * Get all #GstVideoFormat which belong to #GstVaapiChromaType.
- *
- * Returns: an array of #GstVideoFormat.
- **/
-GArray *
-gst_vaapi_video_format_get_formats_by_chroma (guint chroma)
-{
-  const GstVideoFormatMap *entry;
-  GArray *formats;
-  guint i;
-
-  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
-  if (!formats)
-    return NULL;
-
-  for (i = 0; i < gst_vaapi_video_formats_map->len; i++) {
-    entry = &g_array_index (gst_vaapi_video_formats_map, GstVideoFormatMap, i);
-    if (entry->chroma_type == chroma)
-      g_array_append_val (formats, entry->format);
-  }
-
-  if (formats->len == 0) {
-    g_array_unref (formats);
-    return NULL;
-  }
-
-  return formats;
-}
-
 struct ImageFormatsData
 {
   VAImageFormat *formats;
diff --git a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.h b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.h
index 22e14fbfd8..0b654fb1ef 100644
--- a/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.h
+++ b/subprojects/gstreamer-vaapi/gst-libs/gst/vaapi/video-format.h
@@ -64,9 +64,6 @@ gst_vaapi_video_format_from_chroma (guint chroma);
 GstVideoFormat
 gst_vaapi_video_format_get_best_native (GstVideoFormat format);
 
-GArray *
-gst_vaapi_video_format_get_formats_by_chroma (guint chroma);
-
 gboolean
 gst_vaapi_video_format_create_map (VAImageFormat * formats, guint n);
 
diff --git a/subprojects/gstreamer-vaapi/gst/vaapi/gstvaapipluginutil.c b/subprojects/gstreamer-vaapi/gst/vaapi/gstvaapipluginutil.c
index b35af065df..b4445ecf7c 100644
--- a/subprojects/gstreamer-vaapi/gst/vaapi/gstvaapipluginutil.c
+++ b/subprojects/gstreamer-vaapi/gst/vaapi/gstvaapipluginutil.c
@@ -1131,6 +1131,26 @@ gst_vaapi_build_caps_from_formats (GArray * formats, gint min_width,
   return out_caps;
 }
 
+/* these are the commonly used formats in VA for chromas, by i965, iHD and
+ * gallium. We could query the VA config object, but for that there's GstVA
+ * plugin. */
+/* *INDENT-OFF* */
+const static struct {
+  GstVaapiChromaType chroma;
+  GstVideoFormat format;
+} ChromaFormatMap[] = {
+  { GST_VAAPI_CHROMA_TYPE_YUV420, GST_VIDEO_FORMAT_NV12 },
+  { GST_VAAPI_CHROMA_TYPE_YUV422, GST_VIDEO_FORMAT_YUY2 },
+  { GST_VAAPI_CHROMA_TYPE_YUV444, GST_VIDEO_FORMAT_VUYA },
+  { GST_VAAPI_CHROMA_TYPE_YUV420_10BPP, GST_VIDEO_FORMAT_P010_10LE },
+  { GST_VAAPI_CHROMA_TYPE_YUV422_10BPP, GST_VIDEO_FORMAT_Y210 },
+  { GST_VAAPI_CHROMA_TYPE_YUV444_10BPP, GST_VIDEO_FORMAT_Y410 },
+  { GST_VAAPI_CHROMA_TYPE_YUV420_12BPP, GST_VIDEO_FORMAT_P012_LE },
+  { GST_VAAPI_CHROMA_TYPE_YUV422_12BPP, GST_VIDEO_FORMAT_Y212_LE },
+  { GST_VAAPI_CHROMA_TYPE_YUV444_12BPP, GST_VIDEO_FORMAT_Y412_LE }
+};
+/* *INDENT-ON* */
+
 /**
  * gst_vaapi_build_template_raw_caps_by_codec:
  * @display: a #GstVaapiDisplay
@@ -1195,23 +1215,25 @@ gst_vaapi_build_template_raw_caps_by_codec (GstVaapiDisplay * display,
 
   for (gst_chroma = GST_VAAPI_CHROMA_TYPE_YUV420;
       gst_chroma <= GST_VAAPI_CHROMA_TYPE_YUV444_12BPP; gst_chroma++) {
-    GArray *fmts;
+    GstVideoFormat fmt;
     if (!(chroma & from_GstVaapiChromaType (gst_chroma)))
       continue;
 
-    fmts = gst_vaapi_video_format_get_formats_by_chroma (gst_chroma);
-    if (!fmts)
+    fmt = GST_VIDEO_FORMAT_UNKNOWN;
+    for (i = 0; i < G_N_ELEMENTS (ChromaFormatMap); i++) {
+      if (gst_chroma == ChromaFormatMap[i].chroma) {
+        fmt = ChromaFormatMap[i].format;
+        break;
+      }
+    }
+    if (fmt == GST_VIDEO_FORMAT_UNKNOWN)
       continue;
 
     /* One format can not belong to different chroma, no need to merge */
-    if (supported_fmts == NULL) {
-      supported_fmts = fmts;
-    } else {
-      for (i = 0; i < fmts->len; i++)
-        g_array_append_val (supported_fmts,
-            g_array_index (fmts, GstVideoFormat, i));
-      g_array_unref (fmts);
-    }
+    if (supported_fmts == NULL)
+      supported_fmts = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+
+    g_array_append_val (supported_fmts, fmt);
   }
 
   if (!supported_fmts)