From a50b466fc72d69146c7009b9ed1d931a66e1014b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Wed, 12 Aug 2020 17:50:50 +0200
Subject: [PATCH] vaapidecode: expose raw src caps with same chroma

The try-and-error approach for getting the possible image formats from
a surface has brought several problems in different drivers, from
crashes to drop in performance.

Instead of that we change the algorithm to determine the possible
image formats based in the surface chroma: only those available image
formats with same chroma are exposed as possible raw caps.

Do this is important to avoid performance degrading in raw sinks
which doesn't handle NV12 but it does YV12 or I420.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/381>
---
 gst/vaapi/gstvaapidecode.c | 50 +++++++++++++++++++++++++++++++-------
 1 file changed, 41 insertions(+), 9 deletions(-)

diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 7563a2f4b8..b92edd1785 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -200,6 +200,7 @@ gst_vaapidecode_update_sink_caps (GstVaapiDecode * decode, GstCaps * caps)
 static gboolean
 gst_vaapidecode_ensure_allowed_srcpad_caps (GstVaapiDecode * decode)
 {
+  GstVaapiDisplay *const display = GST_VAAPI_PLUGIN_BASE_DISPLAY (decode);
   GstCaps *out_caps, *raw_caps, *va_caps, *dma_caps, *gltexup_caps, *base_caps;
   GArray *formats;
   gint min_width, min_height, max_width, max_height;
@@ -209,7 +210,7 @@ gst_vaapidecode_ensure_allowed_srcpad_caps (GstVaapiDecode * decode)
   if (decode->allowed_srcpad_caps)
     return TRUE;
 
-  if (!GST_VAAPI_PLUGIN_BASE_DISPLAY (decode))
+  if (!display)
     return FALSE;
 
   if (!decode->decoder)
@@ -228,14 +229,41 @@ gst_vaapidecode_ensure_allowed_srcpad_caps (GstVaapiDecode * decode)
   gst_vaapi_caps_set_width_and_height_range (base_caps, min_width, min_height,
       max_width, max_height);
 
-  raw_caps = gst_vaapi_plugin_base_get_allowed_srcpad_raw_caps
-      (GST_VAAPI_PLUGIN_BASE (decode),
-      GST_VIDEO_INFO_FORMAT (&decode->decoded_info));
-  if (!raw_caps)
-    goto bail;
-  raw_caps = gst_caps_copy (raw_caps);
-  gst_vaapi_caps_set_width_and_height_range (raw_caps, min_width, min_height,
-      max_width, max_height);
+  {
+    GArray *img_formats = gst_vaapi_display_get_image_formats (display);
+    GstVideoFormat decoded_format =
+        GST_VIDEO_INFO_FORMAT (&decode->decoded_info);
+
+    if (!img_formats)
+      img_formats = g_array_ref (formats);
+
+    if (decoded_format != GST_VIDEO_FORMAT_UNKNOWN) {
+      guint decoded_chroma =
+          gst_vaapi_video_format_get_chroma_type (decoded_format);
+      GArray *new_img_formats =
+          g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+      gint i;
+
+      for (i = 0; i < img_formats->len; i++) {
+        const GstVideoFormat fmt =
+            g_array_index (img_formats, GstVideoFormat, i);
+        if (gst_vaapi_video_format_get_chroma_type (fmt) == decoded_chroma)
+          g_array_append_val (new_img_formats, fmt);
+      }
+
+      if (new_img_formats->len == 0) {
+        g_clear_pointer (&new_img_formats, g_array_unref);
+      } else {
+        g_clear_pointer (&img_formats, g_array_unref);
+        img_formats = new_img_formats;
+      }
+    }
+
+    raw_caps = gst_vaapi_video_format_new_template_caps_from_list (img_formats);
+    gst_vaapi_caps_set_width_and_height_range (raw_caps, min_width, min_height,
+        max_width, max_height);
+    g_array_unref (img_formats);
+  }
 
   va_caps = gst_caps_copy (base_caps);
   gst_caps_set_features_simple (va_caps,
@@ -447,6 +475,10 @@ is_surface_resolution_changed (GstVaapiDecode * decode,
       surface_format = GST_VIDEO_FORMAT_NV12;
   }
 
+  /* reset allowed source caps since they are dependant of the decoded
+   * surface format */
+  gst_caps_replace (&decode->allowed_srcpad_caps, NULL);
+
   gst_video_info_set_format (vinfo, surface_format, surface_width,
       surface_height);