From 3e992d8adbb31e07d90f91264cd2605720f9098e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?=
 <vjaquez@igalia.com>
Date: Wed, 27 Feb 2019 13:02:10 +0100
Subject: [PATCH] plugin: if any caps in downstream, negotiate raw video

When downstream has any caps, vaapi should not shovel vaapi featured
buffers, but rather plain raw video, assuming always the worst case
scenario (downstream cannot handle featured video memory but raw
system memory buffers).

This patch query the peer caps without any filter, to know if
donwstream just ask for any caps, if so jump to the color space
checking, otherwise do the caps intersection and continue with the
feature selection algorithm.

Fixes: #139
---
 gst/vaapi/gstvaapipluginutil.c | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/gst/vaapi/gstvaapipluginutil.c b/gst/vaapi/gstvaapipluginutil.c
index 23b3b6a1ee..fcbbc324c3 100644
--- a/gst/vaapi/gstvaapipluginutil.c
+++ b/gst/vaapi/gstvaapipluginutil.c
@@ -638,21 +638,31 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstCaps * allowed_caps,
 {
   GstVaapiCapsFeature feature = GST_VAAPI_CAPS_FEATURE_NOT_NEGOTIATED;
   guint i, j, num_structures;
-  GstCaps *out_caps, *caps = NULL;
+  GstCaps *peer_caps, *out_caps = NULL, *caps = NULL;
   static const guint feature_list[] = { GST_VAAPI_CAPS_FEATURE_VAAPI_SURFACE,
     GST_VAAPI_CAPS_FEATURE_DMABUF,
     GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META,
     GST_VAAPI_CAPS_FEATURE_SYSTEM_MEMORY,
   };
 
-  out_caps = gst_pad_peer_query_caps (pad, allowed_caps);
-  if (!out_caps)
+  /* query with no filter */
+  peer_caps = gst_pad_peer_query_caps (pad, NULL);
+  if (!peer_caps)
+    goto cleanup;
+  if (gst_caps_is_empty (peer_caps))
     goto cleanup;
 
-  if (gst_caps_is_any (out_caps) || gst_caps_is_empty (out_caps))
-    goto cleanup;
+  /* filter against our allowed caps */
+  out_caps = gst_caps_intersect_full (allowed_caps, peer_caps,
+      GST_CAPS_INTERSECT_FIRST);
 
+  /* default feature */
   feature = GST_VAAPI_CAPS_FEATURE_SYSTEM_MEMORY;
+
+  /* if downstream requests caps ANY, system memory is preferred */
+  if (gst_caps_is_any (peer_caps))
+    goto find_format;
+
   num_structures = gst_caps_get_size (out_caps);
   for (i = 0; i < num_structures; i++) {
     GstCapsFeatures *const features = gst_caps_get_features (out_caps, i);
@@ -685,13 +695,14 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstCaps * allowed_caps,
   if (!caps)
     goto cleanup;
 
+find_format:
   if (out_format_ptr) {
     GstVideoFormat out_format;
     GstStructure *structure;
     const GValue *format_list;
 
     /* if the best feature is SystemMemory, we should use the first
-     * caps in the peer caps set, which is the preferred by
+     * caps in the filtered peer caps set, which is the preferred by
      * downstream. */
     if (feature == GST_VAAPI_CAPS_FEATURE_SYSTEM_MEMORY)
       gst_caps_replace (&caps, out_caps);
@@ -713,6 +724,7 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstCaps * allowed_caps,
 cleanup:
   gst_caps_replace (&caps, NULL);
   gst_caps_replace (&out_caps, NULL);
+  gst_caps_replace (&peer_caps, NULL);
   return feature;
 }