plugins: check type of display obtained from neighbours.

Fix display creation code to check that any display obtained from a
neighbour actually has the type we expect. Note: if display type is
set to "any", we can then accept any VA display type.

gst-libs/gst/vaapi/gstvaapidisplay.c

@@ -176,6 +176,34 @@ gst_vaapi_display_type_get_type(void)
     return g_type;
 }
 
+/**
+ * gst_vaapi_display_type_is_compatible:
+ * @type1: the #GstVaapiDisplayType to test
+ * @type2: the reference #GstVaapiDisplayType
+ *
+ * Compares whether #GstVaapiDisplay @type1 is compatible with @type2.
+ * That is, if @type2 is in "any" category, or derived from @type1.
+ *
+ * Returns: %TRUE if @type1 is compatible with @type2, %FALSE otherwise.
+ */
+gboolean
+gst_vaapi_display_type_is_compatible(GstVaapiDisplayType type1,
+    GstVaapiDisplayType type2)
+{
+    if (type1 == type2)
+        return TRUE;
+
+    switch (type1) {
+    case GST_VAAPI_DISPLAY_TYPE_GLX:
+        if (type2 == GST_VAAPI_DISPLAY_TYPE_X11)
+            return TRUE;
+        break;
+    default:
+        break;
+    }
+    return type2 == GST_VAAPI_DISPLAY_TYPE_ANY;
+}
+
 /* Append GstVideoFormat to formats array */
 static inline void
 append_format(GArray *formats, GstVideoFormat format, guint flags)

gst-libs/gst/vaapi/gstvaapidisplay.h

@@ -61,6 +61,10 @@ typedef enum {
 GType
 gst_vaapi_display_type_get_type(void) G_GNUC_CONST;
 
+gboolean
+gst_vaapi_display_type_is_compatible(GstVaapiDisplayType type1,
+    GstVaapiDisplayType type2);
+
 /**
  * GstVaapiDisplayInfo:
  *

gst/vaapi/gstvaapipluginbase.c

@@ -101,14 +101,6 @@ gst_vaapi_plugin_base_set_display_type (GstVaapiPluginBase * plugin,
   plugin->display_type_req = display_type;
 }
 
-/* Checks wether display type 1 is compatible with display type 2 */
-static gboolean
-display_type_is_compatible (GstVaapiDisplayType type1,
-    GstVaapiDisplayType type2)
-{
-  return (type1 == type2 || type2 == GST_VAAPI_DISPLAY_TYPE_ANY);
-}
-
 /**
  * gst_vaapi_plugin_base_ensure_display:
  * @plugin: a #GstVaapiPluginBase
@@ -122,7 +114,8 @@ display_type_is_compatible (GstVaapiDisplayType type1,
 gboolean
 gst_vaapi_plugin_base_ensure_display (GstVaapiPluginBase * plugin)
 {
-  if (plugin->display && display_type_is_compatible (plugin->display_type,
+  if (plugin->display
+      && gst_vaapi_display_type_is_compatible (plugin->display_type,
           plugin->display_type_req))
     return TRUE;
   gst_vaapi_display_replace (&plugin->display, NULL);

gst/vaapi/gstvaapipluginutil.c

@@ -121,7 +121,8 @@ gst_vaapi_ensure_display(gpointer element, GstVaapiDisplayType type)
     gst_vaapi_video_context_prepare(context, display_types);
 
     /* Neighbour found and it updated the display */
-    if (plugin->display)
+    if (plugin->display && gst_vaapi_display_type_is_compatible(
+            plugin->display_type, type))
         return TRUE;
 
     /* If no neighboor, or application not interested, use system default */