From 87137e335311ac32a68a73e2938adc1b67732713 Mon Sep 17 00:00:00 2001
From: Matthew Waters <matthew@centricular.com>
Date: Tue, 10 Nov 2015 23:41:24 +1100
Subject: [PATCH] glcolorconvert: try to keep the same texture-target if
 possible

Fixes issues attempting to passthrough external-oes textures
---
 gst-libs/gst/gl/gstglcolorconvert.c | 39 ++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index c40c147a66..d14fed922b 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -790,6 +790,7 @@ gst_gl_color_convert_fixate_caps (GstGLContext * convert,
   GValue item = G_VALUE_INIT;
   const GValue *targets, *other_targets;
   guint targets_mask = 0, other_targets_mask = 0, result_mask;
+  GstVideoInfo info, other_info;
   GstStructure *s, *s_other;
 
   other = gst_caps_make_writable (other);
@@ -802,21 +803,36 @@ gst_gl_color_convert_fixate_caps (GstGLContext * convert,
   targets_mask = _get_target_bitmask_from_g_value (targets);
   other_targets_mask = _get_target_bitmask_from_g_value (other_targets);
 
+  /* XXX: attempt to fixate the format/colorimetry/etc */
+  other = gst_caps_fixate (other);
+
   result_mask = targets_mask & other_targets_mask;
   if (result_mask == 0) {
     /* nothing we can do here */
-    return gst_caps_fixate (other);
+    return other;
   }
 
-  if (direction == GST_PAD_SINK) {
-    result_mask &=
-        (1 << GST_GL_TEXTURE_TARGET_2D | 1 << GST_GL_TEXTURE_TARGET_RECTANGLE);
-  } else {
-    /* if the src caps has 2D support we can 'convert' to anything */
-    if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D))
-      result_mask = -1;
-    else
-      result_mask = other_targets_mask;
+  caps = gst_caps_copy (caps);
+  caps = gst_caps_fixate (caps);
+
+  gst_video_info_from_caps (&info, caps);
+  gst_video_info_from_caps (&other_info, other);
+
+  if (!_gst_gl_color_convert_can_passthrough_info (&info, &other_info)) {
+    if (direction == GST_PAD_SINK) {
+      /* this effectively limits us to 2D | RECTANGLE for case where we
+       * have to convert */
+      result_mask &=
+          (1 << GST_GL_TEXTURE_TARGET_2D | 1 <<
+          GST_GL_TEXTURE_TARGET_RECTANGLE);
+    } else {
+      /* if the src caps has 2D support we can 'convert' to anything */
+      if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D | 1 <<
+              GST_GL_TEXTURE_TARGET_RECTANGLE))
+        result_mask = -1;
+      else
+        result_mask = other_targets_mask;
+    }
   }
 
   g_value_init (&item, G_TYPE_STRING);
@@ -831,8 +847,7 @@ gst_gl_color_convert_fixate_caps (GstGLContext * convert,
   gst_structure_set_value (s_other, "texture-target", &item);
 
   g_value_unset (&item);
-
-  other = gst_caps_fixate (other);
+  gst_caps_unref (caps);
 
   return other;
 }