videoaggregator: operate on caps rather than video info

Otherwise the CapsFeatures will be lost along with the possibility
of multiple output types and formats.

https://bugzilla.gnome.org/show_bug.cgi?id=738129

ext/gl/gstglmixer.c

@@ -36,6 +36,10 @@
 
 #include "gstglmixer.h"
 
+#if GST_GL_HAVE_PLATFORM_EGL
+#include <gst-libs/gst/gl/egl/gsteglimagememory.h>
+#endif
+
 #define gst_gl_mixer_parent_class parent_class
 G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_VIDEO_AGGREGATOR);
 static gboolean gst_gl_mixer_do_bufferpool (GstGLMixer * mix,
@@ -559,6 +563,98 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query)
   return TRUE;
 }
 
+static GstCaps *
+gst_gl_mixer_set_caps_features (const GstCaps * caps,
+    const gchar * feature_name)
+{
+  GstCaps *tmp = gst_caps_copy (caps);
+  guint n = gst_caps_get_size (tmp);
+  guint i = 0;
+
+  for (i = 0; i < n; i++) {
+    GstCapsFeatures *features = gst_caps_get_features (tmp, i);
+    if (features) {
+      guint n_f = gst_caps_features_get_size (features);
+      guint j = 0;
+      for (j = 0; j < n_f; j++) {
+        gst_caps_features_remove_id (features,
+            gst_caps_features_get_nth_id (features, j));
+      }
+    }
+
+    gst_caps_features_add (features, feature_name);
+    gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL);
+  }
+
+  return tmp;
+}
+
+/* copies the given caps */
+static GstCaps *
+gst_gl_mixer_caps_remove_format_info (GstCaps * caps)
+{
+  GstStructure *st;
+  GstCapsFeatures *f;
+  gint i, n;
+  GstCaps *res;
+
+  res = gst_caps_new_empty ();
+
+  n = gst_caps_get_size (caps);
+  for (i = 0; i < n; i++) {
+    st = gst_caps_get_structure (caps, i);
+    f = gst_caps_get_features (caps, i);
+
+    /* If this is already expressed by the existing caps
+     * skip this structure */
+    if (i > 0 && gst_caps_is_subset_structure_full (res, st, f))
+      continue;
+
+    st = gst_structure_copy (st);
+    /* Only remove format info for the cases when we can actually convert */
+    if (!gst_caps_features_is_any (f)
+        && gst_caps_features_is_equal (f,
+            GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY))
+      gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site",
+          "width", "height", NULL);
+
+    gst_caps_append_structure_full (res, st, gst_caps_features_copy (f));
+  }
+
+  return res;
+}
+
+GstCaps *
+gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps)
+{
+  GstCaps *result = NULL;
+  GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps,
+      GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+#if GST_GL_HAVE_PLATFORM_EGL
+  GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps,
+      GST_CAPS_FEATURE_MEMORY_EGL_IMAGE);
+#endif
+  GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps,
+      GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META);
+  GstCaps *raw_caps =
+      gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS));
+
+  result = gst_caps_new_empty ();
+
+  result = gst_caps_merge (result, glcaps);
+#if GST_GL_HAVE_PLATFORM_EGL
+  result = gst_caps_merge (result, eglcaps);
+#endif
+  result = gst_caps_merge (result, uploadcaps);
+  result = gst_caps_merge (result, raw_caps);
+
+  result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps));
+
+  GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result);
+
+  return result;
+}
+
 static gboolean
 gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query)
 {

ext/gl/gstglmixer.h

@@ -92,6 +92,7 @@ struct _GstGLMixerFrameData
 GType gst_gl_mixer_get_type(void);
 
 gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf);
+GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps);
 
 G_END_DECLS
 #endif /* __GST_GL_MIXER_H__ */

ext/gl/gstglvideomixer.c

@@ -65,7 +65,7 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id,
 static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec);
 
-static gboolean _update_info (GstVideoAggregator * vagg, GstVideoInfo * info);
+static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps);
 static void gst_gl_video_mixer_reset (GstGLMixer * mixer);
 static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer,
     GstCaps * outcaps);
@@ -335,7 +335,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass)
   GST_GL_MIXER_CLASS (klass)->process_textures =
       gst_gl_video_mixer_process_textures;
 
-  vagg_class->update_info = _update_info;
+  vagg_class->update_caps = _update_caps;
 
   agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD;
 }
@@ -380,12 +380,17 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id,
   }
 }
 
-static gboolean
+static GstCaps *
-_update_info (GstVideoAggregator * vagg, GstVideoInfo * info)
+_update_caps (GstVideoAggregator * vagg, GstCaps * caps)
 {
   GList *l;
   gint best_width = -1, best_height = -1;
-  gboolean ret = FALSE;
+  GstVideoInfo info;
+  GstCaps *ret = NULL;
+  int i;
+
+  caps = gst_caps_make_writable (caps);
+  gst_video_info_from_caps (&info, caps);
 
   GST_OBJECT_LOCK (vagg);
   for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
@@ -417,10 +422,13 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info)
   }
   GST_OBJECT_UNLOCK (vagg);
 
-  if (best_width > 0 && best_height > 0) {
+  ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps);
-    info->width = best_width;
+
-    info->height = best_height;
+  for (i = 0; i < gst_caps_get_size (ret); i++) {
-    ret = TRUE;
+    GstStructure *s = gst_caps_get_structure (ret, i);
+
+    gst_structure_set (s, "width", G_TYPE_INT, best_width, "height", G_TYPE_INT,
+        best_height, NULL);
   }
 
   return ret;

gst-libs/gst/video/gstvideoaggregator.c

@@ -559,7 +559,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg)
   }
 
   if (best_width > 0 && best_height > 0 && best_fps > 0) {
-    GstCaps *caps, *peercaps;
+    GstCaps *caps, *peercaps, *info_caps;
     GstStructure *s;
     GstVideoInfo info;
 
@@ -578,25 +578,40 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg)
     info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info);
     info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info);
 
-    if (vagg_klass->update_info) {
+    info_caps = gst_video_info_to_caps (&info);
-      if (!vagg_klass->update_info (vagg, &info)) {
+
+    if (vagg_klass->update_caps) {
+      if (!(caps = vagg_klass->update_caps (vagg, info_caps))) {
+        gst_caps_unref (info_caps);
         ret = FALSE;
         goto done;
       }
+      gst_caps_unref (info_caps);
+    } else {
+      caps = info_caps;
     }
 
-    caps = gst_video_info_to_caps (&info);
-
     peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL);
     if (peercaps) {
       GstCaps *tmp;
+      int i;
 
       s = gst_caps_get_structure (caps, 0);
-      gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height",
+      gst_structure_get (s, "width", G_TYPE_INT, &best_width, "height",
-          GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE,
+          G_TYPE_INT, &best_height, NULL);
-          0, 1, G_MAXINT, 1, NULL);
+
+      for (i = 0; i < gst_caps_get_size (caps); i++) {
+        s = gst_caps_get_structure (caps, i);
+        gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
+            "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate",
+            GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
+      }
 
       tmp = gst_caps_intersect (caps, peercaps);
+      GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT
+          " with peer caps %" GST_PTR_FORMAT " result %" GST_PTR_FORMAT, caps,
+          peercaps, tmp);
+
       gst_caps_unref (caps);
       gst_caps_unref (peercaps);
       caps = tmp;
@@ -608,8 +623,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg)
 
       caps = gst_caps_truncate (caps);
       s = gst_caps_get_structure (caps, 0);
-      gst_structure_fixate_field_nearest_int (s, "width", info.width);
+      gst_structure_fixate_field_nearest_int (s, "width", best_width);
-      gst_structure_fixate_field_nearest_int (s, "height", info.height);
+      gst_structure_fixate_field_nearest_int (s, "height", best_height);
       gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
           best_fps_d);
 
@@ -618,9 +633,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg)
       gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d);
     }
 
-    gst_caps_unref (caps);
-    caps = gst_video_info_to_caps (&info);
-
     if (gst_videoaggregator_src_setcaps (vagg, caps)) {
       if (vagg_klass->negotiated_caps)
         ret =

gst-libs/gst/video/gstvideoaggregator.h

@@ -73,9 +73,9 @@ struct _GstVideoAggregator
  * @disable_frame_conversion: Optional.
  *                            Allows subclasses to disable the frame colorspace
  *                            conversion feature
- * @update_info:              Optional.
+ * @update_caps:              Optional.
- *                            Lets subclasses update the src #GstVideoInfo representing
+ *                            Lets subclasses update the #GstCaps representing
- *                            the src pad caps before usage.
+ *                            the src pad caps before usage.  Return %NULL to indicate failure.
  * @aggregate_frames:         Lets subclasses aggregate frames that are ready. Subclasses
  *                            should iterate the GstElement.sinkpads and use the already
  *                            mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame
@@ -96,8 +96,8 @@ struct _GstVideoAggregatorClass
   /*< public >*/
   gboolean           disable_frame_conversion;
 
-  gboolean           (*update_info)               (GstVideoAggregator *  videoaggregator,
+  GstCaps *          (*update_caps)               (GstVideoAggregator *  videoaggregator,
-                                                   GstVideoInfo       *  info);
+                                                   GstCaps            *  caps);
   GstFlowReturn      (*aggregate_frames)          (GstVideoAggregator *  videoaggregator,
                                                    GstBuffer          *  outbuffer);
   GstFlowReturn      (*get_output_buffer)         (GstVideoAggregator *  videoaggregator,

gst/compositor/compositor.c

@@ -437,12 +437,15 @@ set_functions (GstCompositor * self, GstVideoInfo * info)
   return ret;
 }
 
-static gboolean
+static GstCaps *
-_update_info (GstVideoAggregator * vagg, GstVideoInfo * info)
+_update_caps (GstVideoAggregator * vagg, GstCaps * caps)
 {
   GList *l;
   gint best_width = -1, best_height = -1;
-  gboolean ret = FALSE;
+  GstVideoInfo info;
+  GstCaps *ret = NULL;
+
+  gst_video_info_from_caps (&info, caps);
 
   GST_OBJECT_LOCK (vagg);
   for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
@@ -468,9 +471,10 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info)
   GST_OBJECT_UNLOCK (vagg);
 
   if (best_width > 0 && best_height > 0) {
-    gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info),
+    info.width = best_width;
-        best_width, best_height);
+    info.height = best_height;
-    ret = set_functions (GST_COMPOSITOR (vagg), info);
+    if (set_functions (GST_COMPOSITOR (vagg), &info))
+      ret = gst_video_info_to_caps (&info);
   }
 
   return ret;
@@ -559,7 +563,7 @@ gst_compositor_class_init (GstCompositorClass * klass)
   gobject_class->set_property = gst_compositor_set_property;
 
   agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD;
-  videoaggregator_class->update_info = _update_info;
+  videoaggregator_class->update_caps = _update_caps;
   videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames;
 
   g_object_class_install_property (gobject_class, PROP_BACKGROUND,