gl: readd glupload/download onto element pads

Allows insertion of gl elements into non-gl pipelines without converter
(upload/download) elements.

https://bugzilla.gnome.org/show_bug.cgi?id=743974

ext/gl/gstglcolorconvertelement.c

@@ -32,8 +32,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_gl_color_convert_element_debug);
 G_DEFINE_TYPE_WITH_CODE (GstGLColorConvertElement, gst_gl_color_convert_element,
     GST_TYPE_GL_BASE_FILTER,
     GST_DEBUG_CATEGORY_INIT (gst_gl_color_convert_element_debug,
-        "glconvertelement", 0, "convert");
+        "glconvertelement", 0, "convert"););
-    );
 
 static gboolean gst_gl_color_convert_element_set_caps (GstBaseTransform * bt,
     GstCaps * in_caps, GstCaps * out_caps);
@@ -52,22 +51,24 @@ static GstCaps *gst_gl_color_convert_element_fixate_caps (GstBaseTransform *
     bt, GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
 
 static GstStaticPadTemplate gst_gl_color_convert_element_src_pad_template =
-GST_STATIC_PAD_TEMPLATE ("src",
+    GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_GL_COLOR_CONVERT_VIDEO_CAPS));
+    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-
+        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-static GstStaticPadTemplate gst_gl_color_convert_element_sink_pad_template =
+            GST_GL_COLOR_CONVERT_FORMATS) ";"
-GST_STATIC_PAD_TEMPLATE ("sink",
+        GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)));
-    GST_PAD_SINK,
-    GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_GL_COLOR_CONVERT_VIDEO_CAPS));
 
 static gboolean
 gst_gl_color_convert_element_stop (GstBaseTransform * bt)
 {
   GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (bt);
 
+  if (convert->upload) {
+    gst_object_unref (convert->upload);
+    convert->upload = NULL;
+  }
+
   if (convert->convert) {
     gst_object_unref (convert->convert);
     convert->convert = NULL;
@@ -86,6 +87,7 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass)
 {
   GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass);
   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+  GstCaps *upload_caps;
 
   bt_class->transform_caps = gst_gl_color_convert_element_transform_caps;
   bt_class->set_caps = gst_gl_color_convert_element_set_caps;
@@ -102,9 +104,11 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass)
   gst_element_class_add_pad_template (element_class,
       gst_static_pad_template_get
       (&gst_gl_color_convert_element_src_pad_template));
+
+  upload_caps = gst_gl_upload_get_input_template_caps ();
   gst_element_class_add_pad_template (element_class,
-      gst_static_pad_template_get
+      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, upload_caps));
-      (&gst_gl_color_convert_element_sink_pad_template));
+  gst_caps_unref (upload_caps);
 
   gst_element_class_set_metadata (element_class,
       "OpenGL color converter", "Filter/Converter/Video",
@@ -128,9 +132,6 @@ gst_gl_color_convert_element_set_caps (GstBaseTransform * bt,
   gst_caps_replace (&convert->in_caps, in_caps);
   gst_caps_replace (&convert->out_caps, out_caps);
 
-  if (convert->convert)
-    gst_gl_color_convert_set_caps (convert->convert, in_caps, out_caps);
-
   return TRUE;
 }
 
@@ -139,8 +140,43 @@ gst_gl_color_convert_element_transform_caps (GstBaseTransform * bt,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 {
   GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
+  GstCaps *tmp, *ret;
 
-  return gst_gl_color_convert_transform_caps (context, direction, caps, filter);
+  if (direction == GST_PAD_SINK) {
+    ret = gst_gl_upload_transform_caps (context, direction, caps, NULL);
+  } else {
+    tmp =
+        gst_gl_caps_replace_all_caps_features (caps,
+        GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+    ret = gst_caps_merge (gst_caps_ref (caps), tmp);
+  }
+
+  GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, ret);
+
+  tmp = gst_gl_color_convert_transform_caps (context, direction, ret, NULL);
+  gst_caps_unref (ret);
+  ret = tmp;
+  GST_DEBUG_OBJECT (bt, "convert returned %" GST_PTR_FORMAT, ret);
+
+  if (direction == GST_PAD_SRC) {
+    tmp = gst_gl_upload_transform_caps (context, direction, ret, NULL);
+    gst_caps_unref (ret);
+    ret = tmp;
+  } else {
+    tmp =
+        gst_gl_caps_replace_all_caps_features (ret,
+        GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+    ret = gst_caps_merge (ret, tmp);
+  }
+  GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, ret);
+
+  if (filter) {
+    tmp = gst_caps_intersect_full (filter, ret, GST_CAPS_INTERSECT_FIRST);
+    gst_caps_unref (ret);
+    ret = tmp;
+  }
+
+  return ret;
 }
 
 static gboolean
@@ -163,6 +199,7 @@ gst_gl_color_convert_element_decide_allocation (GstBaseTransform * trans,
 {
   GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (trans);
   GstGLContext *context;
+  GstCaps *converted_caps;
 
   /* get gl context */
   if (!GST_BASE_TRANSFORM_CLASS
@@ -172,12 +209,35 @@ gst_gl_color_convert_element_decide_allocation (GstBaseTransform * trans,
 
   context = GST_GL_BASE_FILTER (trans)->context;
 
+  if (!convert->upload)
+    convert->upload = gst_gl_upload_new (context);
+
+  if (convert->upload_caps)
+    gst_caps_unref (convert->upload_caps);
+  convert->upload_caps = gst_caps_copy (convert->in_caps);
+  gst_caps_set_features (convert->upload_caps, 0,
+      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
+
+  if (!gst_gl_upload_set_caps (convert->upload, convert->in_caps,
+          convert->upload_caps)) {
+    GST_ERROR_OBJECT (trans, "failed to set caps for upload");
+    return FALSE;
+  }
+
   if (!convert->convert)
     convert->convert = gst_gl_color_convert_new (context);
 
-  if (!gst_gl_color_convert_set_caps (convert->convert, convert->in_caps,
+  converted_caps = gst_caps_copy (convert->out_caps);
-          convert->out_caps))
+  gst_caps_set_features (converted_caps, 0,
+      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
+
+  if (!gst_gl_color_convert_set_caps (convert->convert, convert->upload_caps,
+          converted_caps)) {
+    gst_caps_unref (converted_caps);
+    GST_ERROR_OBJECT (trans, "failed to set caps for conversion");
     return FALSE;
+  }
+  gst_caps_unref (converted_caps);
 
   return TRUE;
 }
@@ -187,21 +247,35 @@ gst_gl_color_convert_element_prepare_output_buffer (GstBaseTransform * bt,
     GstBuffer * inbuf, GstBuffer ** outbuf)
 {
   GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (bt);
+  GstBuffer *uploaded_buffer;
 
   if (gst_base_transform_is_passthrough (bt)) {
     *outbuf = inbuf;
     return GST_FLOW_OK;
   }
 
-  if (!convert->convert)
+  if (!convert->upload || !convert->convert)
     return GST_FLOW_NOT_NEGOTIATED;
 
-  *outbuf = gst_gl_color_convert_perform (convert->convert, inbuf);
+  if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (convert->upload,
+          inbuf, &uploaded_buffer) || !uploaded_buffer) {
+    GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s",
+            "failed to upload buffer"), (NULL));
+    return GST_FLOW_ERROR;
+  }
+
+  *outbuf = gst_gl_color_convert_perform (convert->convert, uploaded_buffer);
+  gst_buffer_unref (uploaded_buffer);
+  if (!*outbuf) {
+    GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND,
+        ("%s", "failed to convert buffer"), (NULL));
+    return GST_FLOW_ERROR;
+  }
 
   /* basetransform doesn't unref if they're the same */
   if (inbuf == *outbuf)
     gst_buffer_unref (*outbuf);
-  if (*outbuf)
+  else if (*outbuf)
     gst_buffer_copy_into (*outbuf, inbuf,
         GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
 

ext/gl/gstglcolorconvertelement.h

@@ -44,8 +44,10 @@ struct _GstGLColorConvertElement
 {
   GstGLBaseFilter        parent;
 
+  GstGLUpload *upload;
   GstGLColorConvert *convert;
   GstCaps *in_caps;
+  GstCaps *upload_caps;
   GstCaps *out_caps;
 };
 

ext/gl/gstglimagesink.c

@@ -156,22 +156,6 @@ static void
 gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
     gboolean handle_events);
 
-static GstStaticPadTemplate gst_glimage_sink_template =
-    GST_STATIC_PAD_TEMPLATE ("sink",
-    GST_PAD_SINK,
-    GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA") "; "
-#if GST_GL_HAVE_PLATFORM_EGL
-        GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, "RGBA") "; "
-#endif
-        GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META,
-            "RGBA") "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS))
-    );
-
 enum
 {
   ARG_0,
@@ -262,6 +246,7 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass)
   GstBaseSinkClass *gstbasesink_class;
   GstVideoSinkClass *gstvideosink_class;
   GstElementClass *element_class;
+  GstCaps *upload_caps, *elem_caps, *templ;
 
   gobject_class = (GObjectClass *) klass;
   gstelement_class = (GstElementClass *) klass;
@@ -343,8 +328,14 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass)
       G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
       G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT);
 
+  elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA");
+  upload_caps = gst_gl_upload_get_input_template_caps ();
+  templ = gst_caps_intersect (elem_caps, upload_caps);
   gst_element_class_add_pad_template (element_class,
-      gst_static_pad_template_get (&gst_glimage_sink_template));
+      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, templ));
+  gst_caps_unref (upload_caps);
+  gst_caps_unref (elem_caps);
+  gst_caps_unref (templ);
 
   gobject_class->finalize = gst_glimage_sink_finalize;
 
@@ -640,19 +631,6 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
   return res;
 }
 
-static gboolean
-gst_glimage_sink_stop (GstBaseSink * bsink)
-{
-  GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink);
-
-  if (glimage_sink->pool) {
-    gst_object_unref (glimage_sink->pool);
-    glimage_sink->pool = NULL;
-  }
-
-  return TRUE;
-}
-
 static void
 gst_glimage_sink_set_context (GstElement * element, GstContext * context)
 {
@@ -723,11 +701,6 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
         glimage_sink->upload = NULL;
       }
 
-      if (glimage_sink->convert) {
-        gst_object_unref (glimage_sink->convert);
-        glimage_sink->convert = NULL;
-      }
-
       glimage_sink->window_id = 0;
       /* but do not reset glimage_sink->new_window_id */
 
@@ -818,13 +791,6 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
 
   tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink));
 
-  result =
-      gst_gl_color_convert_transform_caps (gl_sink->context, GST_PAD_SRC, tmp,
-      NULL);
-  gst_caps_unref (tmp);
-  tmp = result;
-  GST_DEBUG_OBJECT (bsink, "convert returned caps %" GST_PTR_FORMAT, tmp);
-
   result =
       gst_gl_upload_transform_caps (gl_sink->context, GST_PAD_SRC, tmp, NULL);
   gst_caps_unref (tmp);
@@ -854,8 +820,6 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
   gint display_par_n, display_par_d;
   guint display_ratio_num, display_ratio_den;
   GstVideoInfo vinfo;
-  GstCapsFeatures *gl_features;
-  GstCaps *uploaded_caps;
 
   GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps);
 
@@ -923,33 +887,17 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
     gst_object_unref (glimage_sink->upload);
   glimage_sink->upload = gst_gl_upload_new (glimage_sink->context);
 
-  gl_features =
-      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
-
-  uploaded_caps = gst_caps_copy (caps);
-  gst_caps_set_features (uploaded_caps, 0,
-      gst_caps_features_copy (gl_features));
-  gst_gl_upload_set_caps (glimage_sink->upload, caps, uploaded_caps);
-
   if (glimage_sink->gl_caps)
     gst_caps_unref (glimage_sink->gl_caps);
   glimage_sink->gl_caps = gst_caps_copy (caps);
   gst_caps_set_simple (glimage_sink->gl_caps, "format", G_TYPE_STRING, "RGBA",
       NULL);
   gst_caps_set_features (glimage_sink->gl_caps, 0,
-      gst_caps_features_copy (gl_features));
+      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
 
-  if (glimage_sink->convert)
+  if (!gst_gl_upload_set_caps (glimage_sink->upload, caps,
-    gst_object_unref (glimage_sink->convert);
+          glimage_sink->gl_caps))
-  glimage_sink->convert = gst_gl_color_convert_new (glimage_sink->context);
-  if (!gst_gl_color_convert_set_caps (glimage_sink->convert, uploaded_caps,
-          glimage_sink->gl_caps)) {
-    gst_caps_unref (uploaded_caps);
-    gst_caps_features_free (gl_features);
     return FALSE;
-  }
-  gst_caps_unref (uploaded_caps);
-  gst_caps_features_free (gl_features);
 
   glimage_sink->caps_change = TRUE;
 
@@ -960,7 +908,7 @@ static GstFlowReturn
 gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
 {
   GstGLImageSink *glimage_sink;
-  GstBuffer *uploaded_buffer, *next_buffer = NULL;
+  GstBuffer *next_buffer = NULL;
   GstVideoFrame gl_frame;
   GstVideoInfo gl_info;
 
@@ -977,25 +925,16 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
     return GST_FLOW_NOT_NEGOTIATED;
 
   if (gst_gl_upload_perform_with_buffer (glimage_sink->upload, buf,
-          &uploaded_buffer) != GST_GL_UPLOAD_DONE)
+          &next_buffer) != GST_GL_UPLOAD_DONE)
     goto upload_failed;
 
-  if (!(next_buffer =
-          gst_gl_color_convert_perform (glimage_sink->convert,
-              uploaded_buffer))) {
-    gst_buffer_unref (uploaded_buffer);
-    goto upload_failed;
-  }
-
   gst_video_info_from_caps (&gl_info, glimage_sink->gl_caps);
 
   if (!gst_video_frame_map (&gl_frame, &gl_info, next_buffer,
           GST_MAP_READ | GST_MAP_GL)) {
-    gst_buffer_unref (uploaded_buffer);
     gst_buffer_unref (next_buffer);
     goto upload_failed;
   }
-  gst_buffer_unref (uploaded_buffer);
 
   glimage_sink->next_tex = *(guint *) gl_frame.data[0];
 
@@ -1487,9 +1426,8 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink)
     return FALSE;
 
   if (gst_gl_window_is_running (window)) {
-    gulong handler_id =
+    gulong handler_id = g_signal_handler_find (gl_sink, G_SIGNAL_MATCH_ID,
-        g_signal_handler_find (GST_ELEMENT_PARENT (gl_sink), G_SIGNAL_MATCH_ID,
+        gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0,
-        gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW], 0,
         NULL, NULL, NULL);
 
     if (G_UNLIKELY (!gl_sink->redisplay_shader) && (!handler_id

ext/gl/gstglimagesink.h

@@ -70,7 +70,6 @@ struct _GstGLImageSink
     gboolean ignore_alpha;
 
     GstGLUpload *upload;
-    GstGLColorConvert *convert;
     guint      next_tex;
     GstBuffer *next_buffer;
 

ext/gl/gstglmixer.c

@@ -42,6 +42,9 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec);
 static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec);
+static void gst_gl_mixer_pad_finalize (GObject * object);
+static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix,
+    GstGLMixerFrameData * frame, GstBuffer * buffer);
 
 enum
 {
@@ -71,10 +74,13 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass)
 
   gobject_class->set_property = gst_gl_mixer_pad_set_property;
   gobject_class->get_property = gst_gl_mixer_pad_get_property;
+  gobject_class->finalize = gst_gl_mixer_pad_finalize;
 
   vaggpad_class->set_info = NULL;
   vaggpad_class->prepare_frame = NULL;
   vaggpad_class->clean_frame = NULL;
+
+  klass->upload_buffer = _default_pad_upload_buffer;
 }
 
 static void
@@ -198,12 +204,22 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix,
   return ret;
 }
 
-/* copies the given caps */
+GstCaps *
-static GstCaps *
+gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps)
-_update_caps (GstVideoAggregator * vagg, GstCaps * caps)
 {
-  return gst_gl_caps_replace_all_caps_features (caps,
+  GstGLContext *context = GST_GL_BASE_MIXER (mix)->context;
-      GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+  GstCaps *result, *tmp;
+
+  result =
+      gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+      (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA"));
+
+  tmp = gst_gl_upload_transform_caps (context, GST_PAD_SRC, result, NULL);
+  gst_caps_unref (result);
+  result = tmp;
+  GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, result);
+
+  return result;
 }
 
 static GstCaps *
@@ -284,6 +300,17 @@ gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad)
 {
 }
 
+static void
+gst_gl_mixer_pad_finalize (GObject * object)
+{
+  GstGLMixerPad *pad = GST_GL_MIXER_PAD (object);
+
+  gst_buffer_replace (&pad->uploaded_buffer, NULL);
+  gst_object_replace ((GstObject **) & pad->upload, NULL);
+
+  G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object);
+}
+
 /* GLMixer signals and args */
 enum
 {
@@ -301,15 +328,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
         (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA"))
+            "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA"))
-    );
-
-static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
-    GST_PAD_SINK,
-    GST_PAD_REQUEST,
-    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA"))
     );
 
 static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query);
@@ -340,7 +359,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass)
   GstVideoAggregatorClass *videoaggregator_class =
       (GstVideoAggregatorClass *) klass;
   GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
-  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);;
+  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);
+  GstCaps *upload_caps, *elem_caps, *templ;
 
   GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer");
 
@@ -353,8 +373,15 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass)
 
   gst_element_class_add_pad_template (element_class,
       gst_static_pad_template_get (&src_factory));
+
+  elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA");
+  upload_caps = gst_gl_upload_get_input_template_caps ();
+  templ = gst_caps_intersect (elem_caps, upload_caps);
   gst_element_class_add_pad_template (element_class,
-      gst_static_pad_template_get (&sink_factory));
+      gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, templ));
+  gst_caps_unref (upload_caps);
+  gst_caps_unref (elem_caps);
+  gst_caps_unref (templ);
 
   agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD;
   agg_class->sink_query = gst_gl_mixer_sink_query;
@@ -365,7 +392,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass)
   videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames;
   videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer;
   videoaggregator_class->negotiated_caps = _negotiated_caps;
-  videoaggregator_class->update_caps = _update_caps;
   videoaggregator_class->find_best_format = NULL;
 
   mix_class->propose_allocation = gst_gl_mixer_propose_allocation;
@@ -545,8 +571,11 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query)
     update_pool = FALSE;
   }
 
-  if (!pool)
+  if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) {
+    if (pool)
+      gst_object_unref (pool);
     pool = gst_gl_buffer_pool_new (context);
+  }
   config = gst_buffer_pool_get_config (pool);
 
   gst_buffer_pool_config_set_params (config, caps, size, min, max);
@@ -570,6 +599,63 @@ context_error:
   }
 }
 
+static GstBuffer *
+_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame,
+    GstBuffer * buffer)
+{
+  GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad);
+  GstGLMixerPad *pad = frame->pad;
+  GstVideoInfo gl_info;
+  GstGLSyncMeta *sync_meta;
+
+  gst_video_info_set_format (&gl_info,
+      GST_VIDEO_FORMAT_RGBA,
+      GST_VIDEO_INFO_WIDTH (&vaggpad->info),
+      GST_VIDEO_INFO_HEIGHT (&vaggpad->info));
+
+  if (!pad->upload) {
+    GstCaps *in_caps, *upload_caps;
+
+    in_caps = gst_pad_get_current_caps (GST_PAD (pad));
+    upload_caps = gst_video_info_to_caps (&gl_info);
+    gst_caps_set_features (upload_caps, 0,
+        gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
+
+    pad->upload = gst_gl_upload_new (GST_GL_BASE_MIXER (mix)->context);
+
+    if (!gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps)) {
+      GST_WARNING_OBJECT (pad, "Failed to configure uploader");
+      gst_caps_unref (in_caps);
+      gst_caps_unref (upload_caps);
+      gst_object_unref (pad->upload);
+      pad->upload = NULL;
+      return NULL;
+    }
+
+    gst_caps_unref (in_caps);
+    gst_caps_unref (upload_caps);
+  }
+
+  sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer);
+  if (sync_meta)
+    gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context);
+
+  if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (pad->upload,
+          vaggpad->buffer, &pad->uploaded_buffer) || !pad->uploaded_buffer) {
+    GST_WARNING_OBJECT (pad, "Failed to upload input buffer %p",
+        vaggpad->buffer);
+    return NULL;
+  }
+
+  if (gst_video_frame_map (&frame->v_frame, &gl_info, pad->uploaded_buffer,
+          GST_MAP_READ | GST_MAP_GL)) {
+    frame->texture = *(guint *) frame->v_frame.data[0];
+    frame->mapped = TRUE;
+  }
+
+  return pad->uploaded_buffer;
+}
+
 gboolean
 gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
 {
@@ -577,7 +663,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
   GList *walk;
   guint out_tex;
   gboolean res = TRUE;
-  guint array_index = 0;
+  gint array_index = 0;
   GstVideoFrame out_frame;
   GstElement *element = GST_ELEMENT (mix);
   GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix);
@@ -602,34 +688,28 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
     mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData);
   while (walk) {
     GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data);
+    GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad);
     GstVideoAggregatorPad *vaggpad = walk->data;
     GstGLMixerFrameData *frame;
 
     frame = g_ptr_array_index (mix->frames, array_index);
     frame->pad = pad;
     frame->texture = 0;
+    frame->mapped = FALSE;
 
     walk = g_list_next (walk);
 
     if (vaggpad->buffer != NULL) {
-      GstVideoInfo gl_info;
+      g_assert (pad_class->upload_buffer);
-      GstVideoFrame gl_frame;
-      GstGLSyncMeta *sync_meta;
 
-      gst_video_info_set_format (&gl_info,
+      if (pad->uploaded_buffer)
-          GST_VIDEO_FORMAT_RGBA,
+        gst_buffer_unref (pad->uploaded_buffer);
-          GST_VIDEO_INFO_WIDTH (&vaggpad->info),
+      pad->uploaded_buffer =
-          GST_VIDEO_INFO_HEIGHT (&vaggpad->info));
+          pad_class->upload_buffer (mix, frame, vaggpad->buffer);
 
-      sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer);
+      GST_DEBUG_OBJECT (pad,
-      if (sync_meta)
+          "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT,
-        gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context);
+          pad->uploaded_buffer, vaggpad->buffer);
-
-      if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer,
-              GST_MAP_READ | GST_MAP_GL)) {
-        frame->texture = *(guint *) gl_frame.data[0];
-        gst_video_frame_unmap (&gl_frame);
-      }
     }
 
     ++array_index;
@@ -652,6 +732,19 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
   g_mutex_unlock (&priv->gl_resource_lock);
 
 out:
+  for (--array_index; array_index >= 0; array_index--) {
+    GstGLMixerFrameData *frame;
+
+    frame = g_ptr_array_index (mix->frames, array_index);
+
+    if (frame->mapped)
+      gst_video_frame_unmap (&frame->v_frame);
+    frame->mapped = FALSE;
+    frame->texture = 0;
+
+    gst_buffer_replace (&frame->pad->uploaded_buffer, NULL);
+  }
+
   GST_OBJECT_UNLOCK (mix);
 
   gst_video_frame_unmap (&out_frame);
@@ -761,6 +854,18 @@ gst_gl_mixer_start (GstAggregator * agg)
   return GST_AGGREGATOR_CLASS (parent_class)->start (agg);
 }
 
+static gboolean
+_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad,
+    gpointer user_data)
+{
+  GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad);
+
+  gst_buffer_replace (&pad->uploaded_buffer, NULL);
+  gst_object_replace ((GstObject **) & pad->upload, NULL);
+
+  return TRUE;
+}
+
 static gboolean
 gst_gl_mixer_stop (GstAggregator * agg)
 {
@@ -768,6 +873,9 @@ gst_gl_mixer_stop (GstAggregator * agg)
   GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix);
   GstGLContext *context = GST_GL_BASE_MIXER (mix)->context;
 
+  gst_aggregator_iterate_sinkpads (agg,
+      (GstAggregatorPadForeachFunc) _clean_upload, NULL);
+
   GST_OBJECT_LOCK (agg);
   g_ptr_array_free (mix->frames, TRUE);
   mix->frames = NULL;

ext/gl/gstglmixer.h

@@ -52,11 +52,16 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass;
 struct _GstGLMixerPad
 {
   GstGLBaseMixerPad parent;
+
+  GstGLUpload *upload;
+  GstBuffer *uploaded_buffer;
 };
 
 struct _GstGLMixerPadClass
 {
   GstGLBaseMixerPadClass parent_class;
+
+  GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer);
 };
 
 GType gst_gl_mixer_pad_get_type (void);
@@ -109,12 +114,15 @@ struct _GstGLMixerClass
 struct _GstGLMixerFrameData
 {
   GstGLMixerPad *pad;
+  gboolean mapped;
+  GstVideoFrame v_frame;
   guint texture;
 };
 
 GType gst_gl_mixer_get_type(void);
 
 gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf);
+GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps);
 
 G_END_DECLS
 #endif /* __GST_GL_MIXER_H__ */

ext/gl/gstglmosaic.c

@@ -190,6 +190,9 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps)
 {
   GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer);
 
+  if (mosaic->shader)
+    gst_object_unref (mosaic->shader);
+
   //blocking call, wait the opengl thread has compiled the shader
   return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context,
       mosaic_v_src, mosaic_f_src, &mosaic->shader);

ext/gl/gstgltestsrc.c

@@ -68,7 +68,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
         (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA"))
+            "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA"))
     );
 
 #define gst_gl_test_src_parent_class parent_class
@@ -455,8 +455,7 @@ static GstCaps *
 gst_gl_test_src_getcaps (GstBaseSrc * bsrc, GstCaps * filter)
 {
   GstCaps *tmp = NULL;
-  GstCaps *result =
+  GstCaps *result = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (bsrc));
-      gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA");
 
   if (filter) {
     tmp = gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST);

gst-libs/gst/gl/gstglcolorconvert.c

@@ -676,23 +676,21 @@ gst_gl_color_convert_caps_remove_format_info (GstCaps * caps)
 }
 
 GstCaps *
-gst_gl_color_convert_transform_caps (GstGLContext * convert,
+gst_gl_color_convert_transform_caps (GstGLContext * context,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter)
 {
-  GstCaps *templ, *result;
+  GstCaps *templ, *tmp, *result;
 
   templ =
       gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
       (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_COLOR_CONVERT_FORMATS));
 
-  caps = gst_gl_color_convert_caps_remove_format_info (caps);
+  tmp = gst_gl_color_convert_caps_remove_format_info (caps);
-  result = gst_caps_intersect (caps, templ);
+  result = gst_caps_intersect (tmp, templ);
-  gst_caps_unref (caps);
+  gst_caps_unref (tmp);
   gst_caps_unref (templ);
 
   if (filter) {
-    GstCaps *tmp;
-
     tmp = gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST);
     gst_caps_unref (result);
     result = tmp;

gst-libs/gst/gl/gstglfilter.c

@@ -37,21 +37,12 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
 
 
 static GstStaticPadTemplate gst_gl_filter_src_pad_template =
-GST_STATIC_PAD_TEMPLATE ("src",
+    GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
         (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA"))
+            "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA"))
-    );
-
-static GstStaticPadTemplate gst_gl_filter_sink_pad_template =
-GST_STATIC_PAD_TEMPLATE ("sink",
-    GST_PAD_SINK,
-    GST_PAD_ALWAYS,
-    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
-            "RGBA"))
     );
 
 /* Properties */
@@ -63,7 +54,8 @@ enum
 #define gst_gl_filter_parent_class parent_class
 G_DEFINE_TYPE_WITH_CODE (GstGLFilter, gst_gl_filter, GST_TYPE_GL_BASE_FILTER,
     GST_DEBUG_CATEGORY_INIT (gst_gl_filter_debug, "glfilter", 0,
-        "glfilter element"););
+        "glfilter element");
+    );
 
 static void gst_gl_filter_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec);
@@ -94,6 +86,7 @@ gst_gl_filter_class_init (GstGLFilterClass * klass)
 {
   GObjectClass *gobject_class;
   GstElementClass *element_class;
+  GstCaps *upload_caps, *elem_caps, *templ;
 
   gobject_class = (GObjectClass *) klass;
   element_class = GST_ELEMENT_CLASS (klass);
@@ -118,8 +111,15 @@ gst_gl_filter_class_init (GstGLFilterClass * klass)
 
   gst_element_class_add_pad_template (element_class,
       gst_static_pad_template_get (&gst_gl_filter_src_pad_template));
+
+  elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA");
+  upload_caps = gst_gl_upload_get_input_template_caps ();
+  templ = gst_caps_intersect (elem_caps, upload_caps);
   gst_element_class_add_pad_template (element_class,
-      gst_static_pad_template_get (&gst_gl_filter_sink_pad_template));
+      gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, templ));
+  gst_caps_unref (upload_caps);
+  gst_caps_unref (elem_caps);
+  gst_caps_unref (templ);
 }
 
 static void
@@ -156,6 +156,11 @@ gst_gl_filter_reset (GstGLFilter * filter)
 {
   gst_caps_replace (&filter->out_caps, NULL);
 
+  if (filter->upload) {
+    gst_object_unref (filter->upload);
+    filter->upload = NULL;
+  }
+
   if (filter->pool) {
     gst_object_unref (filter->pool);
     filter->pool = NULL;
@@ -605,8 +610,7 @@ gst_gl_filter_caps_remove_size (GstCaps * caps)
 }
 
 static GstCaps *
-gst_gl_filter_set_caps_features (const GstCaps * caps,
+gst_gl_filter_set_caps_features (GstCaps * caps, const gchar * feature_name)
-    const gchar * feature_name)
 {
   GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name);
   gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
@@ -617,23 +621,48 @@ static GstCaps *
 gst_gl_filter_transform_caps (GstBaseTransform * bt,
     GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps)
 {
+  GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
   GstCaps *tmp = NULL;
   GstCaps *result = NULL;
 
-  result = gst_gl_filter_caps_remove_size (caps);
+  if (direction == GST_PAD_SINK) {
-  tmp = result;
+    result = gst_gl_upload_transform_caps (context, direction, caps, NULL);
-  GST_DEBUG_OBJECT (bt, "size removal returned caps %" GST_PTR_FORMAT, tmp);
+  } else {
+    tmp =
+        gst_gl_caps_replace_all_caps_features (caps,
+        GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+    result = gst_caps_merge (gst_caps_ref (caps), tmp);
+  }
 
-  result =
+  GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, result);
-      gst_gl_filter_set_caps_features (tmp, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+
-  gst_caps_unref (tmp);
+  tmp = gst_gl_filter_caps_remove_size (result);
-  tmp = result;
+  gst_caps_unref (result);
+  result = tmp;
+  GST_DEBUG_OBJECT (bt, "size removal returned caps %" GST_PTR_FORMAT, result);
+
+  tmp =
+      gst_gl_filter_set_caps_features (result,
+      GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+  gst_caps_unref (result);
+  result = tmp;
+
+  if (direction == GST_PAD_SRC) {
+    tmp = gst_gl_upload_transform_caps (context, direction, result, NULL);
+    gst_caps_unref (result);
+    result = tmp;
+  } else {
+    tmp =
+        gst_gl_caps_replace_all_caps_features (result,
+        GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+    result = gst_caps_merge (result, tmp);
+  }
+  GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, result);
 
   if (filter_caps) {
-    result =
+    tmp =
-        gst_caps_intersect_full (filter_caps, tmp, GST_CAPS_INTERSECT_FIRST);
+        gst_caps_intersect_full (filter_caps, result, GST_CAPS_INTERSECT_FIRST);
-    gst_caps_unref (tmp);
+    gst_caps_unref (result);
-  } else {
     result = tmp;
   }
 
@@ -777,10 +806,11 @@ config_failed:
 static gboolean
 gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query)
 {
+  GstGLFilter *filter = GST_GL_FILTER (trans);
   GstGLContext *context;
   GstBufferPool *pool = NULL;
   GstStructure *config;
-  GstCaps *caps;
+  GstCaps *caps, *upload_caps;
   guint min, max, size;
   gboolean update_pool;
 
@@ -832,6 +862,18 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query)
 
   gst_object_unref (pool);
 
+  if (!filter->upload)
+    filter->upload = gst_gl_upload_new (context);
+
+  upload_caps = gst_caps_copy (caps);
+  gst_caps_set_features (upload_caps, 0,
+      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
+  if (!gst_gl_upload_set_caps (filter->upload, caps, upload_caps)) {
+    gst_caps_unref (upload_caps);
+    return FALSE;
+  }
+  gst_caps_unref (upload_caps);
+
   return TRUE;
 }
 
@@ -853,11 +895,18 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
   GstGLFilterClass *filter_class;
   guint in_tex, out_tex;
   GstVideoFrame gl_frame, out_frame;
+  GstBuffer *gl_buffer;
   gboolean ret;
 
   filter_class = GST_GL_FILTER_GET_CLASS (filter);
 
-  if (!gst_video_frame_map (&gl_frame, &filter->in_info, inbuf,
+  if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (filter->upload,
+          inbuf, &gl_buffer) || !gl_buffer) {
+    ret = FALSE;
+    goto gl_buf_error;
+  }
+
+  if (!gst_video_frame_map (&gl_frame, &filter->in_info, gl_buffer,
           GST_MAP_READ | GST_MAP_GL)) {
     ret = FALSE;
     goto inbuf_error;
@@ -883,6 +932,8 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
 unmap_out_error:
   gst_video_frame_unmap (&gl_frame);
 inbuf_error:
+  gst_buffer_unref (gl_buffer);
+gl_buf_error:
 
   return ret;
 }

gst-libs/gst/gl/gstglfilter.h

@@ -66,6 +66,8 @@ struct _GstGLFilter
   GstCaps           *out_caps;
 
   /* <private> */
+  GstGLUpload       *upload;
+
   GLuint             fbo;
   GLuint             depthbuffer;
 

gst-libs/gst/gl/gstglutils.c

@@ -881,26 +881,16 @@ gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align,
 }
 
 GstCaps *
-gst_gl_caps_replace_all_caps_features (const GstCaps * caps,
+gst_gl_caps_replace_all_caps_features (GstCaps * caps,
     const gchar * feature_name)
 {
   GstCaps *tmp = gst_caps_copy (caps);
   guint n = gst_caps_get_size (tmp);
   guint i = 0;
 
-  for (i = 0; i < n; i++) {
+  for (i = 0; i < n; i++)
-    GstCapsFeatures *features = gst_caps_get_features (tmp, i);
+    gst_caps_set_features (tmp, i,
-    if (features) {
+        gst_caps_features_from_string (feature_name));
-      guint n_f = gst_caps_features_get_size (features);
-      guint j = 0;
-      for (j = 0; j < n_f; j++) {
-        gst_caps_features_remove_id (features,
-            gst_caps_features_get_nth_id (features, j));
-      }
-    }
-
-    gst_caps_features_add (features, feature_name);
-  }
 
   return tmp;
 }

gst-libs/gst/gl/gstglutils.h

@@ -103,7 +103,7 @@ gboolean gst_gl_run_query (GstElement * element,
     GstQuery * query, GstPadDirection direction);
 gsize gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align,
     guint plane);
-GstCaps * gst_gl_caps_replace_all_caps_features (const GstCaps * caps,
+GstCaps * gst_gl_caps_replace_all_caps_features (GstCaps * caps,
     const gchar * feature_name);
 
 G_END_DECLS