glupload: rearchitecture for non GLMemory inputs/outputs

Allows other memory types to be implemented/returned/used by the caller.

ext/gl/gstglimagesink.c

@@ -620,6 +620,11 @@ gst_glimage_sink_stop (GstBaseSink * bsink)
     glimage_sink->pool = NULL;
   }
 
+  if (glimage_sink->gl_caps) {
+    gst_caps_unref (glimage_sink->gl_caps);
+    glimage_sink->gl_caps = NULL;
+  }
+
   return TRUE;
 }
 
@@ -764,6 +769,7 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
   GstVideoInfo vinfo;
   GstStructure *structure;
   GstBufferPool *newpool, *oldpool;
+  GstCapsFeatures *gl_features;
 
   GST_DEBUG ("set caps with %" GST_PTR_FORMAT, caps);
 
@@ -848,7 +854,16 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
     gst_object_unref (glimage_sink->upload);
   glimage_sink->upload = gst_gl_upload_new (glimage_sink->context);
 
-  gst_gl_upload_set_format (glimage_sink->upload, &vinfo);
+  gl_features =
+      gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+  if (glimage_sink->gl_caps)
+    gst_caps_unref (glimage_sink->gl_caps);
+  glimage_sink->gl_caps = gst_caps_copy (caps);
+  gst_caps_set_simple (glimage_sink->gl_caps, "format", G_TYPE_STRING, "RGBA",
+      NULL);
+  gst_caps_set_features (glimage_sink->gl_caps, 0, gl_features);
+
+  gst_gl_upload_set_caps (glimage_sink->upload, caps, glimage_sink->gl_caps);
   glimage_sink->caps_change = TRUE;
 
   return TRUE;
@@ -859,6 +874,8 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
 {
   GstGLImageSink *glimage_sink;
   GstBuffer *next_buffer = NULL;
+  GstVideoFrame gl_frame;
+  GstVideoInfo gl_info;
 
   glimage_sink = GST_GLIMAGE_SINK (bsink);
 
@@ -873,12 +890,24 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
     return GST_FLOW_NOT_NEGOTIATED;
 
   if (!gst_gl_upload_perform_with_buffer (glimage_sink->upload, buf,
-          &glimage_sink->next_tex, &next_buffer))
+          &next_buffer))
     goto upload_failed;
 
+  gst_video_info_from_caps (&gl_info, glimage_sink->gl_caps);
+
+  if (!gst_video_frame_map (&gl_frame, &gl_info, next_buffer,
+          GST_MAP_READ | GST_MAP_GL)) {
+    gst_buffer_unref (next_buffer);
+    goto upload_failed;
+  }
+
+  glimage_sink->next_tex = *(guint *) gl_frame.data[0];
+
   gst_buffer_replace (&glimage_sink->next_buffer, next_buffer);
   gst_buffer_unref (next_buffer);
 
+  gst_video_frame_unmap (&gl_frame);
+
   if (glimage_sink->window_id != glimage_sink->new_window_id) {
     GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
 

ext/gl/gstglimagesink.h

@@ -61,6 +61,7 @@ struct _GstGLImageSink
 
     //caps
     GstVideoInfo info;
+    GstCaps *gl_caps;
 
     GstGLDisplay *display;
     GstGLContext *context;

ext/gl/gstglmixer.c

@@ -1064,23 +1064,55 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
     walk = g_list_next (walk);
 
     if (vaggpad->buffer != NULL) {
-      guint in_tex;
+      GstBuffer *gl_buf;
+      GstCaps *gl_caps;
+      GstCapsFeatures *gl_features;
+      GstVideoInfo gl_info;
+      GstVideoFrame gl_frame;
+
+      gst_video_info_set_format (&gl_info,
+          GST_VIDEO_FORMAT_RGBA,
+          GST_VIDEO_INFO_WIDTH (&vaggpad->info),
+          GST_VIDEO_INFO_HEIGHT (&vaggpad->info));
+      gl_features =
+          gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+
+      gl_caps = gst_video_info_to_caps (&gl_info);
+      gst_caps_set_features (gl_caps, 0, gl_features);
 
       if (!pad->upload) {
+        GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad));
+
         pad->upload = gst_gl_upload_new (mix->context);
 
-        gst_gl_upload_set_format (pad->upload, &vaggpad->info);
+        gst_gl_upload_set_caps (pad->upload, in_caps, gl_caps);
+
+        gst_caps_unref (in_caps);
       }
 
       if (!gst_gl_upload_perform_with_buffer (pad->upload,
-              vaggpad->buffer, &in_tex, NULL)) {
+              vaggpad->buffer, &gl_buf)) {
         ++array_index;
         pad->mapped = FALSE;
+        gst_caps_unref (gl_caps);
+        continue;
+      }
+
+      if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buf,
+              GST_MAP_READ | GST_MAP_GL)) {
+        ++array_index;
+        pad->mapped = FALSE;
+        gst_buffer_unref (gl_buf);
+        gst_caps_unref (gl_caps);
         continue;
       }
       pad->mapped = TRUE;
 
-      frame->texture = in_tex;
+      frame->texture = *(guint *) gl_frame.data[0];
+
+      gst_caps_unref (gl_caps);
+      gst_video_frame_unmap (&gl_frame);
+      gst_buffer_unref (gl_buf);
     }
     ++array_index;
   }

gst-libs/gst/gl/egl/gsteglimagememory.c

@@ -25,27 +25,10 @@
 #endif
 
 #include "gsteglimagememory.h"
-#include "gstglcontext_egl.h"
 
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_EGL_IMAGE_MEMORY);
 #define GST_CAT_DEFAULT GST_CAT_EGL_IMAGE_MEMORY
 
-typedef void (*GstEGLImageDestroyNotify) (GstGLContextEGL * context,
-    gpointer data);
-
-typedef struct
-{
-  GstMemory parent;
-
-  GstGLContextEGL *context;
-  EGLImageKHR image;
-  GstVideoGLTextureType type;
-  GstVideoGLTextureOrientation orientation;
-
-  gpointer user_data;
-  GstEGLImageDestroyNotify user_data_destroy;
-} GstEGLImageMemory;
-
 #define GST_EGL_IMAGE_MEMORY(mem) ((GstEGLImageMemory*)(mem))
 
 gboolean
@@ -317,13 +300,7 @@ gst_eglimage_to_gl_texture_upload_meta (GstVideoGLTextureUploadMeta *
 
     gl = GST_GL_CONTEXT (GST_EGL_IMAGE_MEMORY (mem)->context)->gl_vtable;
 
-    if (i == 0)
-      gl->ActiveTexture (GL_TEXTURE0);
-    else if (i == 1)
-      gl->ActiveTexture (GL_TEXTURE1);
-    else if (i == 2)
-      gl->ActiveTexture (GL_TEXTURE2);
-
+    gl->ActiveTexture (GL_TEXTURE0 + i);
     gl->BindTexture (GL_TEXTURE_2D, texture_id[i]);
     gl->EGLImageTargetTexture2D (GL_TEXTURE_2D,
         gst_egl_image_memory_get_image (mem));

gst-libs/gst/gl/egl/gsteglimagememory.h

@@ -29,13 +29,32 @@
 #include <gst/video/video.h>
 
 #include <gst/gl/gl.h>
+#include "gstglcontext_egl.h"
 
 G_BEGIN_DECLS
 
+typedef void (*GstEGLImageDestroyNotify) (GstGLContextEGL * context,
+    gpointer data);
+
+typedef struct _GstEGLImageMemory GstEGLImageMemory;
+
 #define GST_EGL_IMAGE_MEMORY_TYPE "EGLImage"
 
 #define GST_CAPS_FEATURE_MEMORY_EGL_IMAGE "memory:EGLImage"
 
+struct _GstEGLImageMemory
+{
+  GstMemory parent;
+
+  GstGLContextEGL *context;
+  EGLImageKHR image;
+  GstVideoGLTextureType type;
+  GstVideoGLTextureOrientation orientation;
+
+  gpointer user_data;
+  GstEGLImageDestroyNotify user_data_destroy;
+};
+
 void gst_egl_image_memory_init (void);
 gboolean gst_egl_image_memory_setup_buffer (GstGLContext * context, GstVideoInfo * info, GstBuffer * buffer);
 gboolean gst_is_egl_image_memory (GstMemory * mem);

gst-libs/gst/gl/gstglfilter.c

@@ -298,6 +298,11 @@ gst_gl_filter_reset (GstGLFilter * filter)
   if (filter->context)
     gst_object_unref (filter->context);
   filter->context = NULL;
+
+  if (filter->in_gl_caps) {
+    gst_caps_unref (filter->in_gl_caps);
+    filter->in_gl_caps = NULL;
+  }
 }
 
 static gboolean
@@ -1185,7 +1190,8 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
 {
   GstGLFilterClass *filter_class;
   guint in_tex, out_tex;
-  GstVideoFrame out_frame;
+  GstVideoFrame gl_frame, out_frame;
+  GstVideoInfo gl_info;
   gboolean ret;
   gboolean to_download =
       gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY,
@@ -1199,9 +1205,21 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
     filter->uploaded_buffer = NULL;
   }
 
-  if (!gst_gl_upload_perform_with_buffer (filter->upload, inbuf, &in_tex,
-          &filter->uploaded_buffer))
-    return FALSE;
+  if (!gst_gl_upload_perform_with_buffer (filter->upload, inbuf,
+          &filter->uploaded_buffer)) {
+    ret = FALSE;
+    goto upload_error;
+  }
+
+  gst_video_info_from_caps (&gl_info, filter->in_gl_caps);
+
+  if (!gst_video_frame_map (&gl_frame, &gl_info, filter->uploaded_buffer,
+          GST_MAP_READ | GST_MAP_GL)) {
+    ret = FALSE;
+    goto upload_error;
+  }
+
+  in_tex = *(guint *) gl_frame.data[0];
 
   to_download |= !gst_is_gl_memory (gst_buffer_peek_memory (outbuf, 0));
 
@@ -1240,11 +1258,11 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf,
       GST_ELEMENT_ERROR (filter, RESOURCE, NOT_FOUND,
           ("%s", "Failed to download video frame"), (NULL));
       ret = FALSE;
-      goto error;
     }
   }
 
-error:
+  gst_video_frame_unmap (&gl_frame);
+upload_error:
   gst_video_frame_unmap (&out_frame);
 inbuf_error:
   gst_gl_upload_release_buffer (filter->upload);
@@ -1267,8 +1285,28 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf,
     return GST_FLOW_NOT_NEGOTIATED;
 
   if (!filter->upload) {
+    GstCaps *in_caps =
+        gst_pad_get_current_caps (GST_BASE_TRANSFORM_SINK_PAD (bt));
+    GstCapsFeatures *out_features;
+    GstVideoInfo out_info;
+
+    gst_video_info_set_format (&out_info,
+        GST_VIDEO_FORMAT_RGBA,
+        GST_VIDEO_INFO_WIDTH (&filter->in_info),
+        GST_VIDEO_INFO_HEIGHT (&filter->in_info));
+    out_features =
+        gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
+
+    if (filter->in_gl_caps)
+      gst_caps_unref (filter->in_gl_caps);
+    filter->in_gl_caps = gst_video_info_to_caps (&out_info);
+    gst_caps_set_features (filter->in_gl_caps, 0, out_features);
+
     filter->upload = gst_gl_upload_new (filter->context);
-    gst_gl_upload_set_format (filter->upload, &filter->in_info);
+    if (!gst_gl_upload_set_caps (filter->upload, in_caps, filter->in_gl_caps))
+      return GST_FLOW_ERROR;
+
+    gst_caps_unref (in_caps);
   }
 
   g_assert (filter_class->filter || filter_class->filter_texture);

gst-libs/gst/gl/gstglfilter.h

@@ -88,6 +88,8 @@ struct _GstGLFilter
   GstGLContext      *context;
   GstGLContext      *other_context;
 
+  GstCaps           *in_gl_caps;
+
 #if GST_GL_HAVE_GLES2
   GLint draw_attr_position_loc;
   GLint draw_attr_texture_loc;

gst-libs/gst/gl/gstglupload.h

@@ -22,7 +22,6 @@
 #define __GST_GL_UPLOAD_H__
 
 #include <gst/video/video.h>
-#include <gst/gstmemory.h>
 
 #include <gst/gl/gstgl_fwd.h>
 
@@ -47,17 +46,7 @@ struct _GstGLUpload
   GstObject        parent;
 
   GstGLContext    *context;
-  GstGLColorConvert *convert;
 
-  /* input data */
-  GstVideoInfo     in_info;
-
-  gboolean         initted;
-
-  GstGLMemory     *in_tex[GST_VIDEO_MAX_PLANES];
-  GstGLMemory     *out_tex;
-
-  /* <private> */
   GstGLUploadPrivate *priv;
 
   gpointer _reserved[GST_PADDING];
@@ -75,15 +64,11 @@ struct _GstGLUploadClass
 
 GstGLUpload * gst_gl_upload_new            (GstGLContext * context);
 
-void           gst_gl_upload_set_format    (GstGLUpload * upload, GstVideoInfo * in_info);
-GstVideoInfo * gst_gl_upload_get_format    (GstGLUpload * upload);
+gboolean      gst_gl_upload_set_caps      (GstGLUpload * upload, GstCaps * in_caps, GstCaps * out_caps);
+void          gst_gl_upload_get_caps      (GstGLUpload * upload, GstCaps ** in_caps, GstCaps ** out_caps);
 
-gboolean gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer, guint * tex_id, GstBuffer ** outbuf);
+gboolean gst_gl_upload_perform_with_buffer (GstGLUpload * upload, GstBuffer * buffer, GstBuffer ** outbuf);
 void gst_gl_upload_release_buffer (GstGLUpload * upload);
-gboolean gst_gl_upload_perform_with_data          (GstGLUpload * upload, GLuint * texture_id,
-                                                   gpointer data[GST_VIDEO_MAX_PLANES], GstBuffer ** outbuf);
-
-gboolean gst_gl_upload_perform_with_gl_texture_upload_meta (GstGLUpload *upload, GstVideoGLTextureUploadMeta *meta, guint texture_id[4], GstBuffer ** outbuf);
 
 G_END_DECLS
 

tests/check/libs/gstglupload.c

@@ -226,18 +226,33 @@ draw_render (gpointer data)
 
 GST_START_TEST (test_upload_data)
 {
-  gpointer data[GST_VIDEO_MAX_PLANES] = { rgba_data, NULL, NULL, NULL };
-  GstVideoInfo in_info;
+  GstCaps *in_caps, *out_caps;
+  GstBuffer *inbuf, *outbuf;
+  GstMapInfo map_info;
   gboolean res;
   gint i = 0;
 
-  gst_video_info_set_format (&in_info, GST_VIDEO_FORMAT_RGBA, WIDTH, HEIGHT);
+  in_caps = gst_caps_from_string ("video/x-raw,format=RGBA,"
+      "width=10,height=10");
+  out_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),"
+      "format=RGBA,width=10,height=10");
 
-  gst_gl_upload_set_format (upload, &in_info);
+  gst_gl_upload_set_caps (upload, in_caps, out_caps);
 
-  res = gst_gl_upload_perform_with_data (upload, &tex_id, data, NULL);
+  inbuf = gst_buffer_new_wrapped_full (0, rgba_data, WIDTH * HEIGHT * 4,
+      0, WIDTH * HEIGHT * 4, NULL, NULL);
+
+  res = gst_gl_upload_perform_with_buffer (upload, inbuf, &outbuf);
   fail_if (res == FALSE, "Failed to upload buffer: %s\n",
       gst_gl_context_get_error ());
+  fail_unless (GST_IS_BUFFER (outbuf));
+
+  res = gst_buffer_map (outbuf, &map_info, GST_MAP_READ | GST_MAP_GL);
+  fail_if (res == FALSE, "Failed to map gl memory");
+
+  tex_id = *(guint *) map_info.data;
+
+  gst_buffer_unmap (outbuf, &map_info);
 
   gst_gl_window_draw (window, WIDTH, HEIGHT);
 
@@ -248,30 +263,48 @@ GST_START_TEST (test_upload_data)
         context);
     i++;
   }
+
+  gst_buffer_unref (inbuf);
+  gst_buffer_unref (outbuf);
 }
 
 GST_END_TEST;
 
 GST_START_TEST (test_upload_buffer)
 {
-  GstBuffer *buffer;
+  GstBuffer *buffer, *outbuf;
   GstGLMemory *gl_mem;
+  GstCaps *in_caps, *out_caps;
   GstVideoInfo in_info;
+  GstMapInfo map_info;
   gint i = 0;
   gboolean res;
 
-  gst_video_info_set_format (&in_info, GST_VIDEO_FORMAT_RGBA, WIDTH, HEIGHT);
+  in_caps = gst_caps_from_string ("video/x-raw,format=RGBA,width=10,height=10");
+  gst_video_info_from_caps (&in_info, in_caps);
 
   /* create GL buffer */
   buffer = gst_buffer_new ();
   gl_mem = gst_gl_memory_wrapped (context, &in_info, 0, rgba_data, NULL, NULL);
+
+  res =
+      gst_memory_map ((GstMemory *) gl_mem, &map_info,
+      GST_MAP_READ | GST_MAP_GL);
+  fail_if (res == FALSE, "Failed to map gl memory\n");
+  tex_id = *(guint *) map_info.data;
+  gst_memory_unmap ((GstMemory *) gl_mem, &map_info);
+
   gst_buffer_append_memory (buffer, (GstMemory *) gl_mem);
 
-  gst_gl_upload_set_format (upload, &in_info);
+  out_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),"
+      "format=RGBA,width=10,height=10");
 
-  res = gst_gl_upload_perform_with_buffer (upload, buffer, &tex_id, NULL);
+  gst_gl_upload_set_caps (upload, in_caps, out_caps);
+
+  res = gst_gl_upload_perform_with_buffer (upload, buffer, &outbuf);
   fail_if (res == FALSE, "Failed to upload buffer: %s\n",
       gst_gl_context_get_error ());
+  fail_unless (GST_IS_BUFFER (outbuf));
 
   gst_gl_window_draw (window, WIDTH, HEIGHT);
   gst_gl_window_send_message (window, GST_GL_WINDOW_CB (init), context);
@@ -284,6 +317,7 @@ GST_START_TEST (test_upload_buffer)
 
   gst_gl_upload_release_buffer (upload);
   gst_buffer_unref (buffer);
+  gst_buffer_unref (outbuf);
 }
 
 GST_END_TEST;
@@ -312,7 +346,6 @@ GST_START_TEST (test_upload_meta_producer)
   upload_meta = gst_gl_upload_meta_new (context);
   gst_gl_upload_meta_set_format (upload_meta, &in_info);
 
-  gst_gl_upload_set_format (upload, &in_info);
   gst_buffer_add_video_meta_full (buffer, 0, GST_VIDEO_FORMAT_RGBA, WIDTH,
       HEIGHT, 1, in_info.offset, in_info.stride);
   gst_gl_upload_meta_add_to_buffer (upload_meta, buffer);