glmemory: separate pbo transfer from texture transfers

When supported, the potentially longer pbo upload/download can be
initiated before the texture upload/download, potentially increasing
throughput.

ext/gl/gstgldownloadelement.c

@@ -161,8 +161,18 @@ static GstFlowReturn
 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
     GstBuffer * inbuf, GstBuffer ** outbuf)
 {
+  gint i, n;
+
   *outbuf = inbuf;
 
+  n = gst_buffer_n_memory (*outbuf);
+  for (i = 0; i < n; i++) {
+    GstMemory *mem = gst_buffer_peek_memory (*outbuf, i);
+
+    if (gst_is_gl_memory (mem))
+      gst_gl_memory_download_transfer ((GstGLMemory *) mem);
+  }
+
   return GST_FLOW_OK;
 }
 

gst-libs/gst/gl/gstglmemory.c

@@ -422,9 +422,8 @@ _upload_memory (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)
   gpointer data;
   gsize plane_start;
 
-  if (!GST_MEMORY_FLAG_IS_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD)) {
+  if ((gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_UPLOAD) == 0)
     return;
-  }
 
   gl = context->gl_vtable;
 
@@ -472,7 +471,7 @@ _upload_memory (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)
 
   gl->BindTexture (gl_target, 0);
 
-  GST_MEMORY_FLAG_UNSET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+  gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
 }
 
 static inline void
@@ -672,8 +671,7 @@ _gl_mem_new (GstAllocator * allocator, GstMemory * parent,
 }
 
 static gboolean
-_gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
+_gl_mem_read_pixels (GstGLMemory * gl_mem, gpointer read_pointer)
-    gsize size, gpointer read_pointer)
 {
   GstGLContext *context = gl_mem->mem.context;
   const GstGLFuncs *gl = context->gl_vtable;
@@ -685,6 +683,7 @@ _gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
   if (gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_RGB16)
     type = GL_UNSIGNED_SHORT_5_6_5;
 
+  /* FIXME: avoid creating a framebuffer every download/copy */
   gl->GenFramebuffers (1, &fbo);
   gl->BindFramebuffer (GL_FRAMEBUFFER, fbo);
 
@@ -708,43 +707,55 @@ _gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
   return TRUE;
 }
 
-static gpointer
+static gboolean
-_pbo_download_transfer (GstGLMemory * gl_mem, GstMapInfo * info, gsize size)
+_read_pixels_to_pbo (GstGLMemory * gl_mem)
 {
-  GstGLBaseBufferAllocatorClass *alloc_class;
+  const GstGLFuncs *gl = gl_mem->mem.context->gl_vtable;
-  const GstGLFuncs *gl;
-  gpointer data;
 
   if (!gl_mem->mem.id || !CONTEXT_SUPPORTS_PBO_DOWNLOAD (gl_mem->mem.context)
       || gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE
       || gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE_ALPHA)
     /* unsupported */
-    return NULL;
+    return FALSE;
+
+  if (gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
+    /* copy texture data into into the pbo and map that */
+    gsize plane_start = _find_plane_frame_start (gl_mem);
+
+    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, gl_mem->mem.id);
+
+    if (!_gl_mem_read_pixels (gl_mem, (gpointer) plane_start)) {
+      gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
+      return FALSE;
+    }
+
+    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
+  }
+
+  return TRUE;
+}
+
+static gpointer
+_pbo_download_transfer (GstGLMemory * gl_mem, GstMapInfo * info, gsize size)
+{
+  GstGLBaseBufferAllocatorClass *alloc_class;
+  gpointer data;
 
   GST_DEBUG ("downloading texture %u using pbo %u", gl_mem->tex_id,
       gl_mem->mem.id);
 
   alloc_class =
       GST_GL_BASE_BUFFER_ALLOCATOR_CLASS (gst_gl_allocator_parent_class);
-  gl = gl_mem->mem.context->gl_vtable;
 
-  if (GST_MEMORY_FLAG_IS_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD)
+  /* texture -> pbo */
-      && info->flags & GST_MAP_READ) {
+  if (info->flags & GST_MAP_READ)
-    /* copy texture data into into the pbo and map that */
+    if (!_read_pixels_to_pbo (gl_mem))
-    gsize plane_start = _find_plane_frame_start (gl_mem);
-
-    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, gl_mem->mem.id);
-
-    if (!_gl_mem_read_pixels (gl_mem, info, -1, (gpointer) plane_start)) {
-      gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
       return NULL;
-    }
-
-    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
-  }
 
   /* get a cpu accessible mapping from the pbo */
   gl_mem->mem.target = GL_PIXEL_PACK_BUFFER;
+  /* pbo -> data */
   data = alloc_class->map_buffer ((GstGLBaseBuffer *) gl_mem, info, size);
 
   return data;
@@ -756,7 +767,6 @@ _gl_mem_download_get_tex_image (GstGLMemory * gl_mem, GstMapInfo * info,
 {
   GstGLContext *context = gl_mem->mem.context;
   const GstGLFuncs *gl = context->gl_vtable;
-  guint format, type;
 
   if (size != -1 && size != ((GstMemory *) gl_mem)->maxsize)
     return NULL;
@@ -771,14 +781,19 @@ _gl_mem_download_get_tex_image (GstGLMemory * gl_mem, GstMapInfo * info,
 
   gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) gl_mem);
 
-  format = gst_gl_format_from_gl_texture_type (gl_mem->tex_type);
+  if (info->flags & GST_MAP_READ
-  type = GL_UNSIGNED_BYTE;
+      && gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
-  if (gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_RGB16)
+    guint format, type;
-    type = GL_UNSIGNED_SHORT_5_6_5;
 
-  gl->BindTexture (gl_mem->tex_target, gl_mem->tex_id);
+    format = gst_gl_format_from_gl_texture_type (gl_mem->tex_type);
-  gl->GetTexImage (gl_mem->tex_target, 0, format, type, gl_mem->mem.data);
+    type = GL_UNSIGNED_BYTE;
-  gl->BindTexture (gl_mem->tex_target, 0);
+    if (gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_RGB16)
+      type = GL_UNSIGNED_SHORT_5_6_5;
+
+    gl->BindTexture (gl_mem->tex_target, gl_mem->tex_id);
+    gl->GetTexImage (gl_mem->tex_target, 0, format, type, gl_mem->mem.data);
+    gl->BindTexture (gl_mem->tex_target, 0);
+  }
 
   return gl_mem->mem.data;
 }
@@ -792,8 +807,11 @@ _gl_mem_download_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
 
   gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) gl_mem);
 
-  if (!_gl_mem_read_pixels (gl_mem, info, size, gl_mem->mem.data))
+  if (info->flags & GST_MAP_READ
-    return NULL;
+      && gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
+    if (!_gl_mem_read_pixels (gl_mem, gl_mem->mem.data))
+      return NULL;
+  }
 
   return gl_mem->mem.data;
 }
@@ -834,15 +852,16 @@ _gl_mem_map_buffer (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)
 
     if ((info->flags & GST_MAP_WRITE) == GST_MAP_WRITE) {
       GST_TRACE ("mapping GL texture:%u for writing", gl_mem->tex_id);
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
     }
-    GST_MEMORY_FLAG_UNSET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
 
     data = &gl_mem->tex_id;
   } else {                      /* not GL */
     data = _gl_mem_map_cpu_access (gl_mem, info, maxsize);
     if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
   }
 
   return data;
@@ -873,10 +892,10 @@ _gl_mem_unmap_buffer (GstGLMemory * gl_mem, GstMapInfo * info)
   if ((info->flags & GST_MAP_GL) == 0) {
     _gl_mem_unmap_cpu_access (gl_mem, info);
     if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
   } else {
     if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
   }
 }
 
@@ -1057,6 +1076,7 @@ _gl_mem_copy (GstGLMemory * src, gssize offset, gssize size)
     memcpy (dest->mem.data, (guint8 *) src->mem.data + src->mem.mem.offset,
         src->mem.mem.size);
     GST_MINI_OBJECT_FLAG_SET (dest, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+    dest->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
     ret = (GstMemory *) dest;
   } else {
     GstAllocationParams params = { 0, src->mem.mem.align, 0, 0 };
@@ -1091,6 +1111,7 @@ _gl_mem_copy (GstGLMemory * src, gssize offset, gssize size)
     dest = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *)
         dest);
     GST_MINI_OBJECT_FLAG_SET (dest, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+    dest->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
     ret = (GstMemory *) dest;
   }
 
@@ -1250,6 +1271,7 @@ gst_gl_memory_wrapped_texture (GstGLContext * context,
 
   mem = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) mem);
   GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+  mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
 
   return mem;
 }
@@ -1305,17 +1327,68 @@ gst_gl_memory_wrapped (GstGLContext * context, GstVideoInfo * info,
 
   mem = _gl_mem_new (_gl_allocator, NULL, context, NULL, info, valign, plane,
       user_data, notify);
-  mem = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) mem);
   if (!mem)
     return NULL;
 
-  memcpy (mem->mem.data, data, ((GstMemory *) mem)->maxsize);
+  mem->mem.data = data;
 
   GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+  mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
 
   return mem;
 }
 
+static void
+_download_transfer (GstGLContext * context, GstGLMemory * gl_mem)
+{
+  GstGLBaseBuffer *mem = (GstGLBaseBuffer *) gl_mem;
+
+  g_mutex_lock (&mem->lock);
+  _read_pixels_to_pbo (gl_mem);
+  g_mutex_unlock (&mem->lock);
+}
+
+void
+gst_gl_memory_download_transfer (GstGLMemory * gl_mem)
+{
+  g_return_if_fail (gst_is_gl_memory ((GstMemory *) gl_mem));
+
+  gst_gl_context_thread_add (gl_mem->mem.context,
+      (GstGLContextThreadFunc) _download_transfer, gl_mem);
+}
+
+static void
+_upload_transfer (GstGLContext * context, GstGLMemory * gl_mem)
+{
+  GstGLBaseBufferAllocatorClass *alloc_class;
+  GstGLBaseBuffer *mem = (GstGLBaseBuffer *) gl_mem;
+  GstMapInfo info;
+
+  alloc_class =
+      GST_GL_BASE_BUFFER_ALLOCATOR_CLASS (gst_gl_allocator_parent_class);
+
+  info.flags = GST_MAP_READ | GST_MAP_GL;
+  info.memory = (GstMemory *) mem;
+  /* from gst_memory_map() */
+  info.size = mem->mem.size;
+  info.maxsize = mem->mem.maxsize - mem->mem.offset;
+
+  g_mutex_lock (&mem->lock);
+  mem->target = GL_PIXEL_UNPACK_BUFFER;
+  alloc_class->map_buffer (mem, &info, mem->mem.maxsize);
+  alloc_class->unmap_buffer (mem, &info);
+  g_mutex_unlock (&mem->lock);
+}
+
+void
+gst_gl_memory_upload_transfer (GstGLMemory * gl_mem)
+{
+  g_return_if_fail (gst_is_gl_memory ((GstMemory *) gl_mem));
+
+  gst_gl_context_thread_add (gl_mem->mem.context,
+      (GstGLContextThreadFunc) _upload_transfer, gl_mem);
+}
+
 gint
 gst_gl_memory_get_texture_width (GstGLMemory * gl_mem)
 {

gst-libs/gst/gl/gstglmemory.h

@@ -41,6 +41,15 @@ GType gst_gl_allocator_get_type(void);
 #define GST_GL_ALLOCATOR_CLASS(klass)         (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_GL_ALLOCATOR, GstGLAllocatorClass))
 #define GST_GL_ALLOCATOR_CAST(obj)            ((GstGLAllocator *)(obj))
 
+typedef enum _GstGLMemoryTransfer
+{
+  /* force a transfer between the texture and the PBO (if available) */
+  GST_GL_MEMORY_TRANSFER_NEED_UPLOAD = (1 << 0),
+  GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD = (1 << 1),
+} GstGLMemoryTransfer;
+
+#define GST_GL_MEMORY_ADD_TRANSFER(mem,state) ((GstGLMemory *)mem)->transfer_state |= state
+
 /**
  * GstGLMemory:
  * @mem: the parent object
@@ -68,6 +77,7 @@ struct _GstGLMemory
   gfloat                tex_scaling[2];
 
   /* <private> */
+  GstGLMemoryTransfer   transfer_state;
   gboolean              texture_wrapped;
   GDestroyNotify        notify;
   gpointer              user_data;
@@ -112,6 +122,9 @@ GstGLMemory * gst_gl_memory_wrapped_texture (GstGLContext * context,
                                              gpointer user_data,
                                              GDestroyNotify notify);
 
+void          gst_gl_memory_download_transfer (GstGLMemory * gl_mem);
+void          gst_gl_memory_upload_transfer   (GstGLMemory * gl_mem);
+
 gboolean      gst_gl_memory_copy_into_texture (GstGLMemory *gl_mem,
                                                guint tex_id,
                                                GstVideoGLTextureType tex_type,

gst-libs/gst/gl/gstglupload.c

@@ -269,6 +269,8 @@ _gl_memory_upload_perform (gpointer impl, GstBuffer * buffer,
     if (!gst_gl_context_can_share (upload->upload->context,
             gl_mem->mem.context))
       return GST_GL_UPLOAD_UNSHARED_GL_CONTEXT;
+
+    gst_gl_memory_upload_transfer (gl_mem);
   }
 
   *outbuf = gst_buffer_ref (buffer);

tests/check/libs/gstglmemory.c

@@ -182,6 +182,7 @@ GST_START_TEST (test_transfer)
           ((GstGLMemory *) mem)->tex_id, GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 1, 1,
           4, FALSE));
   GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+  GST_GL_MEMORY_ADD_TRANSFER (mem, GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD);
 
   fail_unless (!GST_MEMORY_FLAG_IS_SET (mem2,
           GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD));
@@ -249,6 +250,51 @@ GST_START_TEST (test_transfer)
 
 GST_END_TEST;
 
+GST_START_TEST (test_separate_transfer)
+{
+  GstAllocator *gl_allocator;
+  GstVideoInfo v_info;
+  GstMemory *mem;
+  GstMapInfo info;
+
+  gl_allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR);
+  fail_if (gl_allocator == NULL);
+
+  gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, 1, 1);
+
+  mem =
+      (GstMemory *) gst_gl_memory_wrapped (context, &v_info, 0, NULL,
+      rgba_pixel, NULL, NULL);
+  fail_if (mem == NULL);
+  fail_unless (!GST_MEMORY_FLAG_IS_SET (mem,
+          GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD));
+
+  gst_gl_memory_upload_transfer ((GstGLMemory *) mem);
+
+  fail_unless (!GST_MEMORY_FLAG_IS_SET (mem,
+          GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD));
+
+  fail_unless (gst_memory_map (mem, &info, GST_MAP_READ));
+
+  fail_unless (((gchar *) info.data)[0] == rgba_pixel[0]);
+  fail_unless (((gchar *) info.data)[1] == rgba_pixel[1]);
+  fail_unless (((gchar *) info.data)[2] == rgba_pixel[2]);
+  fail_unless (((gchar *) info.data)[3] == rgba_pixel[3]);
+
+  gst_memory_unmap (mem, &info);
+
+  /* FIXME: add download transfer */
+
+  if (gst_gl_context_get_error ())
+    printf ("%s\n", gst_gl_context_get_error ());
+  fail_if (gst_gl_context_get_error () != NULL);
+
+  gst_memory_unref (mem);
+  gst_object_unref (gl_allocator);
+}
+
+GST_END_TEST;
+
 static Suite *
 gst_gl_memory_suite (void)
 {
@@ -259,6 +305,7 @@ gst_gl_memory_suite (void)
   tcase_add_checked_fixture (tc_chain, setup, teardown);
   tcase_add_test (tc_chain, test_basic);
   tcase_add_test (tc_chain, test_transfer);
+  tcase_add_test (tc_chain, test_separate_transfer);
 
   return s;
 }