glmemory: separate pbo transfer from texture transfers

When supported, the potentially longer pbo upload/download can be initiated before the texture upload/download, potentially increasing throughput.
2024-11-23 18:21:04 +00:00 · 2015-06-11 18:26:50 +10:00 · 2015-06-11 18:26:50 +10:00 · 2210ba5d46
commit 2210ba5d46
parent 900bcea9de
5 changed files with 187 additions and 42 deletions
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@ -161,8 +161,18 @@ static GstFlowReturn
 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
    GstBuffer * inbuf, GstBuffer ** outbuf)
 {
+  gint i, n;
+
  *outbuf = inbuf;

+  n = gst_buffer_n_memory (*outbuf);
+  for (i = 0; i < n; i++) {
+    GstMemory *mem = gst_buffer_peek_memory (*outbuf, i);
+
+    if (gst_is_gl_memory (mem))
+      gst_gl_memory_download_transfer ((GstGLMemory *) mem);
+  }
+
  return GST_FLOW_OK;
 }

--- a/gst-libs/gst/gl/gstglmemory.c
+++ b/gst-libs/gst/gl/gstglmemory.c
@ -422,9 +422,8 @@ _upload_memory (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)
  gpointer data;
  gsize plane_start;

-  if (!GST_MEMORY_FLAG_IS_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD)) {
+  if ((gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_UPLOAD) == 0)
    return;
-  }

  gl = context->gl_vtable;

@ -472,7 +471,7 @@ _upload_memory (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)

  gl->BindTexture (gl_target, 0);

-  GST_MEMORY_FLAG_UNSET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+  gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
 }

 static inline void
@ -672,8 +671,7 @@ _gl_mem_new (GstAllocator * allocator, GstMemory * parent,
 }

 static gboolean
-_gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
-    gsize size, gpointer read_pointer)
+_gl_mem_read_pixels (GstGLMemory * gl_mem, gpointer read_pointer)
 {
  GstGLContext *context = gl_mem->mem.context;
  const GstGLFuncs *gl = context->gl_vtable;
@ -685,6 +683,7 @@ _gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
  if (gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_RGB16)
    type = GL_UNSIGNED_SHORT_5_6_5;

+  /* FIXME: avoid creating a framebuffer every download/copy */
  gl->GenFramebuffers (1, &fbo);
  gl->BindFramebuffer (GL_FRAMEBUFFER, fbo);

@ -708,43 +707,55 @@ _gl_mem_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,
  return TRUE;
 }

-static gpointer
-_pbo_download_transfer (GstGLMemory * gl_mem, GstMapInfo * info, gsize size)
+static gboolean
+_read_pixels_to_pbo (GstGLMemory * gl_mem)
 {
-  GstGLBaseBufferAllocatorClass *alloc_class;
-  const GstGLFuncs *gl;
-  gpointer data;
+  const GstGLFuncs *gl = gl_mem->mem.context->gl_vtable;

  if (!gl_mem->mem.id || !CONTEXT_SUPPORTS_PBO_DOWNLOAD (gl_mem->mem.context)
      || gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE
      || gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE_ALPHA)
    /* unsupported */
-    return NULL;
+    return FALSE;
+
+  if (gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
+    /* copy texture data into into the pbo and map that */
+    gsize plane_start = _find_plane_frame_start (gl_mem);
+
+    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, gl_mem->mem.id);
+
+    if (!_gl_mem_read_pixels (gl_mem, (gpointer) plane_start)) {
+      gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
+      return FALSE;
+    }
+
+    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
+  }
+
+  return TRUE;
+}
+
+static gpointer
+_pbo_download_transfer (GstGLMemory * gl_mem, GstMapInfo * info, gsize size)
+{
+  GstGLBaseBufferAllocatorClass *alloc_class;
+  gpointer data;

  GST_DEBUG ("downloading texture %u using pbo %u", gl_mem->tex_id,
      gl_mem->mem.id);

  alloc_class =
      GST_GL_BASE_BUFFER_ALLOCATOR_CLASS (gst_gl_allocator_parent_class);
-  gl = gl_mem->mem.context->gl_vtable;

-  if (GST_MEMORY_FLAG_IS_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD)
-      && info->flags & GST_MAP_READ) {
-    /* copy texture data into into the pbo and map that */
-    gsize plane_start = _find_plane_frame_start (gl_mem);
-
-    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, gl_mem->mem.id);
-
-    if (!_gl_mem_read_pixels (gl_mem, info, -1, (gpointer) plane_start)) {
-      gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
+  /* texture -> pbo */
+  if (info->flags & GST_MAP_READ)
+    if (!_read_pixels_to_pbo (gl_mem))
      return NULL;
-    }
-
-    gl->BindBuffer (GL_PIXEL_PACK_BUFFER, 0);
-  }

  /* get a cpu accessible mapping from the pbo */
  gl_mem->mem.target = GL_PIXEL_PACK_BUFFER;
+  /* pbo -> data */
  data = alloc_class->map_buffer ((GstGLBaseBuffer *) gl_mem, info, size);

  return data;
@ -756,7 +767,6 @@ _gl_mem_download_get_tex_image (GstGLMemory * gl_mem, GstMapInfo * info,
 {
  GstGLContext *context = gl_mem->mem.context;
  const GstGLFuncs *gl = context->gl_vtable;
-  guint format, type;

  if (size != -1 && size != ((GstMemory *) gl_mem)->maxsize)
    return NULL;
@ -771,6 +781,10 @@ _gl_mem_download_get_tex_image (GstGLMemory * gl_mem, GstMapInfo * info,

  gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) gl_mem);

+  if (info->flags & GST_MAP_READ
+      && gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
+    guint format, type;
+
    format = gst_gl_format_from_gl_texture_type (gl_mem->tex_type);
    type = GL_UNSIGNED_BYTE;
    if (gl_mem->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_RGB16)
@ -779,6 +793,7 @@ _gl_mem_download_get_tex_image (GstGLMemory * gl_mem, GstMapInfo * info,
    gl->BindTexture (gl_mem->tex_target, gl_mem->tex_id);
    gl->GetTexImage (gl_mem->tex_target, 0, format, type, gl_mem->mem.data);
    gl->BindTexture (gl_mem->tex_target, 0);
+  }

  return gl_mem->mem.data;
 }
@ -792,8 +807,11 @@ _gl_mem_download_read_pixels (GstGLMemory * gl_mem, GstMapInfo * info,

  gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) gl_mem);

-  if (!_gl_mem_read_pixels (gl_mem, info, size, gl_mem->mem.data))
+  if (info->flags & GST_MAP_READ
+      && gl_mem->transfer_state & GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD) {
+    if (!_gl_mem_read_pixels (gl_mem, gl_mem->mem.data))
      return NULL;
+  }

  return gl_mem->mem.data;
 }
@ -834,15 +852,16 @@ _gl_mem_map_buffer (GstGLMemory * gl_mem, GstMapInfo * info, gsize maxsize)

    if ((info->flags & GST_MAP_WRITE) == GST_MAP_WRITE) {
      GST_TRACE ("mapping GL texture:%u for writing", gl_mem->tex_id);
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
    }
-    GST_MEMORY_FLAG_UNSET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;

    data = &gl_mem->tex_id;
  } else {                      /* not GL */
    data = _gl_mem_map_cpu_access (gl_mem, info, maxsize);
    if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
+    gl_mem->transfer_state &= ~GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
  }

  return data;
@ -873,10 +892,10 @@ _gl_mem_unmap_buffer (GstGLMemory * gl_mem, GstMapInfo * info)
  if ((info->flags & GST_MAP_GL) == 0) {
    _gl_mem_unmap_cpu_access (gl_mem, info);
    if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
  } else {
    if (info->flags & GST_MAP_WRITE)
-      GST_MINI_OBJECT_FLAG_SET (gl_mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+      gl_mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
  }
 }

@ -1057,6 +1076,7 @@ _gl_mem_copy (GstGLMemory * src, gssize offset, gssize size)
    memcpy (dest->mem.data, (guint8 *) src->mem.data + src->mem.mem.offset,
        src->mem.mem.size);
    GST_MINI_OBJECT_FLAG_SET (dest, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+    dest->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;
    ret = (GstMemory *) dest;
  } else {
    GstAllocationParams params = { 0, src->mem.mem.align, 0, 0 };
@ -1091,6 +1111,7 @@ _gl_mem_copy (GstGLMemory * src, gssize offset, gssize size)
    dest = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *)
        dest);
    GST_MINI_OBJECT_FLAG_SET (dest, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+    dest->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;
    ret = (GstMemory *) dest;
  }

@ -1250,6 +1271,7 @@ gst_gl_memory_wrapped_texture (GstGLContext * context,

  mem = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) mem);
  GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+  mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD;

  return mem;
 }
@ -1305,17 +1327,68 @@ gst_gl_memory_wrapped (GstGLContext * context, GstVideoInfo * info,

  mem = _gl_mem_new (_gl_allocator, NULL, context, NULL, info, valign, plane,
      user_data, notify);
-  mem = (GstGLMemory *) gst_gl_base_buffer_alloc_data ((GstGLBaseBuffer *) mem);
  if (!mem)
    return NULL;

-  memcpy (mem->mem.data, data, ((GstMemory *) mem)->maxsize);
+  mem->mem.data = data;

  GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD);
+  mem->transfer_state |= GST_GL_MEMORY_TRANSFER_NEED_UPLOAD;

  return mem;
 }

+static void
+_download_transfer (GstGLContext * context, GstGLMemory * gl_mem)
+{
+  GstGLBaseBuffer *mem = (GstGLBaseBuffer *) gl_mem;
+
+  g_mutex_lock (&mem->lock);
+  _read_pixels_to_pbo (gl_mem);
+  g_mutex_unlock (&mem->lock);
+}
+
+void
+gst_gl_memory_download_transfer (GstGLMemory * gl_mem)
+{
+  g_return_if_fail (gst_is_gl_memory ((GstMemory *) gl_mem));
+
+  gst_gl_context_thread_add (gl_mem->mem.context,
+      (GstGLContextThreadFunc) _download_transfer, gl_mem);
+}
+
+static void
+_upload_transfer (GstGLContext * context, GstGLMemory * gl_mem)
+{
+  GstGLBaseBufferAllocatorClass *alloc_class;
+  GstGLBaseBuffer *mem = (GstGLBaseBuffer *) gl_mem;
+  GstMapInfo info;
+
+  alloc_class =
+      GST_GL_BASE_BUFFER_ALLOCATOR_CLASS (gst_gl_allocator_parent_class);
+
+  info.flags = GST_MAP_READ | GST_MAP_GL;
+  info.memory = (GstMemory *) mem;
+  /* from gst_memory_map() */
+  info.size = mem->mem.size;
+  info.maxsize = mem->mem.maxsize - mem->mem.offset;
+
+  g_mutex_lock (&mem->lock);
+  mem->target = GL_PIXEL_UNPACK_BUFFER;
+  alloc_class->map_buffer (mem, &info, mem->mem.maxsize);
+  alloc_class->unmap_buffer (mem, &info);
+  g_mutex_unlock (&mem->lock);
+}
+
+void
+gst_gl_memory_upload_transfer (GstGLMemory * gl_mem)
+{
+  g_return_if_fail (gst_is_gl_memory ((GstMemory *) gl_mem));
+
+  gst_gl_context_thread_add (gl_mem->mem.context,
+      (GstGLContextThreadFunc) _upload_transfer, gl_mem);
+}
+
 gint
 gst_gl_memory_get_texture_width (GstGLMemory * gl_mem)
 {
--- a/gst-libs/gst/gl/gstglmemory.h
+++ b/gst-libs/gst/gl/gstglmemory.h
@ -41,6 +41,15 @@ GType gst_gl_allocator_get_type(void);
 #define GST_GL_ALLOCATOR_CLASS(klass)         (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_GL_ALLOCATOR, GstGLAllocatorClass))
 #define GST_GL_ALLOCATOR_CAST(obj)            ((GstGLAllocator *)(obj))

+typedef enum _GstGLMemoryTransfer
+{
+  /* force a transfer between the texture and the PBO (if available) */
+  GST_GL_MEMORY_TRANSFER_NEED_UPLOAD = (1 << 0),
+  GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD = (1 << 1),
+} GstGLMemoryTransfer;
+
+#define GST_GL_MEMORY_ADD_TRANSFER(mem,state) ((GstGLMemory *)mem)->transfer_state |= state
+
 /**
 * GstGLMemory:
 * @mem: the parent object
@ -68,6 +77,7 @@ struct _GstGLMemory
  gfloat                tex_scaling[2];

  /* <private> */
+  GstGLMemoryTransfer   transfer_state;
  gboolean              texture_wrapped;
  GDestroyNotify        notify;
  gpointer              user_data;
@ -112,6 +122,9 @@ GstGLMemory * gst_gl_memory_wrapped_texture (GstGLContext * context,
                                             gpointer user_data,
                                             GDestroyNotify notify);

+void          gst_gl_memory_download_transfer (GstGLMemory * gl_mem);
+void          gst_gl_memory_upload_transfer   (GstGLMemory * gl_mem);
+
 gboolean      gst_gl_memory_copy_into_texture (GstGLMemory *gl_mem,
                                               guint tex_id,
                                               GstVideoGLTextureType tex_type,
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@ -269,6 +269,8 @@ _gl_memory_upload_perform (gpointer impl, GstBuffer * buffer,
    if (!gst_gl_context_can_share (upload->upload->context,
            gl_mem->mem.context))
      return GST_GL_UPLOAD_UNSHARED_GL_CONTEXT;
+
+    gst_gl_memory_upload_transfer (gl_mem);
  }

  *outbuf = gst_buffer_ref (buffer);
--- a/tests/check/libs/gstglmemory.c
+++ b/tests/check/libs/gstglmemory.c
@ -182,6 +182,7 @@ GST_START_TEST (test_transfer)
          ((GstGLMemory *) mem)->tex_id, GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 1, 1,
          4, FALSE));
  GST_MINI_OBJECT_FLAG_SET (mem, GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD);
+  GST_GL_MEMORY_ADD_TRANSFER (mem, GST_GL_MEMORY_TRANSFER_NEED_DOWNLOAD);

  fail_unless (!GST_MEMORY_FLAG_IS_SET (mem2,
          GST_GL_BASE_BUFFER_FLAG_NEED_UPLOAD));
@ -249,6 +250,51 @@ GST_START_TEST (test_transfer)

 GST_END_TEST;

+GST_START_TEST (test_separate_transfer)
+{
+  GstAllocator *gl_allocator;
+  GstVideoInfo v_info;
+  GstMemory *mem;
+  GstMapInfo info;
+
+  gl_allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR);
+  fail_if (gl_allocator == NULL);
+
+  gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, 1, 1);
+
+  mem =
+      (GstMemory *) gst_gl_memory_wrapped (context, &v_info, 0, NULL,
+      rgba_pixel, NULL, NULL);
+  fail_if (mem == NULL);
+  fail_unless (!GST_MEMORY_FLAG_IS_SET (mem,
+          GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD));
+
+  gst_gl_memory_upload_transfer ((GstGLMemory *) mem);
+
+  fail_unless (!GST_MEMORY_FLAG_IS_SET (mem,
+          GST_GL_BASE_BUFFER_FLAG_NEED_DOWNLOAD));
+
+  fail_unless (gst_memory_map (mem, &info, GST_MAP_READ));
+
+  fail_unless (((gchar *) info.data)[0] == rgba_pixel[0]);
+  fail_unless (((gchar *) info.data)[1] == rgba_pixel[1]);
+  fail_unless (((gchar *) info.data)[2] == rgba_pixel[2]);
+  fail_unless (((gchar *) info.data)[3] == rgba_pixel[3]);
+
+  gst_memory_unmap (mem, &info);
+
+  /* FIXME: add download transfer */
+
+  if (gst_gl_context_get_error ())
+    printf ("%s\n", gst_gl_context_get_error ());
+  fail_if (gst_gl_context_get_error () != NULL);
+
+  gst_memory_unref (mem);
+  gst_object_unref (gl_allocator);
+}
+
+GST_END_TEST;
+
 static Suite *
 gst_gl_memory_suite (void)
 {
@ -259,6 +305,7 @@ gst_gl_memory_suite (void)
  tcase_add_checked_fixture (tc_chain, setup, teardown);
  tcase_add_test (tc_chain, test_basic);
  tcase_add_test (tc_chain, test_transfer);
+  tcase_add_test (tc_chain, test_separate_transfer);

  return s;
 }