From 936b2fdfbc71dfda8c3f51caba84daa20d56732f Mon Sep 17 00:00:00 2001 From: Ilya Konstantinov Date: Thu, 2 Apr 2015 20:04:18 +0300 Subject: [PATCH] applemedia: implement GstAppleCoreVideoMemory Implement a new memory type wrapping CVPixelBuffer. There are two immediate advantages: a) Make the GstMemory itself retain the CVPixelBuffer. Previously, the containing GstBuffer was solely responsible for the lifetime of the backing CVPixelBuffer. With this change, we remove the GST_MEMORY_FLAG_NO_SHARE so that GstMemory objects be referenced by multiple GstBuffers (doing away with the need to copy.) b) Delay locking CVPixelBuffer into CPU memory until it's actually mapped -- possibly never. The CVPixelBuffer object is shared among references, shares and (in planar formats) planes, so a wrapper GstAppleCoreVideoPixelBuffer structure was introduced to manage locking. https://bugzilla.gnome.org/show_bug.cgi?id=747216 --- sys/applemedia/Makefile.am | 1 + sys/applemedia/coremediabuffer.c | 71 +---- sys/applemedia/corevideobuffer.c | 132 +++++---- sys/applemedia/corevideobuffer.h | 5 + sys/applemedia/corevideomemory.c | 465 +++++++++++++++++++++++++++++++ sys/applemedia/corevideomemory.h | 109 ++++++++ sys/applemedia/plugin.m | 3 + 7 files changed, 659 insertions(+), 127 deletions(-) create mode 100644 sys/applemedia/corevideomemory.c create mode 100644 sys/applemedia/corevideomemory.h diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am index c95b059d29..2502a9ea35 100644 --- a/sys/applemedia/Makefile.am +++ b/sys/applemedia/Makefile.am @@ -3,6 +3,7 @@ plugin_LTLIBRARIES = libgstapplemedia.la libgstapplemedia_la_SOURCES = \ plugin.m \ vtutil.c \ + corevideomemory.c \ corevideobuffer.c \ coremediabuffer.c \ videotexturecache.m \ diff --git a/sys/applemedia/coremediabuffer.c b/sys/applemedia/coremediabuffer.c index 681fe14f69..fa512fd5ed 100644 --- a/sys/applemedia/coremediabuffer.c +++ b/sys/applemedia/coremediabuffer.c @@ -19,13 +19,14 @@ * Boston, MA 02110-1301, USA. */ +#include "corevideobuffer.h" #include "coremediabuffer.h" +#include "corevideomemory.h" static void gst_core_media_meta_free (GstCoreMediaMeta * meta, GstBuffer * buf) { if (meta->image_buf != NULL) { - CVPixelBufferUnlockBaseAddress (meta->image_buf, 0); CVBufferRelease (meta->image_buf); } if (meta->block_buf != NULL) { @@ -84,68 +85,6 @@ gst_core_media_buffer_get_video_format (OSType format) } } -static gboolean -gst_core_media_buffer_wrap_pixel_buffer (GstBuffer * buf, GstVideoInfo * info, - CVPixelBufferRef pixel_buf, gboolean * has_padding, gboolean map) -{ - guint n_planes; - gsize offset[GST_VIDEO_MAX_PLANES] = { 0 }; - gint stride[GST_VIDEO_MAX_PLANES] = { 0 }; - GstVideoMeta *video_meta; - UInt32 size; - - if (map && CVPixelBufferLockBaseAddress (pixel_buf, 0) != kCVReturnSuccess) { - GST_ERROR ("Could not lock pixel buffer base address"); - return FALSE; - } - - *has_padding = FALSE; - - if (CVPixelBufferIsPlanar (pixel_buf)) { - gint i, size = 0, plane_offset = 0; - - n_planes = CVPixelBufferGetPlaneCount (pixel_buf); - for (i = 0; i < n_planes; i++) { - stride[i] = CVPixelBufferGetBytesPerRowOfPlane (pixel_buf, i); - - if (stride[i] != GST_VIDEO_INFO_PLANE_STRIDE (info, i)) { - *has_padding = TRUE; - } - - size = stride[i] * CVPixelBufferGetHeightOfPlane (pixel_buf, i); - offset[i] = plane_offset; - plane_offset += size; - - if (map) { - gst_buffer_append_memory (buf, - gst_memory_new_wrapped (GST_MEMORY_FLAG_NO_SHARE, - CVPixelBufferGetBaseAddressOfPlane (pixel_buf, i), size, 0, - size, NULL, NULL)); - } - } - } else { - - n_planes = 1; - stride[0] = CVPixelBufferGetBytesPerRow (pixel_buf); - offset[0] = 0; - size = stride[0] * CVPixelBufferGetHeight (pixel_buf); - - if (map) { - gst_buffer_append_memory (buf, - gst_memory_new_wrapped (GST_MEMORY_FLAG_NO_SHARE, - CVPixelBufferGetBaseAddress (pixel_buf), size, 0, size, NULL, - NULL)); - } - } - - video_meta = - gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE, - GST_VIDEO_INFO_FORMAT (info), info->width, info->height, n_planes, offset, - stride); - - return TRUE; -} - static gboolean gst_core_media_buffer_wrap_block_buffer (GstBuffer * buf, CMBlockBufferRef block_buf) @@ -280,10 +219,8 @@ gst_core_media_buffer_new (CMSampleBufferRef sample_buf, goto error; } - if (!gst_core_media_buffer_wrap_pixel_buffer (buf, &info, meta->pixel_buf, - &has_padding, map)) { - goto error; - } + gst_core_video_wrap_pixel_buffer (buf, &info, meta->pixel_buf, &has_padding, + map); /* If the video meta API is not supported, remove padding by * copying the core media buffer to a system memory buffer */ diff --git a/sys/applemedia/corevideobuffer.c b/sys/applemedia/corevideobuffer.c index 269e19d0e7..5979cb3b07 100644 --- a/sys/applemedia/corevideobuffer.c +++ b/sys/applemedia/corevideobuffer.c @@ -18,14 +18,11 @@ */ #include "corevideobuffer.h" +#include "corevideomemory.h" static void gst_core_video_meta_free (GstCoreVideoMeta * meta, GstBuffer * buf) { - if (meta->pixbuf != NULL) { - CVPixelBufferUnlockBaseAddress (meta->pixbuf, 0); - } - CVBufferRelease (meta->cvbuf); } @@ -58,6 +55,74 @@ gst_core_video_meta_get_info (void) return core_video_meta_info; } +void +gst_core_video_wrap_pixel_buffer (GstBuffer * buf, GstVideoInfo * info, + CVPixelBufferRef pixel_buf, gboolean * has_padding, gboolean map) +{ + guint n_planes; + gsize offset[GST_VIDEO_MAX_PLANES] = { 0 }; + gint stride[GST_VIDEO_MAX_PLANES] = { 0 }; + UInt32 size; + + *has_padding = FALSE; + + if (CVPixelBufferIsPlanar (pixel_buf)) { + gint i, size = 0, plane_offset = 0; + GstAppleCoreVideoPixelBuffer *gpixbuf; + + if (map) { + gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf); + } + + n_planes = CVPixelBufferGetPlaneCount (pixel_buf); + for (i = 0; i < n_planes; i++) { + stride[i] = CVPixelBufferGetBytesPerRowOfPlane (pixel_buf, i); + + if (stride[i] != GST_VIDEO_INFO_PLANE_STRIDE (info, i)) { + *has_padding = TRUE; + } + + size = stride[i] * CVPixelBufferGetHeightOfPlane (pixel_buf, i); + offset[i] = plane_offset; + plane_offset += size; + + if (map) { + gst_buffer_append_memory (buf, + gst_apple_core_video_memory_new_wrapped (gpixbuf, i, size)); + } + } + + if (map) { + gst_apple_core_video_pixel_buffer_unref (gpixbuf); + } + } else { + + n_planes = 1; + stride[0] = CVPixelBufferGetBytesPerRow (pixel_buf); + offset[0] = 0; + size = stride[0] * CVPixelBufferGetHeight (pixel_buf); + + if (map) { + GstAppleCoreVideoPixelBuffer *gpixbuf; + + gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf); + gst_buffer_append_memory (buf, + gst_apple_core_video_memory_new_wrapped (gpixbuf, + GST_APPLE_CORE_VIDEO_NO_PLANE, size)); + gst_apple_core_video_pixel_buffer_unref (gpixbuf); + } + } + + if (info) { + GstVideoMeta *video_meta; + + video_meta = + gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE, + GST_VIDEO_INFO_FORMAT (info), info->width, info->height, n_planes, + offset, stride); + } +} + GstBuffer * gst_core_video_buffer_new (CVBufferRef cvbuf, GstVideoInfo * vinfo, gboolean map) @@ -65,20 +130,14 @@ gst_core_video_buffer_new (CVBufferRef cvbuf, GstVideoInfo * vinfo, CVPixelBufferRef pixbuf = NULL; GstBuffer *buf; GstCoreVideoMeta *meta; - guint n_planes; - gsize offset[GST_VIDEO_MAX_PLANES]; - gint stride[GST_VIDEO_MAX_PLANES]; + gboolean has_padding; /* not used for now */ if (CFGetTypeID (cvbuf) != CVPixelBufferGetTypeID ()) /* TODO: Do we need to handle other buffer types? */ - goto error; + return NULL; pixbuf = (CVPixelBufferRef) cvbuf; - if (map && CVPixelBufferLockBaseAddress (pixbuf, 0) != kCVReturnSuccess) { - goto error; - } - buf = gst_buffer_new (); /* add the corevideo meta to free the underlying corevideo buffer */ @@ -87,54 +146,7 @@ gst_core_video_buffer_new (CVBufferRef cvbuf, GstVideoInfo * vinfo, meta->cvbuf = CVBufferRetain (cvbuf); meta->pixbuf = pixbuf; - /* set stride, offset and size */ - memset (&offset, 0, sizeof (offset)); - memset (&stride, 0, sizeof (stride)); - - if (CVPixelBufferIsPlanar (pixbuf)) { - int i, size, off; - - n_planes = CVPixelBufferGetPlaneCount (pixbuf); - off = 0; - for (i = 0; i < n_planes; ++i) { - stride[i] = CVPixelBufferGetBytesPerRowOfPlane (pixbuf, i); - size = stride[i] * CVPixelBufferGetHeightOfPlane (pixbuf, i); - offset[i] = off; - off += size; - - if (map) { - gst_buffer_append_memory (buf, - gst_memory_new_wrapped (GST_MEMORY_FLAG_NO_SHARE, - CVPixelBufferGetBaseAddressOfPlane (pixbuf, i), size, 0, size, - NULL, NULL)); - } - } - } else { - int size; - - n_planes = 1; - stride[0] = CVPixelBufferGetBytesPerRow (pixbuf); - offset[0] = 0; - size = stride[0] * CVPixelBufferGetHeight (pixbuf); - - if (map) { - gst_buffer_append_memory (buf, - gst_memory_new_wrapped (GST_MEMORY_FLAG_NO_SHARE, - CVPixelBufferGetBaseAddress (pixbuf), size, 0, size, NULL, NULL)); - } - } - - if (vinfo) { - GstVideoMeta *video_meta; - - video_meta = - gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE, - vinfo->finfo->format, CVPixelBufferGetWidth (pixbuf), - CVPixelBufferGetHeight (pixbuf), n_planes, offset, stride); - } + gst_core_video_wrap_pixel_buffer (buf, vinfo, pixbuf, &has_padding, map); return buf; - -error: - return NULL; } diff --git a/sys/applemedia/corevideobuffer.h b/sys/applemedia/corevideobuffer.h index 8f58a6e48c..445cef3911 100644 --- a/sys/applemedia/corevideobuffer.h +++ b/sys/applemedia/corevideobuffer.h @@ -43,6 +43,11 @@ typedef struct _GstCoreVideoMeta GstBuffer * gst_core_video_buffer_new (CVBufferRef cvbuf, GstVideoInfo *info, gboolean map); +void gst_core_video_wrap_pixel_buffer (GstBuffer * buf, + GstVideoInfo * info, + CVPixelBufferRef pixel_buf, + gboolean * has_padding, + gboolean map); GType gst_core_video_meta_api_get_type (void); G_END_DECLS diff --git a/sys/applemedia/corevideomemory.c b/sys/applemedia/corevideomemory.c new file mode 100644 index 0000000000..4cb085d9b2 --- /dev/null +++ b/sys/applemedia/corevideomemory.c @@ -0,0 +1,465 @@ +/* GStreamer Apple Core Video memory + * Copyright (C) 2015 Ilya Konstantinov + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for mordetails. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "corevideomemory.h" + +GST_DEBUG_CATEGORY_STATIC (GST_CAT_APPLE_CORE_VIDEO_MEMORY); +#define GST_CAT_DEFAULT GST_CAT_APPLE_CORE_VIDEO_MEMORY + +static const char *_lock_state_names[] = { + "Unlocked", "Locked Read-Only", "Locked Read-Write" +}; + +/** + * gst_apple_core_video_pixel_buffer_new: + * @buf: an unlocked CVPixelBuffer + * + * Initializes a wrapper to manage locking state for a CVPixelBuffer. + * This function expects to receive unlocked CVPixelBuffer, and further assumes + * that no one else will lock it (as long as the wrapper exists). + * + * This function retains @buf. + * + * Returns: The wrapped @buf. + */ +GstAppleCoreVideoPixelBuffer * +gst_apple_core_video_pixel_buffer_new (CVPixelBufferRef buf) +{ + GstAppleCoreVideoPixelBuffer *gpixbuf = + g_slice_new (GstAppleCoreVideoPixelBuffer); + gpixbuf->refcount = 1; + g_mutex_init (&gpixbuf->mutex); + gpixbuf->buf = CVPixelBufferRetain (buf); + gpixbuf->lock_state = GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED; + gpixbuf->lock_count = 0; + return gpixbuf; +} + +GstAppleCoreVideoPixelBuffer * +gst_apple_core_video_pixel_buffer_ref (GstAppleCoreVideoPixelBuffer * gpixbuf) +{ + g_atomic_int_inc (&gpixbuf->refcount); + return gpixbuf; +} + +void +gst_apple_core_video_pixel_buffer_unref (GstAppleCoreVideoPixelBuffer * gpixbuf) +{ + if (g_atomic_int_dec_and_test (&gpixbuf->refcount)) { + if (gpixbuf->lock_state != GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED) { + GST_ERROR + ("%p: CVPixelBuffer memory still locked (lock_count = %d), likely forgot to unmap GstAppleCoreVideoMemory", + gpixbuf, gpixbuf->lock_count); + } + CVPixelBufferRelease (gpixbuf->buf); + g_mutex_clear (&gpixbuf->mutex); + g_slice_free (GstAppleCoreVideoPixelBuffer, gpixbuf); + } +} + +/** + * gst_apple_core_video_pixel_buffer_lock: + * @gpixbuf: the wrapped CVPixelBuffer + * @flags: mapping flags for either read-only or read-write locking + * + * Locks the pixel buffer into CPU memory for reading only, or + * reading and writing. The desired lock mode is deduced from @flags. + * + * For planar buffers, each plane's #GstAppleCoreVideoMemory will reference + * the same #GstAppleCoreVideoPixelBuffer; therefore this function will be + * called multiple times for the same @gpixbuf. Each call to this function + * should be matched by a call to gst_apple_core_video_pixel_buffer_unlock(). + * + * Notes: + * + * - Read-only locking improves performance by preventing Core Video + * from invalidating existing caches of the buffer’s contents. + * + * - Only the first call actually locks; subsequent calls succeed + * as long as their requested flags are compatible with how the buffer + * is already locked. + * + * For example, the following code will succeed: + * |[ + * gst_memory_map(plane1, GST_MAP_READWRITE); + * gst_memory_map(plane2, GST_MAP_READ); + * ]| + * while the ƒollowing code will fail: + * |[ + * gst_memory_map(plane1, GST_MAP_READ); + * gst_memory_map(plane2, GST_MAP_READWRITE); /* ERROR: already locked for read-only */ + * ]| + * + * Returns: %TRUE if the buffer was locked as requested + */ +static gboolean +gst_apple_core_video_pixel_buffer_lock (GstAppleCoreVideoPixelBuffer * gpixbuf, + GstMapFlags flags) +{ + CVReturn cvret; + CVOptionFlags lockFlags; + + g_mutex_lock (&gpixbuf->mutex); + + switch (gpixbuf->lock_state) { + case GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED: + lockFlags = (flags & GST_MAP_WRITE) ? 0 : kCVPixelBufferLock_ReadOnly; + cvret = CVPixelBufferLockBaseAddress (gpixbuf->buf, lockFlags); + if (cvret != kCVReturnSuccess) { + g_mutex_unlock (&gpixbuf->mutex); + /* TODO: Map kCVReturnError etc. into strings */ + GST_ERROR ("%p: unable to lock base address for pixbuf %p: %d", gpixbuf, + gpixbuf->buf, cvret); + return FALSE; + } + gpixbuf->lock_state = + (flags & GST_MAP_WRITE) ? + GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READ_WRITE : + GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READONLY; + break; + + case GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READONLY: + if (flags & GST_MAP_WRITE) { + g_mutex_unlock (&gpixbuf->mutex); + GST_ERROR ("%p: pixel buffer %p already locked for read-only access", + gpixbuf, gpixbuf->buf); + return FALSE; + } + break; + + case GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READ_WRITE: + break; /* nothing to do, already most permissive mapping */ + } + + g_atomic_int_inc (&gpixbuf->lock_count); + + g_mutex_unlock (&gpixbuf->mutex); + + GST_DEBUG ("%p: pixbuf %p, %s (%d times)", + gpixbuf, + gpixbuf->buf, + _lock_state_names[gpixbuf->lock_state], gpixbuf->lock_count); + + return TRUE; +} + +/** + * gst_apple_core_video_pixel_buffer_unlock: + * @gpixbuf: the wrapped CVPixelBuffer + * + * Unlocks the pixel buffer from CPU memory. Should be called + * for every gst_apple_core_video_pixel_buffer_lock() call. + */ +static gboolean +gst_apple_core_video_pixel_buffer_unlock (GstAppleCoreVideoPixelBuffer * + gpixbuf) +{ + CVOptionFlags lockFlags; + CVReturn cvret; + + if (gpixbuf->lock_state == GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED) { + GST_ERROR ("%p: pixel buffer %p not locked", gpixbuf, gpixbuf->buf); + return FALSE; + } + + if (!g_atomic_int_dec_and_test (&gpixbuf->lock_count)) { + return TRUE; /* still locked, by current and/or other callers */ + } + + g_mutex_lock (&gpixbuf->mutex); + + lockFlags = + (gpixbuf->lock_state == + GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READONLY) ? kCVPixelBufferLock_ReadOnly + : 0; + cvret = CVPixelBufferUnlockBaseAddress (gpixbuf->buf, lockFlags); + if (cvret != kCVReturnSuccess) { + g_mutex_unlock (&gpixbuf->mutex); + g_atomic_int_inc (&gpixbuf->lock_count); + /* TODO: Map kCVReturnError etc. into strings */ + GST_ERROR ("%p: unable to unlock base address for pixbuf %p: %d", gpixbuf, + gpixbuf->buf, cvret); + return FALSE; + } + + gpixbuf->lock_state = GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED; + + g_mutex_unlock (&gpixbuf->mutex); + + GST_DEBUG ("%p: pixbuf %p, %s (%d locks remaining)", + gpixbuf, + gpixbuf->buf, + _lock_state_names[gpixbuf->lock_state], gpixbuf->lock_count); + + return TRUE; +} + +/* + * GstAppleCoreVideoAllocator + */ + +struct _GstAppleCoreVideoAllocatorClass +{ + GstAllocatorClass parent_class; +}; + +typedef struct _GstAppleCoreVideoAllocatorClass GstAppleCoreVideoAllocatorClass; + +struct _GstAppleCoreVideoAllocator +{ + GstAllocator parent_instance; +}; + +typedef struct _GstAppleCoreVideoAllocator GstAppleCoreVideoAllocator; + +/* GType for GstAppleCoreVideoAllocator */ +GType gst_apple_core_video_allocator_get_type (void); +#define GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR (gst_apple_core_video_allocator_get_type()) +#define GST_IS_APPLE_CORE_VIDEO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR)) +#define GST_IS_APPLE_CORE_VIDEO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR)) +#define GST_APPLE_CORE_VIDEO_ALLOCATOR_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR, GstAppleCoreVideoAllocatorClass)) +#define GST_APPLE_CORE_VIDEO_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR, GstAppleCoreVideoAllocator)) +#define GST_APPLE_CORE_VIDEO_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR, GstAppleCoreVideoAllocatorClass)) + +G_DEFINE_TYPE (GstAppleCoreVideoAllocator, gst_apple_core_video_allocator, + GST_TYPE_ALLOCATOR); + +/* Name for allocator registration */ +#define GST_APPLE_CORE_VIDEO_ALLOCATOR_NAME "AppleCoreVideoMemory" + +/* Singleton instance of GstAppleCoreVideoAllocator */ +static GstAppleCoreVideoAllocator *_apple_core_video_allocator; + +/** + * gst_apple_core_video_memory_init: + * + * Initializes the Core Video Memory allocator. This function must be called + * before #GstAppleCoreVideoMemory can be created. + * + * It is safe to call this function multiple times. + */ +void +gst_apple_core_video_memory_init (void) +{ + static volatile gsize _init = 0; + + if (g_once_init_enter (&_init)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_APPLE_CORE_VIDEO_MEMORY, "corevideomemory", + 0, "Apple Core Video Memory"); + + _apple_core_video_allocator = + g_object_new (GST_TYPE_APPLE_CORE_VIDEO_ALLOCATOR, NULL); + + gst_allocator_register (GST_APPLE_CORE_VIDEO_ALLOCATOR_NAME, + gst_object_ref (_apple_core_video_allocator)); + g_once_init_leave (&_init, 1); + } +} + +/** + * gst_is_apple_core_video_memory: + * @mem: #GstMemory + * + * Checks whether @mem is backed by a CVPixelBuffer. + * This has limited use since #GstAppleCoreVideoMemory is transparently + * mapped into CPU memory on request. + * + * Returns: %TRUE when @mem is backed by a CVPixelBuffer + */ +gboolean +gst_is_apple_core_video_memory (GstMemory * mem) +{ + g_return_val_if_fail (mem != NULL, FALSE); + + return GST_IS_APPLE_CORE_VIDEO_ALLOCATOR (mem->allocator); +} + +/** + * gst_apple_core_video_memory_new: + * + * Helper function for gst_apple_core_video_mem_share(). + * Users should call gst_apple_core_video_memory_new_wrapped() instead. + */ +static GstMemory * +gst_apple_core_video_memory_new (GstMemoryFlags flags, GstMemory * parent, + GstAppleCoreVideoPixelBuffer * gpixbuf, gsize plane, gsize maxsize, + gsize align, gsize offset, gsize size) +{ + GstAppleCoreVideoMemory *mem; + + g_return_val_if_fail (gpixbuf != NULL, NULL); + + mem = g_slice_new0 (GstAppleCoreVideoMemory); + gst_memory_init (GST_MEMORY_CAST (mem), flags, + GST_ALLOCATOR_CAST (_apple_core_video_allocator), parent, maxsize, align, + offset, size); + + mem->gpixbuf = gst_apple_core_video_pixel_buffer_ref (gpixbuf); + mem->plane = plane; + + GST_DEBUG ("%p: gpixbuf %p, plane: %" G_GSSIZE_FORMAT ", size %" + G_GSIZE_FORMAT, mem, mem->gpixbuf, mem->plane, mem->mem.size); + + return (GstMemory *) mem; +} + +/** + * gst_apple_core_video_memory_new_wrapped: + * @gpixbuf: the backing #GstAppleCoreVideoPixelBuffer + * @plane: the plane this memory will represent, or #GST_APPLE_CORE_VIDEO_NO_PLANE for non-planar buffer + * @size: the size of the buffer or specific plane + * + * Returns: a newly allocated #GstAppleCoreVideoMemory + */ +GstMemory * +gst_apple_core_video_memory_new_wrapped (GstAppleCoreVideoPixelBuffer * gpixbuf, + gsize plane, gsize size) +{ + return gst_apple_core_video_memory_new (0, NULL, gpixbuf, plane, size, 0, 0, + size); +} + +static gpointer +gst_apple_core_video_mem_map (GstMemory * gmem, gsize maxsize, + GstMapFlags flags) +{ + GstAppleCoreVideoMemory *mem = (GstAppleCoreVideoMemory *) gmem; + gpointer ret; + + if (!gst_apple_core_video_pixel_buffer_lock (mem->gpixbuf, flags)) + return NULL; + + if (mem->plane != GST_APPLE_CORE_VIDEO_NO_PLANE) { + ret = CVPixelBufferGetBaseAddressOfPlane (mem->gpixbuf->buf, mem->plane); + + if (ret != NULL) + GST_DEBUG ("%p: pixbuf %p plane %" G_GSIZE_FORMAT + " flags %08x: mapped %p", mem, mem->gpixbuf->buf, mem->plane, flags, + ret); + else + GST_ERROR ("%p: invalid plane base address (NULL) for pixbuf %p plane %" + G_GSIZE_FORMAT, mem, mem->gpixbuf->buf, mem->plane); + } else { + ret = CVPixelBufferGetBaseAddress (mem->gpixbuf->buf); + + if (ret != NULL) + GST_DEBUG ("%p: pixbuf %p flags %08x: mapped %p", mem, mem->gpixbuf->buf, + flags, ret); + else + GST_ERROR ("%p: invalid base address (NULL) for pixbuf %p" + G_GSIZE_FORMAT, mem, mem->gpixbuf->buf); + } + + return ret; +} + +static void +gst_apple_core_video_mem_unmap (GstMemory * gmem) +{ + GstAppleCoreVideoMemory *mem = (GstAppleCoreVideoMemory *) gmem; + (void) gst_apple_core_video_pixel_buffer_unlock (mem->gpixbuf); + if (mem->plane != GST_APPLE_CORE_VIDEO_NO_PLANE) + GST_DEBUG ("%p: pixbuf %p plane %" G_GSIZE_FORMAT, mem, + mem->gpixbuf->buf, mem->plane); + else + GST_DEBUG ("%p: pixbuf %p", mem, mem->gpixbuf->buf); +} + +static GstMemory * +gst_apple_core_video_mem_share (GstMemory * gmem, gssize offset, gssize size) +{ + GstAppleCoreVideoMemory *mem; + GstMemory *parent, *sub; + + mem = (GstAppleCoreVideoMemory *) gmem; + + /* find the real parent */ + parent = gmem->parent; + if (parent == NULL) + parent = gmem; + + if (size == -1) + size = gmem->size - offset; + + /* the shared memory is always readonly */ + sub = + gst_apple_core_video_memory_new (GST_MINI_OBJECT_FLAGS (parent) | + GST_MINI_OBJECT_FLAG_LOCK_READONLY, parent, mem->gpixbuf, mem->plane, + gmem->maxsize, gmem->align, gmem->offset + offset, size); + + return sub; +} + +static gboolean +gst_apple_core_video_mem_is_span (GstMemory * mem1, GstMemory * mem2, + gsize * offset) +{ + /* We may only return FALSE since: + * 1) Core Video gives no guarantees about planes being consecutive. + * We may only know this after mapping. + * 2) GstAppleCoreVideoMemory instances for planes do not share a common + * parent -- i.e. they're not offsets into the same parent + * memory instance. + * + * It's not unlikely that planes will be stored in consecutive memory + * but it should be checked by the user after mapping. + */ + return FALSE; +} + +static void +gst_apple_core_video_mem_free (GstAllocator * allocator, GstMemory * gmem) +{ + GstAppleCoreVideoMemory *mem = (GstAppleCoreVideoMemory *) gmem; + + gst_apple_core_video_pixel_buffer_unref (mem->gpixbuf); + + g_slice_free (GstAppleCoreVideoMemory, mem); +} + +static void +gst_apple_core_video_allocator_class_init (GstAppleCoreVideoAllocatorClass * + klass) +{ + GstAllocatorClass *allocator_class; + + allocator_class = (GstAllocatorClass *) klass; + + /* we don't do allocations, only wrap existing pixel buffers */ + allocator_class->alloc = NULL; + allocator_class->free = gst_apple_core_video_mem_free; +} + +static void +gst_apple_core_video_allocator_init (GstAppleCoreVideoAllocator * allocator) +{ + GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator); + + alloc->mem_type = GST_APPLE_CORE_VIDEO_ALLOCATOR_NAME; + alloc->mem_map = gst_apple_core_video_mem_map; + alloc->mem_unmap = gst_apple_core_video_mem_unmap; + alloc->mem_share = gst_apple_core_video_mem_share; + alloc->mem_is_span = gst_apple_core_video_mem_is_span; + + GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC); +} diff --git a/sys/applemedia/corevideomemory.h b/sys/applemedia/corevideomemory.h new file mode 100644 index 0000000000..d81781c239 --- /dev/null +++ b/sys/applemedia/corevideomemory.h @@ -0,0 +1,109 @@ +/* GStreamer Apple Core Video memory + * Copyright (C) 2015 Ilya Konstantinov + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef __GST_APPLE_CORE_VIDEO_MEMORY_H__ +#define __GST_APPLE_CORE_VIDEO_MEMORY_H__ + +#include + +#include "CoreVideo/CoreVideo.h" + +G_BEGIN_DECLS + +/** + * GstAppleCoreVideoLockState: + * + * Specifies whether the backing CVPixelBuffer is locked for read-only + * or read-write. + * + * Locking for reading only improves performance by preventing + * Core Video from invalidating existing caches of the buffer’s contents. + */ +typedef enum +{ + GST_APPLE_CORE_VIDEO_MEMORY_UNLOCKED, + GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READONLY, + GST_APPLE_CORE_VIDEO_MEMORY_LOCKED_READ_WRITE +} GstAppleCoreVideoLockState; + +/** + * GstAppleCoreVideoPixelBuffer: + * + * This structure wraps CVPixelBuffer, managing its lock states and reference count. + * It will be referenced by one or more #GstAppleCoreVideoMemory. + */ +typedef struct +{ + guint refcount; + GMutex mutex; + CVPixelBufferRef buf; + /* Allows mem_map to refuse Read-Write locking a buffer that was previously + * locked for Read-Only. */ + GstAppleCoreVideoLockState lock_state; + /* Counts the number of times the buffer was locked. + * Only the first lock affects whether it's just for reading + * or for reading and writing, as reflected in @lock_state. */ + guint lock_count; +} GstAppleCoreVideoPixelBuffer; + +/** + * GST_APPLE_CORE_VIDEO_NO_PLANE: + * + * Indicates a non-planar pixel buffer. + */ +#define GST_APPLE_CORE_VIDEO_NO_PLANE ((size_t)-1) + +/** + * GstAppleCoreVideoMemory: + * + * Represents a video plane or an entire (non-planar) video image, + * backed by a CVPixelBuffer. + * + * This structure shares a #GstAppleCoreVideoPixelBuffer instance + * with other instances. + */ +typedef struct +{ + GstMemory mem; + + GstAppleCoreVideoPixelBuffer *gpixbuf; + size_t plane; +} GstAppleCoreVideoMemory; + +void +gst_apple_core_video_memory_init (void); + +GstAppleCoreVideoPixelBuffer * +gst_apple_core_video_pixel_buffer_new (CVPixelBufferRef pixbuf); + +GstAppleCoreVideoPixelBuffer * +gst_apple_core_video_pixel_buffer_ref (GstAppleCoreVideoPixelBuffer * shared); + +void +gst_apple_core_video_pixel_buffer_unref (GstAppleCoreVideoPixelBuffer * shared); + +gboolean +gst_is_apple_core_video_memory (GstMemory * mem); + +GstMemory * +gst_apple_core_video_memory_new_wrapped (GstAppleCoreVideoPixelBuffer * shared, gsize plane, gsize size); + +G_END_DECLS + +#endif /* __GST_APPLE_CORE_VIDEO_MEMORY_H__ */ diff --git a/sys/applemedia/plugin.m b/sys/applemedia/plugin.m index c17c38b60e..0ff4ec1a5f 100644 --- a/sys/applemedia/plugin.m +++ b/sys/applemedia/plugin.m @@ -22,6 +22,7 @@ #endif #include +#include "corevideomemory.h" #ifdef HAVE_IOS #include "iosassetsrc.h" #else @@ -63,6 +64,8 @@ plugin_init (GstPlugin * plugin) { gboolean res = TRUE; + gst_apple_core_video_memory_init (); + #ifdef HAVE_IOS res &= gst_element_register (plugin, "iosassetsrc", GST_RANK_SECONDARY, GST_TYPE_IOS_ASSET_SRC);