From c98f92ee03ef07fb2cc692f3201045e479a00d69 Mon Sep 17 00:00:00 2001 From: Matt Fischer Date: Tue, 10 Jan 2017 19:23:58 -0600 Subject: [PATCH] gldownload: Add dmabuf exporting This patch adds code to gldownload to export the image as a dmabuf if requested. The element now exposes memory:DMABuf as a cap feature, and if it is selected, the element exports the texture to an EGL image and then a dmabuf. It also implements a fallback to system memory download in case the exportation failed. https://bugzilla.gnome.org/show_bug.cgi?id=776927 --- ext/gl/gstgldownloadelement.c | 275 +++++++++++++++++++++++++++++- ext/gl/gstgldownloadelement.h | 2 + gst-libs/gst/gl/egl/gsteglimage.c | 59 +++++++ gst-libs/gst/gl/egl/gsteglimage.h | 2 + 4 files changed, 333 insertions(+), 5 deletions(-) diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c index 153a3c3f90..3f65bf3ef9 100644 --- a/ext/gl/gstgldownloadelement.c +++ b/ext/gl/gstgldownloadelement.c @@ -23,6 +23,11 @@ #endif #include +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF +#include +#include +#endif + #include "gstgldownloadelement.h" GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug); @@ -45,12 +50,19 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt, GstBuffer * buffer, GstBuffer ** outbuf); static GstFlowReturn gst_gl_download_element_transform (GstBaseTransform * bt, GstBuffer * buffer, GstBuffer * outbuf); +static gboolean gst_gl_download_element_decide_allocation (GstBaseTransform * + trans, GstQuery * query); +static void gst_gl_download_element_finalize (GObject * object); static GstStaticPadTemplate gst_gl_download_element_src_pad_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS ("video/x-raw; video/x-raw(memory:GLMemory)")); + GST_STATIC_CAPS ( +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF + "video/x-raw(" GST_CAPS_FEATURE_MEMORY_DMABUF "); " +#endif + "video/x-raw; video/x-raw(memory:GLMemory)")); static GstStaticPadTemplate gst_gl_download_element_sink_pad_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -63,6 +75,7 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass) { GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass); GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GObjectClass *object_class = G_OBJECT_CLASS (klass); bt_class->transform_caps = gst_gl_download_element_transform_caps; bt_class->set_caps = gst_gl_download_element_set_caps; @@ -70,6 +83,7 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass) bt_class->prepare_output_buffer = gst_gl_download_element_prepare_output_buffer; bt_class->transform = gst_gl_download_element_transform; + bt_class->decide_allocation = gst_gl_download_element_decide_allocation; bt_class->passthrough_on_same_caps = TRUE; @@ -81,6 +95,8 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass) gst_element_class_set_metadata (element_class, "OpenGL downloader", "Filter/Video", "Downloads data from OpenGL", "Matthew Waters "); + + object_class->finalize = gst_gl_download_element_finalize; } static void @@ -103,8 +119,26 @@ gst_gl_download_element_set_caps (GstBaseTransform * bt, GstCaps * in_caps, features = gst_caps_get_features (out_caps, 0); - dl->do_pbo_transfers = (!features || gst_caps_features_contains (features, - GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY)); + dl->do_pbo_transfers = FALSE; + if (dl->dmabuf_allocator) { + gst_object_unref (GST_OBJECT (dl->dmabuf_allocator)); + dl->dmabuf_allocator = NULL; + } + + if (!features) { + dl->do_pbo_transfers = TRUE; + return TRUE; + } + + if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { + /* do nothing with the buffer */ + } else if (gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_DMABUF)) { + dl->dmabuf_allocator = gst_dmabuf_allocator_new (); + } else if (gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY)) { + dl->do_pbo_transfers = TRUE; + } return TRUE; } @@ -133,8 +167,16 @@ gst_gl_download_element_transform_caps (GstBaseTransform * bt, tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); tmp = gst_caps_merge (gst_caps_ref (caps), tmp); } else { - tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); - tmp = gst_caps_merge (gst_caps_ref (caps), tmp); + GstCaps *newcaps; + tmp = gst_caps_ref (caps); + +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF + newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_DMABUF); + tmp = gst_caps_merge (tmp, newcaps); +#endif + + newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + tmp = gst_caps_merge (tmp, newcaps); } if (filter) { @@ -163,6 +205,168 @@ gst_gl_download_element_get_unit_size (GstBaseTransform * trans, GstCaps * caps, return TRUE; } +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF + +struct DmabufInfo +{ + GstMemory *dmabuf; + gint stride; + gsize offset; +}; + +static void +_free_dmabuf_info (struct DmabufInfo *info) +{ + gst_memory_unref (info->dmabuf); + g_free (info); +} + +static GQuark +_dmabuf_info_quark (void) +{ + static GQuark quark = 0; + + if (!quark) + quark = g_quark_from_static_string ("GstGLDownloadDmabufInfo"); + return quark; +} + +static struct DmabufInfo * +_get_cached_dmabuf_info (GstGLMemory * mem) +{ + return gst_mini_object_get_qdata (GST_MINI_OBJECT (mem), + _dmabuf_info_quark ()); +} + +static void +_set_cached_dmabuf_info (GstGLMemory * mem, struct DmabufInfo *info) +{ + return gst_mini_object_set_qdata (GST_MINI_OBJECT (mem), + _dmabuf_info_quark (), info, (GDestroyNotify) _free_dmabuf_info); +} + +struct DmabufTransfer +{ + GstGLDownloadElement *download; + GstGLMemory *glmem; + struct DmabufInfo *info; +}; + +static void +_create_cached_dmabuf_info (GstGLContext * context, gpointer data) +{ + struct DmabufTransfer *transfer = (struct DmabufTransfer *) data; + GstEGLImage *image; + + image = gst_egl_image_from_texture (context, transfer->glmem, NULL); + if (image) { + int fd; + gint stride; + gsize offset; + + if (gst_egl_image_export_dmabuf (image, &fd, &stride, &offset)) { + GstGLDownloadElement *download = transfer->download; + struct DmabufInfo *info; + gsize maxsize; + + gst_memory_get_sizes (GST_MEMORY_CAST (transfer->glmem), NULL, &maxsize); + + info = g_new0 (struct DmabufInfo, 1); + info->dmabuf = + gst_dmabuf_allocator_alloc (download->dmabuf_allocator, fd, maxsize); + info->stride = stride; + info->offset = offset; + + transfer->info = info; + } + + gst_egl_image_unref (image); + } +} + +static GstBuffer * +_try_export_dmabuf (GstGLDownloadElement * download, GstBuffer * inbuf) +{ + GstGLMemory *glmem; + GstBuffer *buffer = NULL; + int i; + gsize offset[GST_VIDEO_MAX_PLANES]; + gint stride[GST_VIDEO_MAX_PLANES]; + GstCaps *src_caps; + GstVideoInfo out_info; + gsize total_offset; + + glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, 0)); + if (glmem) { + GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context; + if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL) + return NULL; + } + + buffer = gst_buffer_new (); + total_offset = 0; + + for (i = 0; i < gst_buffer_n_memory (inbuf); i++) { + struct DmabufInfo *info; + + glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, i)); + info = _get_cached_dmabuf_info (glmem); + if (!info) { + GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context; + struct DmabufTransfer transfer; + + transfer.download = download; + transfer.glmem = glmem; + transfer.info = NULL; + gst_gl_context_thread_add (context, _create_cached_dmabuf_info, + &transfer); + info = transfer.info; + + if (info) + _set_cached_dmabuf_info (glmem, info); + } + + if (info) { + offset[i] = total_offset + info->offset; + stride[i] = info->stride; + total_offset += gst_memory_get_sizes (info->dmabuf, NULL, NULL); + gst_buffer_insert_memory (buffer, -1, gst_memory_ref (info->dmabuf)); + } else { + gst_buffer_unref (buffer); + buffer = NULL; + goto export_complete; + } + } + + src_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM (download)->srcpad); + gst_video_info_from_caps (&out_info, src_caps); + + if (download->add_videometa) { + gst_buffer_add_video_meta_full (buffer, out_info.flags, + out_info.finfo->format, out_info.width, out_info.height, + out_info.finfo->n_planes, offset, stride); + } else { + int i; + gboolean match = TRUE; + for (i = 0; i < gst_buffer_n_memory (inbuf); i++) { + if (offset[i] != out_info.offset[i] || stride[i] != out_info.stride[i]) { + match = FALSE; + break; + } + } + + if (!match) { + gst_buffer_unref (buffer); + buffer = NULL; + } + } + +export_complete: + + return buffer; +} +#endif /* GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF */ + static GstFlowReturn gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer ** outbuf) @@ -181,6 +385,38 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt, gst_gl_memory_pbo_download_transfer ((GstGLMemoryPBO *) mem); } } +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF + else if (dl->dmabuf_allocator) { + GstBuffer *buffer = _try_export_dmabuf (dl, inbuf); + if (buffer) { + if (GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata) + if (!GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata (bt, inbuf, + buffer)) { + GST_ELEMENT_WARNING (GST_ELEMENT (bt), STREAM, NOT_IMPLEMENTED, + ("could not copy metadata"), (NULL)); + } + + *outbuf = buffer; + } else { + GstCaps *src_caps; + GstCapsFeatures *features; + + gst_object_unref (dl->dmabuf_allocator); + dl->dmabuf_allocator = NULL; + + src_caps = gst_pad_get_current_caps (bt->srcpad); + src_caps = gst_caps_make_writable (src_caps); + features = gst_caps_get_features (src_caps, 0); + gst_caps_features_remove (features, GST_CAPS_FEATURE_MEMORY_DMABUF); + + if (!gst_base_transform_update_src_caps (bt, src_caps)) { + GST_ERROR_OBJECT (bt, "DMABuf exportation didn't work and system " + "memory is not supported."); + return GST_FLOW_NOT_NEGOTIATED; + } + } + } +#endif return GST_FLOW_OK; } @@ -191,3 +427,32 @@ gst_gl_download_element_transform (GstBaseTransform * bt, { return GST_FLOW_OK; } + +static gboolean +gst_gl_download_element_decide_allocation (GstBaseTransform * trans, + GstQuery * query) +{ + GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (trans); + + if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { + download->add_videometa = TRUE; + } else { + download->add_videometa = FALSE; + } + + return GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans, + query); +} + +static void +gst_gl_download_element_finalize (GObject * object) +{ + GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (object); + + if (download->dmabuf_allocator) { + gst_object_unref (GST_OBJECT (download->dmabuf_allocator)); + download->dmabuf_allocator = NULL; + } + + G_OBJECT_CLASS (parent_class)->finalize (object); +} diff --git a/ext/gl/gstgldownloadelement.h b/ext/gl/gstgldownloadelement.h index e7d4ce03a4..a9fb4e1650 100644 --- a/ext/gl/gstgldownloadelement.h +++ b/ext/gl/gstgldownloadelement.h @@ -45,6 +45,8 @@ struct _GstGLDownloadElement GstGLBaseFilter parent; gboolean do_pbo_transfers; + GstAllocator * dmabuf_allocator; + gboolean add_videometa; }; struct _GstGLDownloadElementClass diff --git a/gst-libs/gst/gl/egl/gsteglimage.c b/gst-libs/gst/gl/egl/gsteglimage.c index 92b3862b14..fdd6038073 100644 --- a/gst-libs/gst/gl/egl/gsteglimage.c +++ b/gst-libs/gst/gl/egl/gsteglimage.c @@ -469,4 +469,63 @@ gst_egl_image_from_dmabuf (GstGLContext * context, return gst_egl_image_new_wrapped (context, img, format, NULL, (GstEGLImageDestroyNotify) _destroy_egl_image); } + +gboolean +gst_egl_image_export_dmabuf (GstEGLImage * image, int *fd, gint * stride, + gsize * offset) +{ + EGLBoolean (*gst_eglExportDMABUFImageQueryMESA) (EGLDisplay dpy, + EGLImageKHR image, int *fourcc, int *num_planes, + EGLuint64KHR * modifiers); + EGLBoolean (*gst_eglExportDMABUFImageMESA) (EGLDisplay dpy, EGLImageKHR image, + int *fds, EGLint * strides, EGLint * offsets); + GstGLDisplayEGL *display_egl; + EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; + int num_planes = 0; + int egl_fd = 0; + EGLint egl_stride = 0; + EGLint egl_offset = 0; + + gst_eglExportDMABUFImageQueryMESA = + gst_gl_context_get_proc_address (image->context, + "eglExportDMABUFImageQueryMESA"); + gst_eglExportDMABUFImageMESA = + gst_gl_context_get_proc_address (image->context, + "eglExportDMABUFImageMESA"); + + if (!gst_eglExportDMABUFImageQueryMESA || !gst_eglExportDMABUFImageMESA) + return FALSE; + + display_egl = + (GstGLDisplayEGL *) gst_gl_display_egl_from_gl_display (image-> + context->display); + if (!display_egl) { + GST_WARNING_OBJECT (image->context, + "Failed to retrieve GstGLDisplayEGL from %" GST_PTR_FORMAT, + image->context->display); + return FALSE; + } + egl_display = + (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (display_egl)); + gst_object_unref (display_egl); + + if (!gst_eglExportDMABUFImageQueryMESA (egl_display, image->image, + NULL, &num_planes, NULL)) + return FALSE; + + /* Don't allow multi-plane dmabufs */ + if (num_planes > 1) + return FALSE; + + if (!gst_eglExportDMABUFImageMESA (egl_display, image->image, &egl_fd, + &egl_stride, &egl_offset)) + return FALSE; + + *fd = egl_fd; + *stride = egl_stride; + *offset = egl_offset; + + return TRUE; +} + #endif /* GST_GL_HAVE_DMABUF */ diff --git a/gst-libs/gst/gl/egl/gsteglimage.h b/gst-libs/gst/gl/egl/gsteglimage.h index cfd8f008aa..b96cc2d3d8 100644 --- a/gst-libs/gst/gl/egl/gsteglimage.h +++ b/gst-libs/gst/gl/egl/gsteglimage.h @@ -88,6 +88,8 @@ GstEGLImage * gst_egl_image_from_dmabuf (GstGLContext * GstVideoInfo * in_info, gint plane, gsize offset); +GST_EXPORT +gboolean gst_egl_image_export_dmabuf (GstEGLImage *image, int *fd, gint *stride, gsize *offset); #endif /**