gldownload: Add dmabuf exporting

This patch adds code to gldownload to export the image as a
dmabuf if requested.  The element now exposes memory:DMABuf as
a cap feature, and if it is selected, the element exports the
texture to an EGL image and then a dmabuf. It also implements a
fallback to system memory download in case the exportation failed.

https://bugzilla.gnome.org/show_bug.cgi?id=776927

ext/gl/gstgldownloadelement.c

@@ -23,6 +23,11 @@
 #endif
 
 #include <gst/gl/gl.h>
+#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
+#include <gst/gl/egl/gsteglimage.h>
+#include <gst/allocators/gstdmabuf.h>
+#endif
+
 #include "gstgldownloadelement.h"
 
 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug);
@@ -45,12 +50,19 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
     GstBuffer * buffer, GstBuffer ** outbuf);
 static GstFlowReturn gst_gl_download_element_transform (GstBaseTransform * bt,
     GstBuffer * buffer, GstBuffer * outbuf);
+static gboolean gst_gl_download_element_decide_allocation (GstBaseTransform *
+    trans, GstQuery * query);
+static void gst_gl_download_element_finalize (GObject * object);
 
 static GstStaticPadTemplate gst_gl_download_element_src_pad_template =
     GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
     GST_PAD_ALWAYS,
-    GST_STATIC_CAPS ("video/x-raw; video/x-raw(memory:GLMemory)"));
+    GST_STATIC_CAPS (
+#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
+        "video/x-raw(" GST_CAPS_FEATURE_MEMORY_DMABUF "); "
+#endif
+        "video/x-raw; video/x-raw(memory:GLMemory)"));
 
 static GstStaticPadTemplate gst_gl_download_element_sink_pad_template =
     GST_STATIC_PAD_TEMPLATE ("sink",
@@ -63,6 +75,7 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass)
 {
   GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass);
   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
 
   bt_class->transform_caps = gst_gl_download_element_transform_caps;
   bt_class->set_caps = gst_gl_download_element_set_caps;
@@ -70,6 +83,7 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass)
   bt_class->prepare_output_buffer =
       gst_gl_download_element_prepare_output_buffer;
   bt_class->transform = gst_gl_download_element_transform;
+  bt_class->decide_allocation = gst_gl_download_element_decide_allocation;
 
   bt_class->passthrough_on_same_caps = TRUE;
 
@@ -81,6 +95,8 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass)
   gst_element_class_set_metadata (element_class,
       "OpenGL downloader", "Filter/Video",
       "Downloads data from OpenGL", "Matthew Waters <matthew@centricular.com>");
+
+  object_class->finalize = gst_gl_download_element_finalize;
 }
 
 static void
@@ -103,8 +119,26 @@ gst_gl_download_element_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
 
   features = gst_caps_get_features (out_caps, 0);
 
-  dl->do_pbo_transfers = (!features || gst_caps_features_contains (features,
+  dl->do_pbo_transfers = FALSE;
-          GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY));
+  if (dl->dmabuf_allocator) {
+    gst_object_unref (GST_OBJECT (dl->dmabuf_allocator));
+    dl->dmabuf_allocator = NULL;
+  }
+
+  if (!features) {
+    dl->do_pbo_transfers = TRUE;
+    return TRUE;
+  }
+
+  if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
+    /* do nothing with the buffer */
+  } else if (gst_caps_features_contains (features,
+          GST_CAPS_FEATURE_MEMORY_DMABUF)) {
+    dl->dmabuf_allocator = gst_dmabuf_allocator_new ();
+  } else if (gst_caps_features_contains (features,
+          GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY)) {
+    dl->do_pbo_transfers = TRUE;
+  }
 
   return TRUE;
 }
@@ -133,8 +167,16 @@ gst_gl_download_element_transform_caps (GstBaseTransform * bt,
     tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
     tmp = gst_caps_merge (gst_caps_ref (caps), tmp);
   } else {
-    tmp = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+    GstCaps *newcaps;
-    tmp = gst_caps_merge (gst_caps_ref (caps), tmp);
+    tmp = gst_caps_ref (caps);
+
+#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
+    newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_DMABUF);
+    tmp = gst_caps_merge (tmp, newcaps);
+#endif
+
+    newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
+    tmp = gst_caps_merge (tmp, newcaps);
   }
 
   if (filter) {
@@ -163,6 +205,168 @@ gst_gl_download_element_get_unit_size (GstBaseTransform * trans, GstCaps * caps,
   return TRUE;
 }
 
+#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
+
+struct DmabufInfo
+{
+  GstMemory *dmabuf;
+  gint stride;
+  gsize offset;
+};
+
+static void
+_free_dmabuf_info (struct DmabufInfo *info)
+{
+  gst_memory_unref (info->dmabuf);
+  g_free (info);
+}
+
+static GQuark
+_dmabuf_info_quark (void)
+{
+  static GQuark quark = 0;
+
+  if (!quark)
+    quark = g_quark_from_static_string ("GstGLDownloadDmabufInfo");
+  return quark;
+}
+
+static struct DmabufInfo *
+_get_cached_dmabuf_info (GstGLMemory * mem)
+{
+  return gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
+      _dmabuf_info_quark ());
+}
+
+static void
+_set_cached_dmabuf_info (GstGLMemory * mem, struct DmabufInfo *info)
+{
+  return gst_mini_object_set_qdata (GST_MINI_OBJECT (mem),
+      _dmabuf_info_quark (), info, (GDestroyNotify) _free_dmabuf_info);
+}
+
+struct DmabufTransfer
+{
+  GstGLDownloadElement *download;
+  GstGLMemory *glmem;
+  struct DmabufInfo *info;
+};
+
+static void
+_create_cached_dmabuf_info (GstGLContext * context, gpointer data)
+{
+  struct DmabufTransfer *transfer = (struct DmabufTransfer *) data;
+  GstEGLImage *image;
+
+  image = gst_egl_image_from_texture (context, transfer->glmem, NULL);
+  if (image) {
+    int fd;
+    gint stride;
+    gsize offset;
+
+    if (gst_egl_image_export_dmabuf (image, &fd, &stride, &offset)) {
+      GstGLDownloadElement *download = transfer->download;
+      struct DmabufInfo *info;
+      gsize maxsize;
+
+      gst_memory_get_sizes (GST_MEMORY_CAST (transfer->glmem), NULL, &maxsize);
+
+      info = g_new0 (struct DmabufInfo, 1);
+      info->dmabuf =
+          gst_dmabuf_allocator_alloc (download->dmabuf_allocator, fd, maxsize);
+      info->stride = stride;
+      info->offset = offset;
+
+      transfer->info = info;
+    }
+
+    gst_egl_image_unref (image);
+  }
+}
+
+static GstBuffer *
+_try_export_dmabuf (GstGLDownloadElement * download, GstBuffer * inbuf)
+{
+  GstGLMemory *glmem;
+  GstBuffer *buffer = NULL;
+  int i;
+  gsize offset[GST_VIDEO_MAX_PLANES];
+  gint stride[GST_VIDEO_MAX_PLANES];
+  GstCaps *src_caps;
+  GstVideoInfo out_info;
+  gsize total_offset;
+
+  glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, 0));
+  if (glmem) {
+    GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context;
+    if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
+      return NULL;
+  }
+
+  buffer = gst_buffer_new ();
+  total_offset = 0;
+
+  for (i = 0; i < gst_buffer_n_memory (inbuf); i++) {
+    struct DmabufInfo *info;
+
+    glmem = GST_GL_MEMORY_CAST (gst_buffer_peek_memory (inbuf, i));
+    info = _get_cached_dmabuf_info (glmem);
+    if (!info) {
+      GstGLContext *context = GST_GL_BASE_MEMORY_CAST (glmem)->context;
+      struct DmabufTransfer transfer;
+
+      transfer.download = download;
+      transfer.glmem = glmem;
+      transfer.info = NULL;
+      gst_gl_context_thread_add (context, _create_cached_dmabuf_info,
+          &transfer);
+      info = transfer.info;
+
+      if (info)
+        _set_cached_dmabuf_info (glmem, info);
+    }
+
+    if (info) {
+      offset[i] = total_offset + info->offset;
+      stride[i] = info->stride;
+      total_offset += gst_memory_get_sizes (info->dmabuf, NULL, NULL);
+      gst_buffer_insert_memory (buffer, -1, gst_memory_ref (info->dmabuf));
+    } else {
+      gst_buffer_unref (buffer);
+      buffer = NULL;
+      goto export_complete;
+    }
+  }
+
+  src_caps = gst_pad_get_current_caps (GST_BASE_TRANSFORM (download)->srcpad);
+  gst_video_info_from_caps (&out_info, src_caps);
+
+  if (download->add_videometa) {
+    gst_buffer_add_video_meta_full (buffer, out_info.flags,
+        out_info.finfo->format, out_info.width, out_info.height,
+        out_info.finfo->n_planes, offset, stride);
+  } else {
+    int i;
+    gboolean match = TRUE;
+    for (i = 0; i < gst_buffer_n_memory (inbuf); i++) {
+      if (offset[i] != out_info.offset[i] || stride[i] != out_info.stride[i]) {
+        match = FALSE;
+        break;
+      }
+    }
+
+    if (!match) {
+      gst_buffer_unref (buffer);
+      buffer = NULL;
+    }
+  }
+
+export_complete:
+
+  return buffer;
+}
+#endif /* GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF */
+
 static GstFlowReturn
 gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
     GstBuffer * inbuf, GstBuffer ** outbuf)
@@ -181,6 +385,38 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
         gst_gl_memory_pbo_download_transfer ((GstGLMemoryPBO *) mem);
     }
   }
+#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF
+  else if (dl->dmabuf_allocator) {
+    GstBuffer *buffer = _try_export_dmabuf (dl, inbuf);
+    if (buffer) {
+      if (GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata)
+        if (!GST_BASE_TRANSFORM_GET_CLASS (bt)->copy_metadata (bt, inbuf,
+                buffer)) {
+          GST_ELEMENT_WARNING (GST_ELEMENT (bt), STREAM, NOT_IMPLEMENTED,
+              ("could not copy metadata"), (NULL));
+        }
+
+      *outbuf = buffer;
+    } else {
+      GstCaps *src_caps;
+      GstCapsFeatures *features;
+
+      gst_object_unref (dl->dmabuf_allocator);
+      dl->dmabuf_allocator = NULL;
+
+      src_caps = gst_pad_get_current_caps (bt->srcpad);
+      src_caps = gst_caps_make_writable (src_caps);
+      features = gst_caps_get_features (src_caps, 0);
+      gst_caps_features_remove (features, GST_CAPS_FEATURE_MEMORY_DMABUF);
+
+      if (!gst_base_transform_update_src_caps (bt, src_caps)) {
+        GST_ERROR_OBJECT (bt, "DMABuf exportation didn't work and system "
+            "memory is not supported.");
+        return GST_FLOW_NOT_NEGOTIATED;
+      }
+    }
+  }
+#endif
 
   return GST_FLOW_OK;
 }
@@ -191,3 +427,32 @@ gst_gl_download_element_transform (GstBaseTransform * bt,
 {
   return GST_FLOW_OK;
 }
+
+static gboolean
+gst_gl_download_element_decide_allocation (GstBaseTransform * trans,
+    GstQuery * query)
+{
+  GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (trans);
+
+  if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) {
+    download->add_videometa = TRUE;
+  } else {
+    download->add_videometa = FALSE;
+  }
+
+  return GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
+      query);
+}
+
+static void
+gst_gl_download_element_finalize (GObject * object)
+{
+  GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT_CAST (object);
+
+  if (download->dmabuf_allocator) {
+    gst_object_unref (GST_OBJECT (download->dmabuf_allocator));
+    download->dmabuf_allocator = NULL;
+  }
+
+  G_OBJECT_CLASS (parent_class)->finalize (object);
+}

ext/gl/gstgldownloadelement.h

@@ -45,6 +45,8 @@ struct _GstGLDownloadElement
   GstGLBaseFilter  parent;
 
   gboolean do_pbo_transfers;
+  GstAllocator * dmabuf_allocator;
+  gboolean add_videometa;
 };
 
 struct _GstGLDownloadElementClass

gst-libs/gst/gl/egl/gsteglimage.c

@@ -469,4 +469,63 @@ gst_egl_image_from_dmabuf (GstGLContext * context,
   return gst_egl_image_new_wrapped (context, img, format, NULL,
       (GstEGLImageDestroyNotify) _destroy_egl_image);
 }
+
+gboolean
+gst_egl_image_export_dmabuf (GstEGLImage * image, int *fd, gint * stride,
+    gsize * offset)
+{
+  EGLBoolean (*gst_eglExportDMABUFImageQueryMESA) (EGLDisplay dpy,
+      EGLImageKHR image, int *fourcc, int *num_planes,
+      EGLuint64KHR * modifiers);
+  EGLBoolean (*gst_eglExportDMABUFImageMESA) (EGLDisplay dpy, EGLImageKHR image,
+      int *fds, EGLint * strides, EGLint * offsets);
+  GstGLDisplayEGL *display_egl;
+  EGLDisplay egl_display = EGL_DEFAULT_DISPLAY;
+  int num_planes = 0;
+  int egl_fd = 0;
+  EGLint egl_stride = 0;
+  EGLint egl_offset = 0;
+
+  gst_eglExportDMABUFImageQueryMESA =
+      gst_gl_context_get_proc_address (image->context,
+      "eglExportDMABUFImageQueryMESA");
+  gst_eglExportDMABUFImageMESA =
+      gst_gl_context_get_proc_address (image->context,
+      "eglExportDMABUFImageMESA");
+
+  if (!gst_eglExportDMABUFImageQueryMESA || !gst_eglExportDMABUFImageMESA)
+    return FALSE;
+
+  display_egl =
+      (GstGLDisplayEGL *) gst_gl_display_egl_from_gl_display (image->
+      context->display);
+  if (!display_egl) {
+    GST_WARNING_OBJECT (image->context,
+        "Failed to retrieve GstGLDisplayEGL from %" GST_PTR_FORMAT,
+        image->context->display);
+    return FALSE;
+  }
+  egl_display =
+      (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (display_egl));
+  gst_object_unref (display_egl);
+
+  if (!gst_eglExportDMABUFImageQueryMESA (egl_display, image->image,
+          NULL, &num_planes, NULL))
+    return FALSE;
+
+  /* Don't allow multi-plane dmabufs */
+  if (num_planes > 1)
+    return FALSE;
+
+  if (!gst_eglExportDMABUFImageMESA (egl_display, image->image, &egl_fd,
+          &egl_stride, &egl_offset))
+    return FALSE;
+
+  *fd = egl_fd;
+  *stride = egl_stride;
+  *offset = egl_offset;
+
+  return TRUE;
+}
+
 #endif /* GST_GL_HAVE_DMABUF */

gst-libs/gst/gl/egl/gsteglimage.h

@@ -88,6 +88,8 @@ GstEGLImage *           gst_egl_image_from_dmabuf               (GstGLContext *
                                                                  GstVideoInfo * in_info,
                                                                  gint plane,
                                                                  gsize offset);
+GST_EXPORT
+gboolean                gst_egl_image_export_dmabuf             (GstEGLImage *image, int *fd, gint *stride, gsize *offset);
 #endif
 
 /**