diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c index 2b65088982..4e16d1f980 100644 --- a/ext/gl/gstgldownloadelement.c +++ b/ext/gl/gstgldownloadelement.c @@ -34,6 +34,742 @@ GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug); #define GST_CAT_DEFAULT gst_gl_download_element_debug +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) +#include +#include + +#include "nvbuf_utils.h" + +static const char * +nv_buffer_payload_type_to_string (NvBufferPayloadType ptype) +{ + switch (ptype) { + case NvBufferPayload_SurfArray: + return "SurfArray"; + case NvBufferPayload_MemHandle: + return "MemHandle"; + default: + return ""; + } +} + +static const char * +nv_buffer_pixel_format_to_string (NvBufferColorFormat fmt) +{ + switch (fmt) { + case NvBufferColorFormat_YUV420: + return "YUV420"; + case NvBufferColorFormat_YVU420: + return "YVU420"; + case NvBufferColorFormat_YUV422: + return "YUV422"; + case NvBufferColorFormat_YUV420_ER: + return "YUV420_ER"; + case NvBufferColorFormat_YVU420_ER: + return "YVU420_ER"; + case NvBufferColorFormat_NV12: + return "NV12"; + case NvBufferColorFormat_NV12_ER: + return "NV12_ER"; + case NvBufferColorFormat_NV21: + return "NV21"; + case NvBufferColorFormat_NV21_ER: + return "NV21_ER"; + case NvBufferColorFormat_UYVY: + return "UYVY"; + case NvBufferColorFormat_UYVY_ER: + return "UYVY_ER"; + case NvBufferColorFormat_VYUY: + return "VYUY"; + case NvBufferColorFormat_VYUY_ER: + return "VYUY_ER"; + case NvBufferColorFormat_YUYV: + return "YUYV"; + case NvBufferColorFormat_YUYV_ER: + return "YUYV_ER"; + case NvBufferColorFormat_YVYU: + return "YVYU"; + case NvBufferColorFormat_YVYU_ER: + return "YVYU_ER"; + case NvBufferColorFormat_ABGR32: + return "ABGR32"; + case NvBufferColorFormat_XRGB32: + return "XRGB32"; + case NvBufferColorFormat_ARGB32: + return "ARGB32"; + case NvBufferColorFormat_NV12_10LE: + return "NV12_10LE"; + case NvBufferColorFormat_NV12_10LE_709: + return "NV12_10LE_709"; + case NvBufferColorFormat_NV12_10LE_709_ER: + return "NV12_10LE_709_ER"; + case NvBufferColorFormat_NV12_10LE_2020: + return "NV12_2020"; + case NvBufferColorFormat_NV21_10LE: + return "NV21_10LE"; + case NvBufferColorFormat_NV12_12LE: + return "NV12_12LE"; + case NvBufferColorFormat_NV12_12LE_2020: + return "NV12_12LE_2020"; + case NvBufferColorFormat_NV21_12LE: + return "NV21_12LE"; + case NvBufferColorFormat_YUV420_709: + return "YUV420_709"; + case NvBufferColorFormat_YUV420_709_ER: + return "YUV420_709_ER"; + case NvBufferColorFormat_NV12_709: + return "NV12_709"; + case NvBufferColorFormat_NV12_709_ER: + return "NV12_709_ER"; + case NvBufferColorFormat_YUV420_2020: + return "YUV420_2020"; + case NvBufferColorFormat_NV12_2020: + return "NV12_2020"; + case NvBufferColorFormat_SignedR16G16: + return "SignedR16G16"; + case NvBufferColorFormat_A32: + return "A32"; + case NvBufferColorFormat_YUV444: + return "YUV444"; + case NvBufferColorFormat_GRAY8: + return "GRAY8"; + case NvBufferColorFormat_NV16: + return "NV16"; + case NvBufferColorFormat_NV16_10LE: + return "NV16_10LE"; + case NvBufferColorFormat_NV24: + return "NV24"; + case NvBufferColorFormat_NV16_ER: + return "NV16_ER"; + case NvBufferColorFormat_NV24_ER: + return "NV24_ER"; + case NvBufferColorFormat_NV16_709: + return "NV16_709"; + case NvBufferColorFormat_NV24_709: + return "NV24_709"; + case NvBufferColorFormat_NV16_709_ER: + return "NV16_709_ER"; + case NvBufferColorFormat_NV24_709_ER: + return "NV24_709_ER"; + case NvBufferColorFormat_NV24_10LE_709: + return "NV24_10LE_709"; + case NvBufferColorFormat_NV24_10LE_709_ER: + return "NV24_10LE_709_ER"; + case NvBufferColorFormat_NV24_10LE_2020: + return "NV24_10LE_2020"; + case NvBufferColorFormat_NV24_12LE_2020: + return "NV24_12LE_2020"; + case NvBufferColorFormat_RGBA_10_10_10_2_709: + return "RGBA_10_10_10_2_709"; + case NvBufferColorFormat_RGBA_10_10_10_2_2020: + return "RGBA_10_10_10_2_2020"; + case NvBufferColorFormat_BGRA_10_10_10_2_709: + return "BGRA_10_10_10_2_709"; + case NvBufferColorFormat_BGRA_10_10_10_2_2020: + return "BGRA_10_10_10_2_2020"; + case NvBufferColorFormat_Invalid: + return "Invalid"; + default: + return ""; + } +} + +static void +nv_buffer_dump_params (GstObject * debug_object, NvBufferParamsEx * params) +{ + GST_DEBUG_OBJECT (debug_object, "nvbuffer fd: %u size %i nv_buffer: %p of " + "size %u, payload: (0x%x) %s, pixel format: (0x%x) %s, n_planes: %u, " + "plane 0 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } " + "plane 1 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } " + "plane 2 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u }", + params->params.dmabuf_fd, params->params.memsize, + params->params.nv_buffer, params->params.nv_buffer_size, + params->params.payloadType, + nv_buffer_payload_type_to_string (params->params.payloadType), + params->params.pixel_format, + nv_buffer_pixel_format_to_string (params->params.pixel_format), + params->params.num_planes, params->params.width[0], + params->params.height[0], params->params.pitch[0], + params->params.offset[0], params->params.psize[0], + params->params.offset[0], params->params.width[1], + params->params.height[1], params->params.pitch[1], + params->params.offset[1], params->params.psize[1], + params->params.offset[1], params->params.width[2], + params->params.height[2], params->params.pitch[2], + params->params.offset[2], params->params.psize[2], + params->params.offset[2]); +} + +struct _GstMemoryNVMM +{ + GstMemory parent; + + int dmabuf_fd; + NvBufferParamsEx params; +}; + +typedef struct _GstMemoryNVMM GstMemoryNVMM; + +struct _GstAllocatorNVMM +{ + GstAllocator parent; +}; + +typedef struct _GstAllocatorNVMM GstAllocatorNVMM; + +struct _GstAllocatorNVMMClass +{ + GstAllocatorClass parent_class; +}; + +typedef struct _GstAllocatorNVMMClass GstAllocatorNVMMClass; + +GType gst_allocator_nvmm_get_type (void); +G_DEFINE_TYPE (GstAllocatorNVMM, gst_allocator_nvmm, GST_TYPE_ALLOCATOR); + +static gboolean +gst_memory_nvmm_init (GstMemoryNVMM * nvmm, GstMemoryFlags flags, + GstAllocator * allocator, GstMemory * parent, const GstVideoInfo * vinfo) +{ + gsize size = NvBufferGetSize (); + NvBufferCreateParams create_params = { + .width = GST_VIDEO_INFO_WIDTH (vinfo), + .height = GST_VIDEO_INFO_HEIGHT (vinfo), + .payloadType = NvBufferPayload_SurfArray, + .memsize = GST_VIDEO_INFO_SIZE (vinfo), + .layout = NvBufferLayout_BlockLinear, + .colorFormat = NvBufferColorFormat_ABGR32, + .nvbuf_tag = NvBufferTag_NONE, + }; + + nvmm->dmabuf_fd = -1; + + if (NvBufferCreateEx (&nvmm->dmabuf_fd, &create_params)) { + GST_WARNING_OBJECT (allocator, "Failed to create NvBuffer"); + return FALSE; + } + + if (NvBufferGetParamsEx (nvmm->dmabuf_fd, &nvmm->params)) { + GST_WARNING_OBJECT (allocator, "Failed to get NvBuffer params"); + NvReleaseFd (nvmm->dmabuf_fd); + nvmm->dmabuf_fd = -1; + return FALSE; + } + nv_buffer_dump_params ((GstObject *) allocator, &nvmm->params); + + gst_memory_init (&nvmm->parent, flags, allocator, parent, size, 0, 0, size); + + return TRUE; +} + +static gpointer +gst_memory_nvmm_map_full (GstMemory * mem, GstMapInfo * info, gsize size) +{ + GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem; + + GST_TRACE ("%p fd:%i map", mem, nvmm->dmabuf_fd); + + // This is what the Nvidia elements do so... + return nvmm->params.params.nv_buffer; +} + +static void +gst_memory_nvmm_unmap_full (GstMemory * mem, GstMapInfo * info) +{ + GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem; + + GST_TRACE ("%p fd:%i unmap", mem, nvmm->dmabuf_fd); +} + +static GstMemory * +gst_memory_nvmm_copy (GstMemory * mem, gssize offset, gssize size) +{ + return NULL; +} + +static GstMemory * +gst_memory_nvmm_share (GstMemory * mem, gssize offset, gssize size) +{ + return NULL; +} + +static gboolean +gst_memory_nvmm_is_span (GstMemory * mem, GstMemory * mem2, gsize * offset) +{ + return FALSE; +} + +static gboolean +gst_is_memory_nvmm (GstMemory * mem) +{ + return mem && mem->allocator + && g_type_is_a (G_OBJECT_TYPE (mem->allocator), + gst_allocator_nvmm_get_type ()); +} + +static GstAllocator *_nvmm_allocator; + +static void +init_nvmm_allocator (void) +{ + static gsize _init = 0; + + if (g_once_init_enter (&_init)) { + _nvmm_allocator = g_object_new (gst_allocator_nvmm_get_type (), NULL); +/* gst_allocator_register ("NvBuffer", _nvmm_allocator); */ + GST_OBJECT_FLAG_SET (_nvmm_allocator, GST_OBJECT_FLAG_MAY_BE_LEAKED); + g_once_init_leave (&_init, 1); + } +} + +static GstMemory * +gst_allocator_nvmm_alloc (const GstVideoInfo * info) +{ + GstMemoryNVMM *nvmm = g_new0 (GstMemoryNVMM, 1); + + init_nvmm_allocator (); + + if (!gst_memory_nvmm_init (nvmm, 0, _nvmm_allocator, NULL, info)) { + g_free (nvmm); + return NULL; + } + + return (GstMemory *) nvmm; +} + +static GstMemory * +_gst_allocator_nvmm_alloc (GstAllocator * alloc, gsize size, + GstAllocationParams * params) +{ + g_warning + ("Can't allocate using gst_allocator_alloc(). Use gst_allocator_nvmm_alloc() instead"); + + return NULL; +} + +static void +_gst_allocator_nvmm_free (GstAllocator * alloc, GstMemory * mem) +{ + GstMemoryNVMM *nvmm = (GstMemoryNVMM *) mem; + + if (nvmm->dmabuf_fd > 0) + NvReleaseFd (nvmm->dmabuf_fd); + nvmm->dmabuf_fd = -1; + + g_free (nvmm); +} + +static void +gst_allocator_nvmm_class_init (GstAllocatorNVMMClass * klass) +{ + GstAllocatorClass *alloc_class = (GstAllocatorClass *) klass; + + alloc_class->alloc = _gst_allocator_nvmm_alloc; + alloc_class->free = _gst_allocator_nvmm_free; +} + +static void +gst_allocator_nvmm_init (GstAllocatorNVMM * nvmm) +{ + GstAllocator *alloc = (GstAllocator *) nvmm; + + alloc->mem_map_full = gst_memory_nvmm_map_full; + alloc->mem_unmap_full = gst_memory_nvmm_unmap_full; + alloc->mem_copy = gst_memory_nvmm_copy; + alloc->mem_share = gst_memory_nvmm_share; + alloc->mem_is_span = gst_memory_nvmm_is_span; +} + +GType gst_nvmm_parent_meta_api_get_type (void); +#define GST_NVMM_PARENT_META_API_TYPE (gst_nvmm_parent_meta_api_get_type()) + +#define gst_buffer_get_nvmm_parent_meta(b) \ + ((GstNVMMParentMeta*)gst_buffer_get_meta((b),GST_NVMM_PARENT_META_API_TYPE)) + +const GstMetaInfo *gst_nvmm_parent_meta_get_info (void); +#define GST_NVMM_PARENT_META_INFO (gst_nvmm_parent_meta_get_info()) + +/* GstParentBufferMeta but supporting NULL and no copying to avoid accidentally + * introducing a circular reference when copying GstMeta's */ +struct _GstNVMMParentMeta +{ + GstMeta parent; + + GstBuffer *buffer; +}; +typedef struct _GstNVMMParentMeta GstNVMMParentMeta; + +static GstNVMMParentMeta * +gst_buffer_add_nvmm_parent_meta (GstBuffer * buffer, GstBuffer * ref) +{ + GstNVMMParentMeta *meta; + + g_return_val_if_fail (GST_IS_BUFFER (ref), NULL); + + meta = + (GstNVMMParentMeta *) gst_buffer_add_meta (buffer, + GST_NVMM_PARENT_META_INFO, NULL); + + if (!meta) + return NULL; + + if (ref) + meta->buffer = gst_buffer_ref (ref); + + return meta; +} + +static gboolean +_gst_nvmm_parent_meta_transform (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + return FALSE; +} + +static void +_gst_nvmm_parent_meta_free (GstNVMMParentMeta * parent_meta, GstBuffer * buffer) +{ + GST_DEBUG ("Dropping reference on buffer %p", parent_meta->buffer); + gst_clear_buffer (&parent_meta->buffer); +} + +static gboolean +_gst_nvmm_parent_meta_init (GstNVMMParentMeta * parent_meta, + gpointer params, GstBuffer * buffer) +{ + parent_meta->buffer = NULL; + + return TRUE; +} + +GType +gst_nvmm_parent_meta_api_get_type (void) +{ + static GType type = 0; + static const gchar *tags[] = { GST_META_TAG_MEMORY_STR, NULL }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstNVMMParentMetaAPI", tags); + g_once_init_leave (&type, _type); + } + + return type; +} + +const GstMetaInfo * +gst_nvmm_parent_meta_get_info (void) +{ + static const GstMetaInfo *meta_info = NULL; + + if (g_once_init_enter ((GstMetaInfo **) & meta_info)) { + const GstMetaInfo *meta = + gst_meta_register (gst_nvmm_parent_meta_api_get_type (), + "GstNVMMParentMeta", + sizeof (GstNVMMParentMeta), + (GstMetaInitFunction) _gst_nvmm_parent_meta_init, + (GstMetaFreeFunction) _gst_nvmm_parent_meta_free, + _gst_nvmm_parent_meta_transform); + g_once_init_leave ((GstMetaInfo **) & meta_info, (GstMetaInfo *) meta); + } + + return meta_info; +} + +static GstMiniObjectDisposeFunction parent_gst_buffer_dispose = NULL; + +static gboolean +gst_buffer_nvmm_dispose (GstMiniObject * obj) +{ + GstBuffer *buf = (GstBuffer *) obj; + GstNVMMParentMeta *nv_buf_meta = gst_buffer_get_nvmm_parent_meta (buf); + + GST_TRACE ("nvmm buffer dispose %p, parent_buf_meta %p", obj, nv_buf_meta); + if (nv_buf_meta && nv_buf_meta->buffer) { + GstNVMMParentMeta *gl_buf_meta; + + gl_buf_meta = gst_buffer_get_nvmm_parent_meta (nv_buf_meta->buffer); + if (gl_buf_meta && !gl_buf_meta->buffer) { + // reattache the NVMM buffer to the parent buffer + GST_LOG ("readding nvmm buffer %p %i, to glmemory buffer %p %i", buf, + GST_MINI_OBJECT_REFCOUNT_VALUE (buf), nv_buf_meta->buffer, + GST_MINI_OBJECT_REFCOUNT_VALUE (nv_buf_meta->buffer)); + gl_buf_meta->buffer = gst_buffer_ref (buf); + gst_clear_buffer (&nv_buf_meta->buffer); + return FALSE; + } + } + + return parent_gst_buffer_dispose (obj); +} + +struct _GstGLBufferPoolNVMMPrivate +{ + GstGLVideoAllocationParams *gl_params; +}; +typedef struct _GstGLBufferPoolNVMMPrivate GstGLBufferPoolNVMMPrivate; + +struct _GstGLBufferPoolNVMM +{ + GstGLBufferPool parent; +}; + +#define NVMM_POOL_GET_PRIV(obj) gst_gl_buffer_pool_nvmm_get_instance_private((GstGLBufferPoolNVMM *)(obj)); + +G_DECLARE_FINAL_TYPE (GstGLBufferPoolNVMM, gst_gl_buffer_pool_nvmm, GST, + GL_BUFFER_POOL_NVMM, GstGLBufferPool); +G_DEFINE_TYPE_WITH_CODE (GstGLBufferPoolNVMM, gst_gl_buffer_pool_nvmm, + GST_TYPE_GL_BUFFER_POOL, G_ADD_PRIVATE (GstGLBufferPoolNVMM)); + +static gboolean +gst_gl_buffer_pool_nvmm_set_config (GstBufferPool * pool, GstStructure * config) +{ + GstGLBufferPoolNVMMPrivate *priv; + GstGLBufferPool *glpool = GST_GL_BUFFER_POOL (pool); + GstGLVideoAllocationParams *parent_gl_params; + GstCaps *caps = NULL; + GstVideoInfo vinfo; + GstAllocationParams alloc_params; + + priv = NVMM_POOL_GET_PRIV (pool); + + if (!gst_buffer_pool_config_get_allocator (config, NULL, &alloc_params)) + goto wrong_config; + + if (!gst_buffer_pool_config_get_params (config, &caps, NULL, NULL, NULL)) + goto wrong_config; + + if (caps == NULL) + goto no_caps; + + /* now parse the caps from the config */ + if (!gst_video_info_from_caps (&vinfo, caps)) + goto wrong_caps; + + // TODO: fallback to regular GLMemory PBO/GetTexImage downloads? + if (GST_VIDEO_INFO_FORMAT (&vinfo) != GST_VIDEO_FORMAT_RGBA) + goto wrong_vformat; + + if (!GST_BUFFER_POOL_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->set_config + (pool, config)) + return FALSE; + + parent_gl_params = (GstGLVideoAllocationParams *) + gst_gl_buffer_pool_get_gl_allocation_params (glpool); + + if (priv->gl_params) + gst_gl_allocation_params_free ((GstGLAllocationParams *) priv->gl_params); + priv->gl_params = + gst_gl_video_allocation_params_new_wrapped_gl_handle + (parent_gl_params->parent.context, parent_gl_params->parent.alloc_params, + parent_gl_params->v_info, 0, parent_gl_params->valign, + parent_gl_params->target, parent_gl_params->tex_format, NULL, NULL, NULL); + + gst_buffer_pool_config_set_gl_allocation_params (config, + (GstGLAllocationParams *) priv->gl_params); + gst_gl_allocation_params_free ((GstGLAllocationParams *) parent_gl_params); + + if (!GST_BUFFER_POOL_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->set_config + (pool, config)) + return FALSE; + + return TRUE; + +wrong_config: + { + GST_WARNING_OBJECT (pool, "invalid config"); + return FALSE; + } +no_caps: + { + GST_WARNING_OBJECT (pool, "no caps in config"); + return FALSE; + } +wrong_caps: + { + GST_WARNING_OBJECT (pool, + "failed getting geometry from caps %" GST_PTR_FORMAT, caps); + return FALSE; + } +wrong_vformat: + { + GST_WARNING_OBJECT (pool, "This pool only deals with RGBA textures"); + return FALSE; + } +} + +static void +nv_buffer_egl_image_mem_unref (GstEGLImage * image, GstMemory * mem) +{ + GstGLDisplayEGL *egl_display = NULL; + EGLDisplay display; + + egl_display = gst_gl_display_egl_from_gl_display (image->context->display); + if (!egl_display) { + GST_ERROR ("Could not retrieve GstGLDisplayEGL from GstGLDisplay"); + return; + } + display = + (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display)); + + if (NvDestroyEGLImage (display, image->image)) { + GST_ERROR ("Failed to destroy EGLImage %p from NvBuffer", image->image); + } else { + GST_DEBUG ("destroyed EGLImage %p from NvBuffer", image->image); + } + + gst_memory_unref (mem); + gst_object_unref (egl_display); +} + +static GstFlowReturn +gst_gl_buffer_pool_nvmm_alloc (GstBufferPool * pool, GstBuffer ** outbuf, + GstBufferPoolAcquireParams * acquire_params) +{ + GstGLBufferPool *gl_pool = GST_GL_BUFFER_POOL (pool); + GstGLBufferPoolNVMMPrivate *priv; + GstFlowReturn ret = GST_FLOW_ERROR; + GstBuffer *downstream_buf = NULL; + GstMapInfo in_map_info = GST_MAP_INFO_INIT; + GstGLDisplayEGL *egl_display = NULL; + GstEGLImage *eglimage = NULL; + EGLDisplay display = EGL_NO_DISPLAY; + EGLImageKHR image = EGL_NO_IMAGE; + GstGLMemoryAllocator *allocator = NULL; + GstMemory *nvmm_mem = NULL; + int in_dmabuf_fd; + + priv = NVMM_POOL_GET_PRIV (pool); + + *outbuf = NULL; + downstream_buf = gst_buffer_new (); + if (!parent_gst_buffer_dispose) + parent_gst_buffer_dispose = ((GstMiniObject *) downstream_buf)->dispose; + ((GstMiniObject *) downstream_buf)->dispose = gst_buffer_nvmm_dispose; + + nvmm_mem = gst_allocator_nvmm_alloc (priv->gl_params->v_info); + if (!nvmm_mem) { + GST_WARNING_OBJECT (pool, "Failed to create NVMM GstMemory"); + return GST_FLOW_ERROR; + } + gst_buffer_append_memory (downstream_buf, nvmm_mem); + in_dmabuf_fd = ((GstMemoryNVMM *) nvmm_mem)->dmabuf_fd; + + egl_display = gst_gl_display_egl_from_gl_display (gl_pool->context->display); + if (!egl_display) { + GST_WARNING ("Failed to retrieve GstGLDisplayEGL from GstGLDisplay"); + goto done; + } + display = + (EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display)); + + image = NvEGLImageFromFd (display, in_dmabuf_fd); + if (!image) { + GST_DEBUG_OBJECT (pool, "Failed construct EGLImage " + "from NvBuffer fd %i", in_dmabuf_fd); + goto done; + } + GST_DEBUG_OBJECT (pool, "constructed EGLImage %p " + "from NvBuffer fd %i", image, in_dmabuf_fd); + + eglimage = gst_egl_image_new_wrapped (gl_pool->context, image, + GST_GL_RGBA, gst_memory_ref (nvmm_mem), + (GstEGLImageDestroyNotify) nv_buffer_egl_image_mem_unref); + if (!eglimage) { + GST_WARNING_OBJECT (pool, "Failed to wrap constructed " + "EGLImage from NvBuffer"); + goto done; + } + + gst_buffer_unmap (downstream_buf, &in_map_info); + in_map_info = (GstMapInfo) GST_MAP_INFO_INIT; + + allocator = + GST_GL_MEMORY_ALLOCATOR (gst_allocator_find + (GST_GL_MEMORY_EGL_ALLOCATOR_NAME)); + + /* TODO: buffer pool */ + *outbuf = gst_buffer_new (); + if (!gst_gl_memory_setup_buffer (allocator, *outbuf, priv->gl_params, + NULL, (gpointer *) & eglimage, 1)) { + GST_WARNING_OBJECT (pool, "Failed to setup NVMM -> EGLImage buffer"); + goto done; + } + + gst_egl_image_unref (eglimage); + + /* TODO: NvBuffer has some sync functions that may be more useful here */ + { + GstGLSyncMeta *sync_meta; + + sync_meta = gst_buffer_add_gl_sync_meta (gl_pool->context, *outbuf); + if (sync_meta) { + gst_gl_sync_meta_set_sync_point (sync_meta, gl_pool->context); + } + } + + // possible circular reference here + gst_buffer_add_nvmm_parent_meta (*outbuf, downstream_buf); + gst_buffer_unref (downstream_buf); + + ret = GST_FLOW_OK; + +done: + if (in_map_info.memory) + gst_buffer_unmap (downstream_buf, &in_map_info); + + gst_clear_object (&egl_display); + gst_clear_object (&allocator); + + return ret; +} + +static void +gst_gl_buffer_pool_nvmm_finalize (GObject * object) +{ + GstGLBufferPoolNVMMPrivate *priv = NVMM_POOL_GET_PRIV (object); + + if (priv->gl_params) + gst_gl_allocation_params_free ((GstGLAllocationParams *) priv->gl_params); + priv->gl_params = NULL; + + G_OBJECT_CLASS (gst_gl_buffer_pool_nvmm_parent_class)->finalize (object); +} + +static void +gst_gl_buffer_pool_nvmm_init (GstGLBufferPoolNVMM * pool) +{ +} + +static void +gst_gl_buffer_pool_nvmm_class_init (GstGLBufferPoolNVMMClass * klass) +{ + GstBufferPoolClass *pool_class = (GstBufferPoolClass *) klass; + GObjectClass *gobject_class = (GObjectClass *) klass; + + pool_class->set_config = gst_gl_buffer_pool_nvmm_set_config; + pool_class->alloc_buffer = gst_gl_buffer_pool_nvmm_alloc; + + gobject_class->finalize = gst_gl_buffer_pool_nvmm_finalize; +} + +static GstBufferPool * +gst_gl_buffer_pool_nvmm_new (GstGLContext * context) +{ + GstGLBufferPoolNVMM *pool; + GstGLBufferPool *gl_pool; + + pool = g_object_new (gst_gl_buffer_pool_nvmm_get_type (), NULL); + gst_object_ref_sink (pool); + gl_pool = GST_GL_BUFFER_POOL (pool); + gl_pool->context = gst_object_ref (context); + + GST_LOG_OBJECT (pool, "new NVMM GL buffer pool for context %" GST_PTR_FORMAT, + context); + + return GST_BUFFER_POOL_CAST (pool); +} +#endif /* GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) */ + #define gst_gl_download_element_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLDownloadElement, gst_gl_download_element, GST_TYPE_GL_BASE_FILTER, @@ -57,25 +793,37 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt, GstBuffer * buffer, GstBuffer ** outbuf); static GstFlowReturn gst_gl_download_element_transform (GstBaseTransform * bt, GstBuffer * buffer, GstBuffer * outbuf); +static gboolean gst_gl_download_element_transform_meta (GstBaseTransform * bt, + GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf); static gboolean gst_gl_download_element_decide_allocation (GstBaseTransform * trans, GstQuery * query); static gboolean gst_gl_download_element_sink_event (GstBaseTransform * bt, GstEvent * event); static gboolean gst_gl_download_element_src_event (GstBaseTransform * bt, GstEvent * event); +static gboolean gst_gl_download_element_propose_allocation (GstBaseTransform * + bt, GstQuery * decide_query, GstQuery * query); static void gst_gl_download_element_finalize (GObject * object); -#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF -#define EXTRA_CAPS_TEMPLATE "video/x-raw(" GST_CAPS_FEATURE_MEMORY_DMABUF "); " +#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM" + +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) +#define EXTRA_CAPS_TEMPLATE1 "video/x-raw(" GST_CAPS_FEATURE_MEMORY_NVMM "), format=(string)RGBA; " #else -#define EXTRA_CAPS_TEMPLATE +#define EXTRA_CAPS_TEMPLATE1 +#endif + +#if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF +#define EXTRA_CAPS_TEMPLATE2 "video/x-raw(" GST_CAPS_FEATURE_MEMORY_DMABUF "); " +#else +#define EXTRA_CAPS_TEMPLATE2 #endif static GstStaticPadTemplate gst_gl_download_element_src_pad_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS (EXTRA_CAPS_TEMPLATE + GST_STATIC_CAPS (EXTRA_CAPS_TEMPLATE1 EXTRA_CAPS_TEMPLATE2 "video/x-raw; video/x-raw(memory:GLMemory)")); static GstStaticPadTemplate gst_gl_download_element_sink_pad_template = @@ -103,6 +851,8 @@ gst_gl_download_element_class_init (GstGLDownloadElementClass * klass) bt_class->decide_allocation = gst_gl_download_element_decide_allocation; bt_class->sink_event = gst_gl_download_element_sink_event; bt_class->src_event = gst_gl_download_element_src_event; + bt_class->propose_allocation = gst_gl_download_element_propose_allocation; + bt_class->transform_meta = gst_gl_download_element_transform_meta; bt_class->passthrough_on_same_caps = TRUE; @@ -166,14 +916,23 @@ gst_gl_download_element_set_caps (GstBaseTransform * bt, GstCaps * in_caps, if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) { dl->mode = GST_GL_DOWNLOAD_MODE_PASSTHROUGH; + GST_INFO_OBJECT (dl, "caps signal passthrough"); +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) + } else if (gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_NVMM)) { + dl->mode = GST_GL_DOWNLOAD_MODE_NVMM; + GST_INFO_OBJECT (dl, "caps signal NVMM"); +#endif #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF } else if (g_atomic_int_get (&dl->try_dmabuf_exports) && gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_DMABUF)) { dl->mode = GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS; + GST_INFO_OBJECT (dl, "caps signal dma-buf export"); #endif } else { /* System Memory */ dl->mode = GST_GL_DOWNLOAD_MODE_PBO_TRANSFERS; + GST_INFO_OBJECT (dl, "caps signal sysmem download"); } return TRUE; @@ -221,6 +980,13 @@ gst_gl_download_element_transform_caps (GstBaseTransform * bt, GstCaps *newcaps; tmp = gst_caps_ref (caps); +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) + newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_NVMM); + _remove_field (newcaps, "texture-target"); + // FIXME: RGBA-only? + tmp = gst_caps_merge (tmp, newcaps); +#endif + #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF newcaps = _set_caps_features (caps, GST_CAPS_FEATURE_MEMORY_DMABUF); _remove_field (newcaps, "texture-target"); @@ -461,10 +1227,57 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer ** outbuf) { GstGLDownloadElement *dl = GST_GL_DOWNLOAD_ELEMENT (bt); + GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (bt); gint i, n; *outbuf = inbuf; + (void) bclass; + +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) + if (dl->mode == GST_GL_DOWNLOAD_MODE_NVMM) { + GstNVMMParentMeta *buf_meta = gst_buffer_get_nvmm_parent_meta (inbuf); + GstMemory *mem; + GstMemoryNVMM *nvmm_mem; + + if (!buf_meta || !buf_meta->buffer) { + // TODO: remove this restriction with an e.g. copy... + GST_ERROR_OBJECT (dl, + "Cannot push upstream created buffer when outputting NVMM"); + return GST_FLOW_ERROR; + } + + if (!(mem = gst_buffer_peek_memory (buf_meta->buffer, 0))) { + GST_ERROR_OBJECT (dl, "No memory in buffer?"); + return GST_FLOW_ERROR; + } + + if (!gst_is_memory_nvmm (mem)) { + GST_ERROR_OBJECT (dl, + "Upstream buffer does not contain an attached NVMM GstMemory"); + return GST_FLOW_ERROR; + } + nvmm_mem = (GstMemoryNVMM *) mem; + + /* switch up the parent buffer references so that when the NVMM buffer is + * released, the associated EGLImage/OpenGL texture is as well + */ + GST_DEBUG_OBJECT (dl, "NVMM buffer fd:%i passed through %" GST_PTR_FORMAT, + nvmm_mem->dmabuf_fd, buf_meta->buffer); + *outbuf = buf_meta->buffer; + bclass->copy_metadata (bt, inbuf, *outbuf); + buf_meta->buffer = NULL; + buf_meta = gst_buffer_get_nvmm_parent_meta (*outbuf); + if (!buf_meta) { + buf_meta = gst_buffer_add_nvmm_parent_meta (*outbuf, inbuf); + } else { + gst_clear_buffer (&buf_meta->buffer); + buf_meta->buffer = gst_buffer_ref (inbuf); + } + + return GST_FLOW_OK; + } +#endif #if GST_GL_HAVE_PLATFORM_EGL && GST_GL_HAVE_DMABUF if (dl->mode == GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS) { GstBuffer *buffer = _try_export_dmabuf (dl, inbuf); @@ -529,6 +1342,19 @@ gst_gl_download_element_transform (GstBaseTransform * bt, return GST_FLOW_OK; } +static gboolean +gst_gl_download_element_transform_meta (GstBaseTransform * bt, + GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf) +{ + if (g_type_is_a (meta->info->api, GST_GL_SYNC_META_API_TYPE)) { + GST_LOG_OBJECT (bt, "not copying GstGLSyncMeta onto output buffer"); + return FALSE; + } + + return GST_BASE_TRANSFORM_CLASS (parent_class)->transform_meta (bt, outbuf, + meta, inbuf); +} + static gboolean gst_gl_download_element_decide_allocation (GstBaseTransform * trans, GstQuery * query) @@ -569,6 +1395,83 @@ gst_gl_download_element_src_event (GstBaseTransform * bt, GstEvent * event) return GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (bt, event); } +static gboolean +gst_gl_download_element_propose_allocation (GstBaseTransform * bt, + GstQuery * decide_query, GstQuery * query) +{ + GstBufferPool *pool = NULL; + GstCaps *caps; + GstGLContext *context; + GstStructure *config; + GstVideoInfo info; + gsize size; + + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (bt, + decide_query, query)) + return FALSE; + + gst_query_parse_allocation (query, &caps, NULL); + if (caps == NULL) + goto invalid_caps; + + context = GST_GL_BASE_FILTER (bt)->context; + if (!context) { + GST_ERROR_OBJECT (context, "got no GLContext"); + return FALSE; + } + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + +#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM) + if (!pool && decide_query) { + GstCaps *decide_caps; + + gst_query_parse_allocation (decide_query, &decide_caps, NULL); + if (decide_caps && gst_caps_get_size (decide_caps) > 0) { + GstCapsFeatures *features = gst_caps_get_features (decide_caps, 0); + + if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_NVMM)) { + pool = gst_gl_buffer_pool_nvmm_new (context); + GST_INFO_OBJECT (bt, "have NVMM downstream, proposing NVMM " + "pool %" GST_PTR_FORMAT, pool); + } + } + } +#endif + if (!pool) { + pool = gst_gl_buffer_pool_new (context); + } + config = gst_buffer_pool_get_config (pool); + + /* the normal size of a frame */ + size = info.size; + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_GL_SYNC_META); + + if (!gst_buffer_pool_set_config (pool, config)) { + gst_object_unref (pool); + goto config_failed; + } + gst_query_add_allocation_pool (query, pool, size, 1, 0); + + gst_object_unref (pool); + return TRUE; + +invalid_caps: + { + GST_ERROR_OBJECT (bt, "Invalid Caps specified"); + return FALSE; + } +config_failed: + { + GST_ERROR_OBJECT (bt, "failed setting config"); + return FALSE; + } +} + + static void gst_gl_download_element_finalize (GObject * object) { diff --git a/ext/gl/gstgldownloadelement.h b/ext/gl/gstgldownloadelement.h index 93bfd50c75..7ae7c44ecd 100644 --- a/ext/gl/gstgldownloadelement.h +++ b/ext/gl/gstgldownloadelement.h @@ -43,7 +43,8 @@ typedef enum { GST_GL_DOWNLOAD_MODE_PASSTHROUGH, GST_GL_DOWNLOAD_MODE_PBO_TRANSFERS, - GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS + GST_GL_DOWNLOAD_MODE_DMABUF_EXPORTS, + GST_GL_DOWNLOAD_MODE_NVMM, } GstGlDownloadMode; struct _GstGLDownloadElement diff --git a/ext/gl/meson.build b/ext/gl/meson.build index a0012147b6..7175f0f59b 100644 --- a/ext/gl/meson.build +++ b/ext/gl/meson.build @@ -133,10 +133,17 @@ if ['darwin', 'ios'].contains(host_system) gl_objc_args += ['-fobjc-arc'] endif +extra_c_args = [] +# gl_nvmm and nvbuf_utils_dep are set in gst-libs/gst/gl/meson.build +if gl_nvmm + extra_c_args += ['-DHAVE_NVMM'] + optional_deps += [nvbuf_utils_dep] +endif + gstopengl = library('gstopengl', opengl_sources, - c_args : gst_plugins_base_args, - objc_args : gst_plugins_base_args + gl_objc_args, + c_args : gst_plugins_base_args + extra_c_args, + objc_args : gst_plugins_base_args + gl_objc_args + extra_c_args, link_args : noseh_link_args, include_directories : [configinc], dependencies : [gstgl_dep, video_dep, diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build index 3d7b177c14..80278d3a6e 100644 --- a/gst-libs/gst/gl/meson.build +++ b/gst-libs/gst/gl/meson.build @@ -923,6 +923,8 @@ if host_system == 'android' and need_win_android != 'no' and need_platform_egl ! endif endif +gl_nvmm = false +nvbuf_utils_dep = unneeded_dep if egl_dep.found() # XXX: provide options for this? # c_args and c_link_args can also cover this case just fine e.g.: @@ -931,6 +933,7 @@ if egl_dep.found() nvbuf_utils_h = cc.has_header('nvbuf_utils.h') if nvbuf_utils_dep.found() and nvbuf_utils_h gl_misc_deps += [nvbuf_utils_dep] + gl_nvmm = true gl_cpp_args += ['-DHAVE_NVMM'] endif endif