glupload: add support for uploading memory:NVMM buffers

Currently RGBA-only.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/1071>
This commit is contained in:
Matthew Waters 2021-02-26 18:48:09 +11:00
parent f770982635
commit 2f35aeca8c
2 changed files with 497 additions and 3 deletions

View file

@ -564,7 +564,6 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
gst_caps_append (ret, tmp);
n = gst_caps_get_size (ret);
for (i = 0; i < n; i++) {
GstStructure *s = gst_caps_get_structure (ret, i);
@ -1730,6 +1729,487 @@ static const UploadMethod _directviv_upload = {
#endif /* GST_GL_HAVE_VIV_DIRECTVIV */
#if defined(HAVE_NVMM)
#include "nvbuf_utils.h"
struct NVMMUpload
{
GstGLUpload *upload;
GstGLVideoAllocationParams *params;
guint n_mem;
GstGLTextureTarget target;
GstVideoInfo out_info;
/* only used for pointer comparison */
gpointer out_caps;
};
#define GST_CAPS_FEATURE_MEMORY_NVMM "memory:NVMM"
/* FIXME: other formats? */
static GstStaticCaps _nvmm_upload_caps =
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_NVMM,
"RGBA"));
static gpointer
_nvmm_upload_new (GstGLUpload * upload)
{
struct NVMMUpload *nvmm = g_new0 (struct NVMMUpload, 1);
nvmm->upload = upload;
nvmm->target = GST_GL_TEXTURE_TARGET_EXTERNAL_OES;
return nvmm;
}
static GstCaps *
_nvmm_upload_transform_caps (gpointer impl, GstGLContext * context,
GstPadDirection direction, GstCaps * caps)
{
struct NVMMUpload *nvmm = impl;
GstCapsFeatures *passthrough;
GstCaps *ret;
if (context) {
const GstGLFuncs *gl = context->gl_vtable;
if (!gl->EGLImageTargetTexture2D)
return NULL;
/* Don't propose NVMM caps feature unless it can be supported */
if (gst_gl_context_get_gl_platform (context) != GST_GL_PLATFORM_EGL)
return NULL;
if (!gst_gl_context_check_feature (context, "EGL_KHR_image_base"))
return NULL;
}
passthrough = gst_caps_features_from_string
(GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
if (direction == GST_PAD_SINK) {
GstCaps *tmp;
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY, passthrough);
tmp =
_caps_intersect_texture_target (ret,
1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES);
gst_caps_unref (ret);
ret = tmp;
} else {
gint i, n;
ret =
_set_caps_features_with_passthrough (caps,
GST_CAPS_FEATURE_MEMORY_NVMM, passthrough);
n = gst_caps_get_size (ret);
for (i = 0; i < n; i++) {
GstStructure *s = gst_caps_get_structure (ret, i);
gst_structure_remove_fields (s, "texture-target", NULL);
}
}
gst_caps_features_free (passthrough);
GST_DEBUG_OBJECT (nvmm->upload, "transformed %" GST_PTR_FORMAT " into %"
GST_PTR_FORMAT, caps, ret);
return ret;
}
static gboolean
_nvmm_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
GstCaps * out_caps)
{
struct NVMMUpload *nvmm = impl;
GstVideoInfo *in_info = &nvmm->upload->priv->in_info;
GstVideoInfo *out_info = &nvmm->out_info;
GstVideoMeta *meta;
GstMapInfo in_map_info = GST_MAP_INFO_INIT;
guint n_mem;
guint i;
n_mem = gst_buffer_n_memory (buffer);
if (n_mem != 1) {
GST_DEBUG_OBJECT (nvmm->upload, "NVMM uploader only supports "
"1 memory, not %u", n_mem);
return FALSE;
}
meta = gst_buffer_get_video_meta (buffer);
if (!nvmm->upload->context->gl_vtable->EGLImageTargetTexture2D)
return FALSE;
/* NVMM upload is only supported with EGL contexts. */
if (gst_gl_context_get_gl_platform (nvmm->upload->context) !=
GST_GL_PLATFORM_EGL)
return FALSE;
if (!gst_gl_context_check_feature (nvmm->upload->context,
"EGL_KHR_image_base"))
return FALSE;
if (!gst_buffer_map (buffer, &in_map_info, GST_MAP_READ)) {
GST_DEBUG_OBJECT (nvmm->upload, "Failed to map readonly NvBuffer");
return FALSE;
}
if (in_map_info.size != NvBufferGetSize ()) {
GST_DEBUG_OBJECT (nvmm->upload, "Memory size (%" G_GSIZE_FORMAT ") is "
"not the same as what NvBuffer advertises (%u)", in_map_info.size,
NvBufferGetSize ());
gst_buffer_unmap (buffer, &in_map_info);
return FALSE;
}
gst_buffer_unmap (buffer, &in_map_info);
/* Update video info based on video meta */
if (meta) {
in_info->width = meta->width;
in_info->height = meta->height;
for (i = 0; i < meta->n_planes; i++) {
in_info->offset[i] = meta->offset[i];
in_info->stride[i] = meta->stride[i];
}
}
if (out_caps != nvmm->out_caps) {
nvmm->out_caps = out_caps;
if (!gst_video_info_from_caps (out_info, out_caps))
return FALSE;
}
if (nvmm->params)
gst_gl_allocation_params_free ((GstGLAllocationParams *) nvmm->params);
if (!(nvmm->params =
gst_gl_video_allocation_params_new_wrapped_gl_handle (nvmm->
upload->context, NULL, out_info, -1, NULL, nvmm->target, 0, NULL,
NULL, NULL))) {
return FALSE;
}
return TRUE;
}
static void
_nvmm_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
GstQuery * query)
{
/* nothing to do for now. */
}
static void
_egl_image_mem_unref (GstEGLImage * image, GstMemory * mem)
{
GstGLDisplayEGL *egl_display = NULL;
EGLDisplay display;
egl_display = gst_gl_display_egl_from_gl_display (image->context->display);
if (!egl_display) {
GST_ERROR ("Could not retrieve GstGLDisplayEGL from GstGLDisplay");
return;
}
display =
(EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
if (NvDestroyEGLImage (display, image->image)) {
GST_ERROR ("Failed to destroy EGLImage %p from NvBuffer", image->image);
} else {
GST_DEBUG ("destroyed EGLImage %p from NvBuffer", image->image);
}
gst_memory_unref (mem);
gst_object_unref (egl_display);
}
static const char *
payload_type_to_string (NvBufferPayloadType ptype)
{
switch (ptype) {
case NvBufferPayload_SurfArray:
return "SurfArray";
case NvBufferPayload_MemHandle:
return "MemHandle";
default:
return "<unknown>";
}
}
static const char *
pixel_format_to_string (NvBufferColorFormat fmt)
{
switch (fmt) {
case NvBufferColorFormat_YUV420:
return "YUV420";
case NvBufferColorFormat_YVU420:
return "YVU420";
case NvBufferColorFormat_YUV422:
return "YUV422";
case NvBufferColorFormat_YUV420_ER:
return "YUV420_ER";
case NvBufferColorFormat_YVU420_ER:
return "YVU420_ER";
case NvBufferColorFormat_NV12:
return "NV12";
case NvBufferColorFormat_NV12_ER:
return "NV12_ER";
case NvBufferColorFormat_NV21:
return "NV21";
case NvBufferColorFormat_NV21_ER:
return "NV21_ER";
case NvBufferColorFormat_UYVY:
return "UYVY";
case NvBufferColorFormat_UYVY_ER:
return "UYVY_ER";
case NvBufferColorFormat_VYUY:
return "VYUY";
case NvBufferColorFormat_VYUY_ER:
return "VYUY_ER";
case NvBufferColorFormat_YUYV:
return "YUYV";
case NvBufferColorFormat_YUYV_ER:
return "YUYV_ER";
case NvBufferColorFormat_YVYU:
return "YVYU";
case NvBufferColorFormat_YVYU_ER:
return "YVYU_ER";
case NvBufferColorFormat_ABGR32:
return "ABGR32";
case NvBufferColorFormat_XRGB32:
return "XRGB32";
case NvBufferColorFormat_ARGB32:
return "ARGB32";
case NvBufferColorFormat_NV12_10LE:
return "NV12_10LE";
case NvBufferColorFormat_NV12_10LE_709:
return "NV12_10LE_709";
case NvBufferColorFormat_NV12_10LE_709_ER:
return "NV12_10LE_709_ER";
case NvBufferColorFormat_NV12_10LE_2020:
return "NV12_2020";
case NvBufferColorFormat_NV21_10LE:
return "NV21_10LE";
case NvBufferColorFormat_NV12_12LE:
return "NV12_12LE";
case NvBufferColorFormat_NV12_12LE_2020:
return "NV12_12LE_2020";
case NvBufferColorFormat_NV21_12LE:
return "NV21_12LE";
case NvBufferColorFormat_YUV420_709:
return "YUV420_709";
case NvBufferColorFormat_YUV420_709_ER:
return "YUV420_709_ER";
case NvBufferColorFormat_NV12_709:
return "NV12_709";
case NvBufferColorFormat_NV12_709_ER:
return "NV12_709_ER";
case NvBufferColorFormat_YUV420_2020:
return "YUV420_2020";
case NvBufferColorFormat_NV12_2020:
return "NV12_2020";
case NvBufferColorFormat_SignedR16G16:
return "SignedR16G16";
case NvBufferColorFormat_A32:
return "A32";
case NvBufferColorFormat_YUV444:
return "YUV444";
case NvBufferColorFormat_GRAY8:
return "GRAY8";
case NvBufferColorFormat_NV16:
return "NV16";
case NvBufferColorFormat_NV16_10LE:
return "NV16_10LE";
case NvBufferColorFormat_NV24:
return "NV24";
case NvBufferColorFormat_NV16_ER:
return "NV16_ER";
case NvBufferColorFormat_NV24_ER:
return "NV24_ER";
case NvBufferColorFormat_NV16_709:
return "NV16_709";
case NvBufferColorFormat_NV24_709:
return "NV24_709";
case NvBufferColorFormat_NV16_709_ER:
return "NV16_709_ER";
case NvBufferColorFormat_NV24_709_ER:
return "NV24_709_ER";
case NvBufferColorFormat_NV24_10LE_709:
return "NV24_10LE_709";
case NvBufferColorFormat_NV24_10LE_709_ER:
return "NV24_10LE_709_ER";
case NvBufferColorFormat_NV24_10LE_2020:
return "NV24_10LE_2020";
case NvBufferColorFormat_NV24_12LE_2020:
return "NV24_12LE_2020";
case NvBufferColorFormat_RGBA_10_10_10_2_709:
return "RGBA_10_10_10_2_709";
case NvBufferColorFormat_RGBA_10_10_10_2_2020:
return "RGBA_10_10_10_2_2020";
case NvBufferColorFormat_BGRA_10_10_10_2_709:
return "BGRA_10_10_10_2_709";
case NvBufferColorFormat_BGRA_10_10_10_2_2020:
return "BGRA_10_10_10_2_2020";
case NvBufferColorFormat_Invalid:
return "Invalid";
default:
return "<unknown>";
}
}
static void
dump_nv_buf_params (GstObject * debug_object, NvBufferParamsEx * params)
{
GST_DEBUG_OBJECT (debug_object, "nvbuffer fd: %u size %i nv_buffer: %p of "
"size %u, payload: (0x%x) %s, pixel format: (0x%x) %s, n_planes: %u, "
"plane 0 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
"plane 1 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u } "
"plane 2 { wxh: %ux%u, pitch: %u, offset: %u, psize: %u, layout: %u }",
params->params.dmabuf_fd, params->params.memsize,
params->params.nv_buffer, params->params.nv_buffer_size,
params->params.payloadType,
payload_type_to_string (params->params.payloadType),
params->params.pixel_format,
pixel_format_to_string (params->params.pixel_format),
params->params.num_planes, params->params.width[0],
params->params.height[0], params->params.pitch[0],
params->params.offset[0], params->params.psize[0],
params->params.offset[0], params->params.width[1],
params->params.height[1], params->params.pitch[1],
params->params.offset[1], params->params.psize[1],
params->params.offset[1], params->params.width[2],
params->params.height[2], params->params.pitch[2],
params->params.offset[2], params->params.psize[2],
params->params.offset[2]);
}
static GstGLUploadReturn
_nvmm_upload_perform (gpointer impl, GstBuffer * buffer, GstBuffer ** outbuf)
{
struct NVMMUpload *nvmm = impl;
GstGLMemoryAllocator *allocator = NULL;
GstMapInfo in_map_info = GST_MAP_INFO_INIT;
GstGLDisplayEGL *egl_display = NULL;
GstEGLImage *eglimage = NULL;
EGLDisplay display = EGL_NO_DISPLAY;
EGLImageKHR image = EGL_NO_IMAGE;
int in_dmabuf_fd;
NvBufferParamsEx params = { 0, };
GstGLUploadReturn ret = GST_GL_UPLOAD_ERROR;
if (!gst_buffer_map (buffer, &in_map_info, GST_MAP_READ)) {
GST_DEBUG_OBJECT (nvmm->upload, "Failed to map readonly NvBuffer");
goto done;
}
if (ExtractFdFromNvBuffer (in_map_info.data, &in_dmabuf_fd)) {
GST_DEBUG_OBJECT (nvmm->upload, "Failed to extract fd from NvBuffer");
goto done;
}
if (NvBufferGetParamsEx (in_dmabuf_fd, &params)) {
GST_WARNING_OBJECT (nvmm->upload, "Failed to get NvBuffer params");
goto done;
}
dump_nv_buf_params ((GstObject *) nvmm->upload, &params);
egl_display =
gst_gl_display_egl_from_gl_display (nvmm->upload->context->display);
if (!egl_display) {
GST_WARNING ("Failed to retrieve GstGLDisplayEGL from GstGLDisplay");
goto done;
}
display =
(EGLDisplay) gst_gl_display_get_handle (GST_GL_DISPLAY (egl_display));
image = NvEGLImageFromFd (display, in_dmabuf_fd);
if (!image) {
GST_DEBUG_OBJECT (nvmm->upload, "Failed construct EGLImage "
"from NvBuffer fd %i", in_dmabuf_fd);
goto done;
}
GST_DEBUG_OBJECT (nvmm->upload, "constructed EGLImage %p "
"from NvBuffer fd %i", image, in_dmabuf_fd);
eglimage = gst_egl_image_new_wrapped (nvmm->upload->context, image,
GST_GL_RGBA, gst_memory_ref (in_map_info.memory),
(GstEGLImageDestroyNotify) _egl_image_mem_unref);
if (!eglimage) {
GST_WARNING_OBJECT (nvmm->upload, "Failed to wrap constructed "
"EGLImage from NvBuffer");
goto done;
}
gst_buffer_unmap (buffer, &in_map_info);
in_map_info = (GstMapInfo) GST_MAP_INFO_INIT;
allocator =
GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
(GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
/* TODO: buffer pool */
*outbuf = gst_buffer_new ();
if (!gst_gl_memory_setup_buffer (allocator, *outbuf, nvmm->params,
NULL, (gpointer *) & eglimage, 1)) {
GST_WARNING_OBJECT (nvmm->upload, "Failed to setup "
"NVMM -> EGLImage buffer");
goto done;
}
gst_egl_image_unref (eglimage);
gst_buffer_add_parent_buffer_meta (*outbuf, buffer);
/* TODO: NvBuffer has some sync functions that may be more useful here */
{
GstGLSyncMeta *sync_meta;
sync_meta = gst_buffer_add_gl_sync_meta (nvmm->upload->context, *outbuf);
if (sync_meta) {
gst_gl_sync_meta_set_sync_point (sync_meta, nvmm->upload->context);
}
}
ret = GST_GL_UPLOAD_DONE;
done:
if (in_map_info.memory)
gst_buffer_unmap (buffer, &in_map_info);
gst_clear_object (&egl_display);
gst_clear_object (&allocator);
return ret;
}
static void
_nvmm_upload_free (gpointer impl)
{
struct NVMMUpload *nvmm = impl;
if (nvmm->params)
gst_gl_allocation_params_free ((GstGLAllocationParams *) nvmm->params);
g_free (impl);
}
static const UploadMethod _nvmm_upload = {
"NVMM",
0,
&_nvmm_upload_caps,
&_nvmm_upload_new,
&_nvmm_upload_transform_caps,
&_nvmm_upload_accept,
&_nvmm_upload_propose_allocation,
&_nvmm_upload_perform,
&_nvmm_upload_free
};
#endif /* HAVE_NVMM */
static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
#if GST_GL_HAVE_DMABUF
&_direct_dma_buf_upload,
@ -1739,6 +2219,9 @@ static const UploadMethod *upload_methods[] = { &_gl_memory_upload,
#if GST_GL_HAVE_VIV_DIRECTVIV
&_directviv_upload,
#endif
#if defined(HAVE_NVMM)
&_nvmm_upload,
#endif /* HAVE_NVMM */
&_upload_meta_upload, &_raw_data_upload
};
@ -2067,6 +2550,8 @@ restart:
ret =
upload->priv->method->perform (upload->priv->method_impl, buffer,
&outbuf);
GST_LOG_OBJECT (upload, "uploader %s returned %u, buffer: %p",
upload->priv->method->name, ret, outbuf);
if (ret == GST_GL_UPLOAD_UNSHARED_GL_CONTEXT) {
gint i;

View file

@ -923,8 +923,17 @@ if host_system == 'android' and need_win_android != 'no' and need_platform_egl !
endif
endif
# TODO: Add rest of gl config here.
# iOS, specific support
if egl_dep.found()
# XXX: provide options for this?
# c_args and c_link_args can also cover this case just fine e.g.:
# -Dc_args='-I/usr/src/jetson_multimedia_api/include' -Dc_link_args='-L/usr/lib/aarch64-linux-gnu/tegra/'
nvbuf_utils_dep = cc.find_library('nvbuf_utils', required : false)
nvbuf_utils_h = cc.has_header('nvbuf_utils.h')
if nvbuf_utils_dep.found() and nvbuf_utils_h
gl_misc_deps += [nvbuf_utils_dep]
gl_cpp_args += ['-DHAVE_NVMM']
endif
endif
build_gstgl = true
if enabled_gl_apis.length() == 0