gstreamer/sys/msdk/gstmsdkvideomemory.c
Haihao Xiang 6750123d5c msdk: set correct parameters for BGRx frame
Otherwise when mapping BGRx frame onto CPU's memory, CPU will get wrong
data for B, G, R components

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2205>
2021-04-30 02:12:46 +00:00

589 lines
17 KiB
C

/* GStreamer Intel MSDK plugin
* Copyright (c) 2018, Intel Corporation
* Copyright (c) 2018, Igalia S.L.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _WIN32
#include <unistd.h>
#include <va/va.h>
#endif
#include <stdlib.h>
#include "gstmsdkvideomemory.h"
#include "gstmsdkallocator.h"
#define GST_MSDK_BUFFER_SURFACE gst_msdk_buffer_surface_quark_get ()
static GQuark
gst_msdk_buffer_surface_quark_get (void)
{
static gsize g_quark;
if (g_once_init_enter (&g_quark)) {
gsize quark = (gsize) g_quark_from_static_string ("GstMsdkBufferSurface");
g_once_init_leave (&g_quark, quark);
}
return g_quark;
}
static mfxFrameSurface1 *
gst_msdk_video_allocator_get_surface (GstAllocator * allocator)
{
mfxFrameInfo frame_info = { {0,}, 0, };
mfxFrameSurface1 *surface;
GstMsdkContext *context = NULL;
mfxFrameAllocResponse *resp = NULL;
GstVideoInfo *vinfo = NULL;
if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
context = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->context;
resp = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->alloc_response;
vinfo = &GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->image_info;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
context = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->context;
resp = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->alloc_response;
vinfo = &GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->image_info;
} else {
return NULL;
}
surface = gst_msdk_context_get_surface_available (context, resp);
if (!surface) {
GST_ERROR ("failed to get surface available");
return NULL;
}
gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, vinfo);
surface->Info = frame_info;
return surface;
}
gboolean
gst_msdk_video_memory_get_surface_available (GstMemory * mem)
{
GstAllocator *allocator;
mfxFrameSurface1 *surface;
g_return_val_if_fail (mem, FALSE);
allocator = mem->allocator;
surface = gst_msdk_video_allocator_get_surface (allocator);
if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = surface;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, surface, NULL);
}
return surface ? TRUE : FALSE;
}
/*
* Every time releasing a gst buffer, we need to check the status of surface's lock,
* so that we could manage locked surfaces separately in the context.
* Otherwise, we put the surface to the available list.
*/
void
gst_msdk_video_memory_release_surface (GstMemory * mem)
{
mfxFrameSurface1 *surface = NULL;
GstMsdkContext *context = NULL;
mfxFrameAllocResponse *alloc_response = NULL;
g_return_if_fail (mem);
if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator)) {
surface = GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface;
context = GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->context;
alloc_response =
GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->alloc_response;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator)) {
surface =
gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
GST_MSDK_BUFFER_SURFACE);
context = GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->context;
alloc_response =
GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->alloc_response;
} else {
return;
}
if (surface->Data.Locked > 0)
gst_msdk_context_put_surface_locked (context, alloc_response, surface);
else
gst_msdk_context_put_surface_available (context, alloc_response, surface);
if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator))
GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = NULL;
else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator))
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, NULL, NULL);
return;
}
GstMemory *
gst_msdk_video_memory_new (GstAllocator * base_allocator)
{
GstMsdkVideoAllocator *allocator;
GstVideoInfo *vip;
GstMsdkVideoMemory *mem;
g_return_val_if_fail (base_allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_VIDEO_ALLOCATOR (base_allocator), NULL);
allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_allocator);
mem = g_slice_new0 (GstMsdkVideoMemory);
if (!mem)
return NULL;
mem->surface = gst_msdk_video_allocator_get_surface (base_allocator);
if (!mem->surface) {
g_slice_free (GstMsdkVideoMemory, mem);
return NULL;
}
vip = &allocator->image_info;
gst_memory_init (&mem->parent_instance, 0,
base_allocator, NULL, GST_VIDEO_INFO_SIZE (vip), 0, 0,
GST_VIDEO_INFO_SIZE (vip));
return GST_MEMORY_CAST (mem);
}
gboolean
gst_video_meta_map_msdk_memory (GstVideoMeta * meta, guint plane,
GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags)
{
gboolean ret = FALSE;
GstAllocator *allocator;
GstMsdkVideoAllocator *msdk_video_allocator;
GstMsdkVideoMemory *mem =
GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
GstMsdkMemoryID *mem_id;
guint offset = 0;
gint pitch = 0;
guint plane_id = plane;
g_return_val_if_fail (mem, FALSE);
allocator = GST_MEMORY_CAST (mem)->allocator;
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
if (!GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
GST_WARNING ("The allocator is not MSDK video allocator");
return FALSE;
}
if (!mem->surface) {
GST_WARNING ("The surface is not allocated");
return FALSE;
}
if ((flags & GST_MAP_WRITE) && mem->surface && mem->surface->Data.Locked) {
GST_WARNING ("The surface in memory %p is not still available", mem);
return FALSE;
}
if (!mem->mapped) {
gst_msdk_frame_lock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
}
mem->mapped++;
mem_id = mem->surface->Data.MemId;
/* msdk doesn't support I420 format and we used YV12 internally
* So we need to swap U/V planes for mapping */
if (meta->format == GST_VIDEO_FORMAT_I420)
plane_id = plane ? (plane == 1 ? 2 : 1) : plane;
#ifndef _WIN32
offset = mem_id->image.offsets[plane_id];
pitch = mem_id->image.pitches[plane_id];
#else
/* TODO: This is just to avoid compile errors on Windows.
* Implement handling Windows-specific video-memory.
*/
offset = mem_id->offset;
pitch = mem_id->pitch;
#endif
switch (meta->format) {
case GST_VIDEO_FORMAT_BGRA:
case GST_VIDEO_FORMAT_BGRx:
*data = mem->surface->Data.B + offset;
break;
/* The first channel in memory is V for GST_VIDEO_FORMAT_VUYA */
case GST_VIDEO_FORMAT_VUYA:
*data = mem->surface->Data.V + offset;
break;
case GST_VIDEO_FORMAT_Y410:
case GST_VIDEO_FORMAT_Y412_LE:
*data = mem->surface->Data.U + offset;
break;
default:
*data = mem->surface->Data.Y + offset;
break;
}
*stride = pitch;
info->flags = flags;
ret = (*data != NULL);
return ret;
}
gboolean
gst_video_meta_unmap_msdk_memory (GstVideoMeta * meta, guint plane,
GstMapInfo * info)
{
GstAllocator *allocator;
GstMsdkVideoAllocator *msdk_video_allocator;
GstMsdkVideoMemory *mem =
GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
g_return_val_if_fail (mem, FALSE);
allocator = GST_MEMORY_CAST (mem)->allocator;
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
if (mem->mapped == 1)
gst_msdk_frame_unlock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
mem->mapped--;
return TRUE;
}
static gpointer
gst_msdk_video_memory_map_full (GstMemory * base_mem, GstMapInfo * info,
gsize maxsize)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
GstAllocator *allocator = base_mem->allocator;
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
g_return_val_if_fail (mem, NULL);
if (!mem->surface) {
GST_WARNING ("The surface is not allocated");
return NULL;
}
if ((info->flags & GST_MAP_WRITE) && mem->surface
&& mem->surface->Data.Locked) {
GST_WARNING ("The surface in memory %p is not still available", mem);
return NULL;
}
gst_msdk_frame_lock (msdk_video_allocator->context, mem->surface->Data.MemId,
&mem->surface->Data);
switch (mem->surface->Info.FourCC) {
case MFX_FOURCC_RGB4:
return mem->surface->Data.B; /* The first channel is B */
/* The first channel in memory is V for MFX_FOURCC_AYUV (GST_VIDEO_FORMAT_VUYA) format */
case MFX_FOURCC_AYUV:
return mem->surface->Data.V;
#if (MFX_VERSION >= 1027)
case MFX_FOURCC_Y410:
return mem->surface->Data.U; /* Data.Y410 */
#endif
#if (MFX_VERSION >= 1031)
case MFX_FOURCC_Y416:
return mem->surface->Data.U;
#endif
default:
return mem->surface->Data.Y;
}
}
static void
gst_msdk_video_memory_unmap (GstMemory * base_mem)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
GstAllocator *allocator = base_mem->allocator;
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
gst_msdk_frame_unlock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
}
static GstMemory *
gst_msdk_video_memory_copy (GstMemory * base_mem, gssize offset, gssize size)
{
GstMemory *copy;
GstVideoInfo *info;
GstMsdkVideoAllocator *msdk_video_allocator;
gsize mem_size;
GstMapInfo src_map, dst_map;
/* FIXME: can we consider offset and size here ? */
copy = gst_msdk_video_memory_new (base_mem->allocator);
if (!copy) {
GST_ERROR_OBJECT (base_mem->allocator, "Failed to create new video memory");
return NULL;
}
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_mem->allocator);
info = &msdk_video_allocator->image_info;
mem_size = GST_VIDEO_INFO_SIZE (info);
gst_memory_map (base_mem, &src_map, GST_MAP_READ);
gst_memory_map (copy, &dst_map, GST_MAP_WRITE);
memcpy (dst_map.data, src_map.data, mem_size);
gst_memory_unmap (copy, &dst_map);
gst_memory_unmap (base_mem, &src_map);
return copy;
}
/* GstMsdkVideoAllocator */
G_DEFINE_TYPE (GstMsdkVideoAllocator, gst_msdk_video_allocator,
GST_TYPE_ALLOCATOR);
static GstMemory *
gst_msdk_video_allocator_alloc (GstAllocator * allocator, gsize size,
GstAllocationParams * params)
{
return gst_msdk_video_memory_new (allocator);
}
static void
gst_msdk_video_allocator_finalize (GObject * object)
{
GstMsdkVideoAllocator *allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (object);
gst_msdk_frame_free (allocator->context, allocator->alloc_response);
gst_object_unref (allocator->context);
G_OBJECT_CLASS (gst_msdk_video_allocator_parent_class)->finalize (object);
}
static void
gst_msdk_video_allocator_free (GstAllocator * allocator, GstMemory * base_mem)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
g_slice_free (GstMsdkVideoMemory, mem);
}
static void
gst_msdk_video_allocator_class_init (GstMsdkVideoAllocatorClass * klass)
{
GObjectClass *const object_class = G_OBJECT_CLASS (klass);
GstAllocatorClass *const allocator_class = GST_ALLOCATOR_CLASS (klass);
object_class->finalize = gst_msdk_video_allocator_finalize;
allocator_class->alloc = gst_msdk_video_allocator_alloc;
allocator_class->free = gst_msdk_video_allocator_free;
}
static void
gst_msdk_video_allocator_init (GstMsdkVideoAllocator * allocator)
{
GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
base_allocator->mem_type = GST_MSDK_VIDEO_MEMORY_NAME;
base_allocator->mem_map_full = gst_msdk_video_memory_map_full;
base_allocator->mem_unmap = gst_msdk_video_memory_unmap;
base_allocator->mem_copy = gst_msdk_video_memory_copy;
GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
}
GstAllocator *
gst_msdk_video_allocator_new (GstMsdkContext * context,
GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
{
GstMsdkVideoAllocator *allocator;
GstMsdkAllocResponse *cached = NULL;
g_return_val_if_fail (context != NULL, NULL);
g_return_val_if_fail (image_info != NULL, NULL);
cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
if (!cached) {
GST_ERROR ("Failed to get the cached alloc response");
return NULL;
}
allocator = g_object_new (GST_TYPE_MSDK_VIDEO_ALLOCATOR, NULL);
if (!allocator)
return NULL;
g_atomic_int_inc (&cached->refcount);
allocator->context = gst_object_ref (context);
allocator->image_info = *image_info;
allocator->mfx_response = *alloc_resp;
allocator->alloc_response = &allocator->mfx_response;
return GST_ALLOCATOR_CAST (allocator);
}
/* GstMsdkDmaBufMemory */
GstMemory *
gst_msdk_dmabuf_memory_new (GstAllocator * base_allocator)
{
#ifndef _WIN32
mfxFrameSurface1 *surface;
g_return_val_if_fail (base_allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (base_allocator), NULL);
surface = gst_msdk_video_allocator_get_surface (base_allocator);
if (!surface)
return NULL;
return gst_msdk_dmabuf_memory_new_with_surface (base_allocator, surface);
#else
return NULL;
#endif
}
GstMemory *
gst_msdk_dmabuf_memory_new_with_surface (GstAllocator * allocator,
mfxFrameSurface1 * surface)
{
#ifndef _WIN32
GstMemory *mem;
GstMsdkMemoryID *mem_id;
gint fd;
gsize size;
g_return_val_if_fail (allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator), NULL);
mem_id = surface->Data.MemId;
g_assert (mem_id->desc.num_objects == 1);
fd = mem_id->desc.objects[0].fd;
size = mem_id->desc.objects[0].size;
if (fd < 0) {
GST_ERROR ("Failed to get dmabuf handle");
return NULL;
}
mem = gst_fd_allocator_alloc (allocator, fd, size,
GST_FD_MEMORY_FLAG_DONT_CLOSE);
if (!mem) {
GST_ERROR ("failed ! dmabuf fd: %d", fd);
return NULL;
}
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, surface, NULL);
return mem;
#else
return NULL;
#endif
}
/* GstMsdkDmaBufAllocator */
G_DEFINE_TYPE (GstMsdkDmaBufAllocator, gst_msdk_dmabuf_allocator,
GST_TYPE_DMABUF_ALLOCATOR);
static void
gst_msdk_dmabuf_allocator_finalize (GObject * object)
{
GstMsdkDmaBufAllocator *allocator = GST_MSDK_DMABUF_ALLOCATOR_CAST (object);
gst_msdk_frame_free (allocator->context, allocator->alloc_response);
gst_object_unref (allocator->context);
G_OBJECT_CLASS (gst_msdk_dmabuf_allocator_parent_class)->finalize (object);
}
static void
gst_msdk_dmabuf_allocator_class_init (GstMsdkDmaBufAllocatorClass * klass)
{
GObjectClass *const object_class = G_OBJECT_CLASS (klass);
object_class->finalize = gst_msdk_dmabuf_allocator_finalize;
}
static void
gst_msdk_dmabuf_allocator_init (GstMsdkDmaBufAllocator * allocator)
{
GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
base_allocator->mem_type = GST_MSDK_DMABUF_MEMORY_NAME;
}
GstAllocator *
gst_msdk_dmabuf_allocator_new (GstMsdkContext * context,
GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
{
GstMsdkDmaBufAllocator *allocator;
GstMsdkAllocResponse *cached = NULL;
g_return_val_if_fail (context != NULL, NULL);
g_return_val_if_fail (image_info != NULL, NULL);
cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
if (!cached) {
GST_ERROR ("Failed to get the cached alloc response");
return NULL;
}
allocator = g_object_new (GST_TYPE_MSDK_DMABUF_ALLOCATOR, NULL);
if (!allocator)
return NULL;
g_atomic_int_inc (&cached->refcount);
allocator->context = gst_object_ref (context);
allocator->image_info = *image_info;
allocator->mfx_response = *alloc_resp;
allocator->alloc_response = &allocator->mfx_response;
return GST_ALLOCATOR_CAST (allocator);
}