gstreamer/sys/msdk/gstmsdkvideomemory.c

579 lines
17 KiB
C
Raw Normal View History

/* GStreamer Intel MSDK plugin
* Copyright (c) 2018, Intel Corporation
* Copyright (c) 2018, Igalia S.L.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _WIN32
#include <unistd.h>
#include <va/va.h>
#endif
#include <stdlib.h>
#include "gstmsdkvideomemory.h"
#include "gstmsdkallocator.h"
#define GST_MSDK_BUFFER_SURFACE gst_msdk_buffer_surface_quark_get ()
static GQuark
gst_msdk_buffer_surface_quark_get (void)
{
static gsize g_quark;
if (g_once_init_enter (&g_quark)) {
gsize quark = (gsize) g_quark_from_static_string ("GstMsdkBufferSurface");
g_once_init_leave (&g_quark, quark);
}
return g_quark;
}
static mfxFrameSurface1 *
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
gst_msdk_video_allocator_get_surface (GstAllocator * allocator)
{
mfxFrameInfo frame_info = { {0,}, 0, };
mfxFrameSurface1 *surface;
GstMsdkContext *context = NULL;
mfxFrameAllocResponse *resp = NULL;
GstVideoInfo *vinfo = NULL;
if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
context = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->context;
resp = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->alloc_response;
vinfo = &GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator)->image_info;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
context = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->context;
resp = GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->alloc_response;
vinfo = &GST_MSDK_DMABUF_ALLOCATOR_CAST (allocator)->image_info;
} else {
return NULL;
}
surface = gst_msdk_context_get_surface_available (context, resp);
if (!surface) {
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
GST_ERROR ("failed to get surface available");
return NULL;
}
gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, vinfo);
surface->Info = frame_info;
return surface;
}
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
gboolean
gst_msdk_video_memory_get_surface_available (GstMemory * mem)
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
{
GstAllocator *allocator;
mfxFrameSurface1 *surface;
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
g_return_val_if_fail (mem, FALSE);
allocator = mem->allocator;
surface = gst_msdk_video_allocator_get_surface (allocator);
if (GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = surface;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator)) {
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, surface, NULL);
}
return surface ? TRUE : FALSE;
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
}
/*
* Every time releasing a gst buffer, we need to check the status of surface's lock,
2019-09-02 19:08:44 +00:00
* so that we could manage locked surfaces separately in the context.
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
* Otherwise, we put the surface to the available list.
*/
void
gst_msdk_video_memory_release_surface (GstMemory * mem)
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
{
mfxFrameSurface1 *surface = NULL;
GstMsdkContext *context = NULL;
mfxFrameAllocResponse *alloc_response = NULL;
g_return_if_fail (mem);
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator)) {
surface = GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface;
context = GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->context;
alloc_response =
GST_MSDK_VIDEO_ALLOCATOR_CAST (mem->allocator)->alloc_response;
} else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator)) {
surface =
gst_mini_object_get_qdata (GST_MINI_OBJECT (mem),
GST_MSDK_BUFFER_SURFACE);
context = GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->context;
alloc_response =
GST_MSDK_DMABUF_ALLOCATOR_CAST (mem->allocator)->alloc_response;
} else {
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
return;
}
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
if (surface->Data.Locked > 0)
gst_msdk_context_put_surface_locked (context, alloc_response, surface);
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
else
gst_msdk_context_put_surface_available (context, alloc_response, surface);
if (GST_IS_MSDK_VIDEO_ALLOCATOR (mem->allocator))
GST_MSDK_VIDEO_MEMORY_CAST (mem)->surface = NULL;
else if (GST_IS_MSDK_DMABUF_ALLOCATOR (mem->allocator))
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, NULL, NULL);
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
return;
}
GstMemory *
gst_msdk_video_memory_new (GstAllocator * base_allocator)
{
GstMsdkVideoAllocator *allocator;
GstVideoInfo *vip;
GstMsdkVideoMemory *mem;
g_return_val_if_fail (base_allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_VIDEO_ALLOCATOR (base_allocator), NULL);
allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_allocator);
mem = g_slice_new0 (GstMsdkVideoMemory);
if (!mem)
return NULL;
msdk: manage MSDK surfaces seperately Currently a gst buffer has one mfxFrameSurface when it's allocated and can't be changed. This is based on that the life of gst buffer and mfxFrameSurface would be same. But it's not true. Sometimes even if a gst buffer of a frame is finished on downstream, mfxFramesurface coupled with the gst buffer is still locked, which means it's still being used in the driver. So this patch does this. Every time a gst buffer is acquired from the pool, it confirms if the surface coupled with the buffer is unlocked. If not, replace it with new unlocked one. In this way, user(decoder or encoder) doesn't need to manage gst buffers including locked surface. To do that, this patch includes the following: 1. GstMsdkContext - Manages MSDK surfaces available, used, locked respectively as the following: 1\ surfaces_avail : surfaces which are free and unused anywhere 2\ surfaces_used : surfaces coupled with a gst buffer and being used now. 3\ surfaces_locked : surfaces still locked even after the gst buffer is released. - Provide an api to get MSDK surface available. - Provide an api to release MSDK surface. 2. GstMsdkVideoMemory - Gets a surface available when it's allocated. - Provide an api to get an available surface with new unlocked one. - Provide an api to release surface in the msdk video memory. 3. GstMsdkBufferPool - In acquire_buffer, every time a gst buffer is acquired, get new available surface from the list. - In release_buffer, it confirms if the buffer's surface is unlocked or not. - If unlocked, it is put to the available list. - If still locked, it is put to the locked list. This also fixes bug #793525. https://bugzilla.gnome.org/show_bug.cgi?id=793413 https://bugzilla.gnome.org/show_bug.cgi?id=793525
2018-03-08 20:37:12 +00:00
mem->surface = gst_msdk_video_allocator_get_surface (base_allocator);
if (!mem->surface) {
g_slice_free (GstMsdkVideoMemory, mem);
return NULL;
}
vip = &allocator->image_info;
gst_memory_init (&mem->parent_instance, 0,
base_allocator, NULL, GST_VIDEO_INFO_SIZE (vip), 0, 0,
GST_VIDEO_INFO_SIZE (vip));
return GST_MEMORY_CAST (mem);
}
gboolean
gst_video_meta_map_msdk_memory (GstVideoMeta * meta, guint plane,
GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags)
{
gboolean ret = FALSE;
GstAllocator *allocator;
GstMsdkVideoAllocator *msdk_video_allocator;
GstMsdkVideoMemory *mem =
GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
GstMsdkMemoryID *mem_id;
guint offset = 0;
gint pitch = 0;
guint plane_id = plane;
g_return_val_if_fail (mem, FALSE);
allocator = GST_MEMORY_CAST (mem)->allocator;
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
if (!GST_IS_MSDK_VIDEO_ALLOCATOR (allocator)) {
GST_WARNING ("The allocator is not MSDK video allocator");
return FALSE;
}
if (!mem->surface) {
GST_WARNING ("The surface is not allocated");
return FALSE;
}
if ((flags & GST_MAP_WRITE) && mem->surface && mem->surface->Data.Locked) {
2019-09-02 19:08:44 +00:00
GST_WARNING ("The surface in memory %p is not still available", mem);
return FALSE;
}
if (!mem->mapped) {
gst_msdk_frame_lock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
}
mem->mapped++;
mem_id = mem->surface->Data.MemId;
/* msdk doesn't support I420 format and we used YV12 internally
* So we need to swap U/V planes for mapping */
if (meta->format == GST_VIDEO_FORMAT_I420)
plane_id = plane ? (plane == 1 ? 2 : 1) : plane;
#ifndef _WIN32
offset = mem_id->image.offsets[plane_id];
pitch = mem_id->image.pitches[plane_id];
#else
/* TODO: This is just to avoid compile errors on Windows.
* Implement handling Windows-specific video-memory.
*/
offset = mem_id->offset;
pitch = mem_id->pitch;
#endif
switch (meta->format) {
case GST_VIDEO_FORMAT_BGRA:
*data = mem->surface->Data.B + offset;
break;
/* The first channel in memory is V for GST_VIDEO_FORMAT_VUYA */
case GST_VIDEO_FORMAT_VUYA:
*data = mem->surface->Data.V + offset;
break;
case GST_VIDEO_FORMAT_Y410:
*data = mem->surface->Data.U + offset; /* Data.Y410 */
break;
default:
*data = mem->surface->Data.Y + offset;
break;
}
*stride = pitch;
info->flags = flags;
ret = (*data != NULL);
return ret;
}
gboolean
gst_video_meta_unmap_msdk_memory (GstVideoMeta * meta, guint plane,
GstMapInfo * info)
{
GstAllocator *allocator;
GstMsdkVideoAllocator *msdk_video_allocator;
GstMsdkVideoMemory *mem =
GST_MSDK_VIDEO_MEMORY_CAST (gst_buffer_peek_memory (meta->buffer, 0));
g_return_val_if_fail (mem, FALSE);
allocator = GST_MEMORY_CAST (mem)->allocator;
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
if (mem->mapped == 1)
gst_msdk_frame_unlock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
mem->mapped--;
return TRUE;
}
static gpointer
gst_msdk_video_memory_map_full (GstMemory * base_mem, GstMapInfo * info,
gsize maxsize)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
GstAllocator *allocator = base_mem->allocator;
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
g_return_val_if_fail (mem, NULL);
if (!mem->surface) {
GST_WARNING ("The surface is not allocated");
return NULL;
}
if ((info->flags & GST_MAP_WRITE) && mem->surface
&& mem->surface->Data.Locked) {
2019-09-02 19:08:44 +00:00
GST_WARNING ("The surface in memory %p is not still available", mem);
return NULL;
}
gst_msdk_frame_lock (msdk_video_allocator->context, mem->surface->Data.MemId,
&mem->surface->Data);
switch (mem->surface->Info.FourCC) {
case MFX_FOURCC_RGB4:
return mem->surface->Data.B; /* The first channel is B */
/* The first channel in memory is V for MFX_FOURCC_AYUV (GST_VIDEO_FORMAT_VUYA) format */
case MFX_FOURCC_AYUV:
return mem->surface->Data.V;
#if (MFX_VERSION >= 1027)
case MFX_FOURCC_Y410:
return mem->surface->Data.U; /* Data.Y410 */
#endif
default:
return mem->surface->Data.Y;
}
}
static void
gst_msdk_video_memory_unmap (GstMemory * base_mem)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
GstAllocator *allocator = base_mem->allocator;
GstMsdkVideoAllocator *msdk_video_allocator =
GST_MSDK_VIDEO_ALLOCATOR_CAST (allocator);
gst_msdk_frame_unlock (msdk_video_allocator->context,
mem->surface->Data.MemId, &mem->surface->Data);
}
static GstMemory *
gst_msdk_video_memory_copy (GstMemory * base_mem, gssize offset, gssize size)
{
GstMemory *copy;
GstVideoInfo *info;
GstMsdkVideoAllocator *msdk_video_allocator;
gsize mem_size;
GstMapInfo src_map, dst_map;
/* FIXME: can we consider offset and size here ? */
copy = gst_msdk_video_memory_new (base_mem->allocator);
if (!copy) {
GST_ERROR_OBJECT (base_mem->allocator, "Failed to create new video memory");
return NULL;
}
msdk_video_allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (base_mem->allocator);
info = &msdk_video_allocator->image_info;
mem_size = GST_VIDEO_INFO_SIZE (info);
gst_memory_map (base_mem, &src_map, GST_MAP_READ);
gst_memory_map (copy, &dst_map, GST_MAP_WRITE);
memcpy (dst_map.data, src_map.data, mem_size);
gst_memory_unmap (copy, &dst_map);
gst_memory_unmap (base_mem, &src_map);
return copy;
}
/* GstMsdkVideoAllocator */
G_DEFINE_TYPE (GstMsdkVideoAllocator, gst_msdk_video_allocator,
GST_TYPE_ALLOCATOR);
static GstMemory *
gst_msdk_video_allocator_alloc (GstAllocator * allocator, gsize size,
GstAllocationParams * params)
{
return gst_msdk_video_memory_new (allocator);
}
static void
gst_msdk_video_allocator_finalize (GObject * object)
{
GstMsdkVideoAllocator *allocator = GST_MSDK_VIDEO_ALLOCATOR_CAST (object);
gst_msdk_frame_free (allocator->context, allocator->alloc_response);
gst_object_unref (allocator->context);
G_OBJECT_CLASS (gst_msdk_video_allocator_parent_class)->finalize (object);
}
static void
gst_msdk_video_allocator_free (GstAllocator * allocator, GstMemory * base_mem)
{
GstMsdkVideoMemory *const mem = GST_MSDK_VIDEO_MEMORY_CAST (base_mem);
g_slice_free (GstMsdkVideoMemory, mem);
}
static void
gst_msdk_video_allocator_class_init (GstMsdkVideoAllocatorClass * klass)
{
GObjectClass *const object_class = G_OBJECT_CLASS (klass);
GstAllocatorClass *const allocator_class = GST_ALLOCATOR_CLASS (klass);
object_class->finalize = gst_msdk_video_allocator_finalize;
allocator_class->alloc = gst_msdk_video_allocator_alloc;
allocator_class->free = gst_msdk_video_allocator_free;
}
static void
gst_msdk_video_allocator_init (GstMsdkVideoAllocator * allocator)
{
GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
base_allocator->mem_type = GST_MSDK_VIDEO_MEMORY_NAME;
base_allocator->mem_map_full = gst_msdk_video_memory_map_full;
base_allocator->mem_unmap = gst_msdk_video_memory_unmap;
base_allocator->mem_copy = gst_msdk_video_memory_copy;
GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
}
GstAllocator *
gst_msdk_video_allocator_new (GstMsdkContext * context,
GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
{
GstMsdkVideoAllocator *allocator;
GstMsdkAllocResponse *cached = NULL;
g_return_val_if_fail (context != NULL, NULL);
g_return_val_if_fail (image_info != NULL, NULL);
cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
if (!cached) {
GST_ERROR ("Failed to get the cached alloc response");
return NULL;
}
allocator = g_object_new (GST_TYPE_MSDK_VIDEO_ALLOCATOR, NULL);
if (!allocator)
return NULL;
g_atomic_int_inc (&cached->refcount);
allocator->context = gst_object_ref (context);
allocator->image_info = *image_info;
allocator->mfx_response = *alloc_resp;
allocator->alloc_response = &allocator->mfx_response;
return GST_ALLOCATOR_CAST (allocator);
}
/* GstMsdkDmaBufMemory */
GstMemory *
gst_msdk_dmabuf_memory_new (GstAllocator * base_allocator)
{
#ifndef _WIN32
mfxFrameSurface1 *surface;
g_return_val_if_fail (base_allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (base_allocator), NULL);
surface = gst_msdk_video_allocator_get_surface (base_allocator);
if (!surface)
return NULL;
return gst_msdk_dmabuf_memory_new_with_surface (base_allocator, surface);
#else
return NULL;
#endif
}
GstMemory *
gst_msdk_dmabuf_memory_new_with_surface (GstAllocator * allocator,
mfxFrameSurface1 * surface)
{
#ifndef _WIN32
GstMemory *mem;
GstMsdkMemoryID *mem_id;
gint fd;
gsize size;
g_return_val_if_fail (allocator, NULL);
g_return_val_if_fail (GST_IS_MSDK_DMABUF_ALLOCATOR (allocator), NULL);
mem_id = surface->Data.MemId;
fd = mem_id->info.handle;
size = mem_id->info.mem_size;
if (fd < 0 || (fd = dup (fd)) < 0) {
GST_ERROR ("Failed to get dmabuf handle");
return NULL;
}
mem = gst_dmabuf_allocator_alloc (allocator, fd, size);
if (!mem) {
GST_ERROR ("failed ! dmabuf fd: %d", fd);
close (fd);
return NULL;
}
gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
GST_MSDK_BUFFER_SURFACE, surface, NULL);
return mem;
#else
return NULL;
#endif
}
/* GstMsdkDmaBufAllocator */
G_DEFINE_TYPE (GstMsdkDmaBufAllocator, gst_msdk_dmabuf_allocator,
GST_TYPE_DMABUF_ALLOCATOR);
static void
gst_msdk_dmabuf_allocator_finalize (GObject * object)
{
GstMsdkDmaBufAllocator *allocator = GST_MSDK_DMABUF_ALLOCATOR_CAST (object);
gst_msdk_frame_free (allocator->context, allocator->alloc_response);
gst_object_unref (allocator->context);
G_OBJECT_CLASS (gst_msdk_dmabuf_allocator_parent_class)->finalize (object);
}
static void
gst_msdk_dmabuf_allocator_class_init (GstMsdkDmaBufAllocatorClass * klass)
{
GObjectClass *const object_class = G_OBJECT_CLASS (klass);
object_class->finalize = gst_msdk_dmabuf_allocator_finalize;
}
static void
gst_msdk_dmabuf_allocator_init (GstMsdkDmaBufAllocator * allocator)
{
GstAllocator *const base_allocator = GST_ALLOCATOR_CAST (allocator);
base_allocator->mem_type = GST_MSDK_DMABUF_MEMORY_NAME;
}
GstAllocator *
gst_msdk_dmabuf_allocator_new (GstMsdkContext * context,
GstVideoInfo * image_info, mfxFrameAllocResponse * alloc_resp)
{
GstMsdkDmaBufAllocator *allocator;
GstMsdkAllocResponse *cached = NULL;
g_return_val_if_fail (context != NULL, NULL);
g_return_val_if_fail (image_info != NULL, NULL);
cached = gst_msdk_context_get_cached_alloc_responses (context, alloc_resp);
if (!cached) {
GST_ERROR ("Failed to get the cached alloc response");
return NULL;
}
allocator = g_object_new (GST_TYPE_MSDK_DMABUF_ALLOCATOR, NULL);
if (!allocator)
return NULL;
g_atomic_int_inc (&cached->refcount);
allocator->context = gst_object_ref (context);
allocator->image_info = *image_info;
allocator->mfx_response = *alloc_resp;
allocator->alloc_response = &allocator->mfx_response;
return GST_ALLOCATOR_CAST (allocator);
}