From 2854af85fd6f905088f51d164ef2ffeef0e59604 Mon Sep 17 00:00:00 2001 From: Mengkejiergeli Ba Date: Fri, 27 May 2022 17:56:33 +0800 Subject: [PATCH] msdk: Add help functions to get mfxFrameSurface1 from GstBuffer and wrap it as GstMsdkSurface Note that the memory abstraction for system memory is for windows path. Part-of: --- .../sys/msdk/gstmsdkallocator.c | 151 ++++++++++++++++++ .../sys/msdk/gstmsdkallocator.h | 10 ++ .../sys/msdk/gstmsdkallocator_libva.c | 114 +++++++++++++ .../sys/msdk/gstmsdkallocator_libva.h | 4 + .../gst-plugins-bad/sys/msdk/meson.build | 1 + subprojects/gst-plugins-bad/sys/msdk/msdk.h | 1 + 6 files changed, 281 insertions(+) create mode 100644 subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.c diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.c new file mode 100644 index 0000000000..2ea71325d7 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.c @@ -0,0 +1,151 @@ +/* + * GStreamer Intel MSDK plugin + * Copyright (c) 2022 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * 3. Neither the name of the copyright holder nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; + * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "gstmsdkallocator.h" + +static gboolean +map_data (GstBuffer * buffer, mfxFrameSurface1 * mfx_surface, GstVideoInfo info) +{ + guint stride; + GstVideoFrame frame; + + if (!gst_video_frame_map (&frame, &info, buffer, GST_MAP_READWRITE)) + return FALSE; + + stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0); + + switch (GST_VIDEO_INFO_FORMAT (&info)) { + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_P010_10LE: + case GST_VIDEO_FORMAT_P012_LE: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.UV = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1); + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_YV12: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 2); + mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1); + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_I420: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1); + mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 2); + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_YUY2: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = mfx_surface->Data.Y + 1; + mfx_surface->Data.V = mfx_surface->Data.Y + 3; + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_UYVY: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = mfx_surface->Data.Y; + mfx_surface->Data.V = mfx_surface->Data.U + 2; + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_VUYA: + mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = mfx_surface->Data.V + 1; + mfx_surface->Data.Y = mfx_surface->Data.V + 2; + mfx_surface->Data.A = mfx_surface->Data.V + 3; + mfx_surface->Data.PitchHigh = (mfxU16) (stride / (1 << 16)); + mfx_surface->Data.PitchLow = (mfxU16) (stride % (1 << 16)); + break; + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_BGRx: + mfx_surface->Data.B = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.G = mfx_surface->Data.B + 1; + mfx_surface->Data.R = mfx_surface->Data.B + 2; + mfx_surface->Data.A = mfx_surface->Data.B + 3; + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_Y210: + case GST_VIDEO_FORMAT_Y212_LE: + mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.U = mfx_surface->Data.Y + 2; + mfx_surface->Data.V = mfx_surface->Data.Y + 6; + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + case GST_VIDEO_FORMAT_Y410: + mfx_surface->Data.Y410 = + (mfxY410 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.Pitch = stride; + break; + case GST_VIDEO_FORMAT_Y412_LE: + mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0); + mfx_surface->Data.Y = mfx_surface->Data.Y + 2; + mfx_surface->Data.V = mfx_surface->Data.Y + 4; + mfx_surface->Data.A = mfx_surface->Data.Y + 6; + mfx_surface->Data.Pitch = (mfxU16) stride; + break; + default: + g_assert_not_reached (); + break; + } + + gst_video_frame_unmap (&frame); + return TRUE; +} + +GstMsdkSurface * +gst_msdk_import_sys_mem_to_msdk_surface (GstBuffer * buf, GstVideoInfo info) +{ + GstMsdkSurface *msdk_surface = NULL; + GstMapInfo map_info; + mfxFrameInfo frame_info = { 0, }; + mfxFrameSurface1 *mfx_surface = NULL; + + if (!gst_buffer_map (buf, &map_info, GST_MAP_READ)) { + GST_ERROR ("Failed to map buffer"); + return msdk_surface; + } + + mfx_surface = g_slice_new0 (mfxFrameSurface1); + mfx_surface->Data.MemId = (mfxMemId) map_info.data; + + if (!map_data (buf, mfx_surface, info)) { + g_slice_free (mfxFrameSurface1, mfx_surface); + return msdk_surface; + } + + gst_buffer_unmap (buf, &map_info); + + gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, &info); + mfx_surface->Info = frame_info; + + msdk_surface = g_slice_new0 (GstMsdkSurface); + msdk_surface->surface = mfx_surface; + + return msdk_surface; +} diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h index e0d5616205..4ef99d649f 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h @@ -39,6 +39,7 @@ G_BEGIN_DECLS typedef struct _GstMsdkMemoryID GstMsdkMemoryID; +typedef struct _GstMsdkSurface GstMsdkSurface; struct _GstMsdkMemoryID { mfxU32 fourcc; @@ -56,6 +57,15 @@ struct _GstMsdkMemoryID { #endif }; +struct _GstMsdkSurface +{ + mfxFrameSurface1 *surface; + GstBuffer *buf; +}; + +GstMsdkSurface * +gst_msdk_import_sys_mem_to_msdk_surface (GstBuffer * buf, GstVideoInfo info); + mfxStatus gst_msdk_frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req, mfxFrameAllocResponse *resp); mfxStatus gst_msdk_frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp); mfxStatus gst_msdk_frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr); diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c index 9f711b008f..2332d172b7 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c @@ -37,6 +37,21 @@ #include "gstmsdkallocator_libva.h" #include "msdk_libva.h" +#include + +#define GST_MSDK_FRAME_SURFACE gst_msdk_frame_surface_quark_get () +static GQuark +gst_msdk_frame_surface_quark_get (void) +{ + static gsize g_quark; + + if (g_once_init_enter (&g_quark)) { + gsize quark = (gsize) g_quark_from_static_string ("GstMsdkFrameSurface"); + g_once_init_leave (&g_quark, quark); + } + return g_quark; +} + mfxStatus gst_msdk_frame_alloc (mfxHDL pthis, mfxFrameAllocRequest * req, mfxFrameAllocResponse * resp) @@ -704,6 +719,105 @@ error_create_surface: } } +static VASurfaceID +_get_va_surface (GstBuffer * buf, GstVideoInfo * info, + GstMsdkContext * msdk_context) +{ + VASurfaceID va_surface = VA_INVALID_ID; + + if (!info) { + va_surface = gst_va_buffer_get_surface (buf); + } else { + /* Update offset/stride/size if there is VideoMeta attached to + * the dma buffer, which is then used to get vasurface */ + GstMemory *mem; + gint i, fd; + GstVideoMeta *vmeta; + + vmeta = gst_buffer_get_video_meta (buf); + if (vmeta) { + if (GST_VIDEO_INFO_FORMAT (info) != vmeta->format || + GST_VIDEO_INFO_WIDTH (info) != vmeta->width || + GST_VIDEO_INFO_HEIGHT (info) != vmeta->height || + GST_VIDEO_INFO_N_PLANES (info) != vmeta->n_planes) { + GST_ERROR ("VideoMeta attached to buffer is not matching" + "the negotiated width/height/format"); + return va_surface; + } + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); ++i) { + GST_VIDEO_INFO_PLANE_OFFSET (info, i) = vmeta->offset[i]; + GST_VIDEO_INFO_PLANE_STRIDE (info, i) = vmeta->stride[i]; + } + GST_VIDEO_INFO_SIZE (info) = gst_buffer_get_size (buf); + } + + mem = gst_buffer_peek_memory (buf, 0); + fd = gst_dmabuf_memory_get_fd (mem); + if (fd < 0) + return va_surface; + /* export dmabuf to vasurface */ + if (!gst_msdk_export_dmabuf_to_vasurface (msdk_context, info, fd, + &va_surface)) + return VA_INVALID_ID; + } + + return va_surface; +} + +GstMsdkSurface * +gst_msdk_import_to_msdk_surface (GstBuffer * buf, GstMsdkContext * msdk_context, + GstVideoInfo * vinfo) +{ + VASurfaceID va_surface = VA_INVALID_ID; + GstMemory *mem = NULL; + mfxFrameInfo frame_info = { 0, }; + GstMsdkSurface *msdk_surface = NULL; + mfxFrameSurface1 *mfx_surface = NULL; + GstMsdkMemoryID *msdk_mid = NULL; + + mem = gst_buffer_peek_memory (buf, 0); + msdk_surface = g_slice_new0 (GstMsdkSurface); + + /* If buffer has qdata pointing to mfxFrameSurface1, directly extract it */ + if ((mfx_surface = gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem), + GST_MSDK_FRAME_SURFACE))) { + msdk_surface->surface = mfx_surface; + return msdk_surface; + } + + if (gst_msdk_is_va_mem (mem)) { + va_surface = _get_va_surface (buf, NULL, NULL); + } else if (gst_is_dmabuf_memory (mem)) { + /* For dma memory, videoinfo is used with dma fd to create va surface. */ + GstVideoInfo info = *vinfo; + va_surface = _get_va_surface (buf, &info, msdk_context); + } + + if (va_surface == VA_INVALID_ID) { + g_slice_free (GstMsdkSurface, msdk_surface); + return NULL; + } + + mfx_surface = g_slice_new0 (mfxFrameSurface1); + msdk_mid = g_slice_new0 (GstMsdkMemoryID); + + msdk_mid->surface = g_slice_new0 (VASurfaceID); + *msdk_mid->surface = va_surface; + + mfx_surface->Data.MemId = (mfxMemId) msdk_mid; + + gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, vinfo); + mfx_surface->Info = frame_info; + + /* Set mfxFrameSurface1 as qdata in buffer */ + gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem), + GST_MSDK_FRAME_SURFACE, mfx_surface, NULL); + + msdk_surface->surface = mfx_surface; + + return msdk_surface; +} + /** * gst_msdk_replace_mfx_memid: * This method replace the internal VA Suface in mfxSurface with a new one diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.h index da2f4d3bfe..fa25821fd8 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.h @@ -45,6 +45,10 @@ gboolean gst_msdk_export_dmabuf_to_vasurface (GstMsdkContext *context, GstVideoInfo *vinfo, gint fd, VASurfaceID *surface_id); +GstMsdkSurface * +gst_msdk_import_to_msdk_surface (GstBuffer * buf, GstMsdkContext * msdk_context, + GstVideoInfo * vinfo); + gboolean gst_msdk_replace_mfx_memid (GstMsdkContext *context, mfxFrameSurface1 *mfx_surface, VASurfaceID surface_id); diff --git a/subprojects/gst-plugins-bad/sys/msdk/meson.build b/subprojects/gst-plugins-bad/sys/msdk/meson.build index a0c1664122..924deace98 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/meson.build +++ b/subprojects/gst-plugins-bad/sys/msdk/meson.build @@ -1,5 +1,6 @@ msdk_sources = [ 'gstmsdk.c', + 'gstmsdkallocator.c', 'gstmsdkbufferpool.c', 'gstmsdkcontext.c', 'gstmsdkcontextutil.c', diff --git a/subprojects/gst-plugins-bad/sys/msdk/msdk.h b/subprojects/gst-plugins-bad/sys/msdk/msdk.h index cb76fd516d..2b197ccce8 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/msdk.h +++ b/subprojects/gst-plugins-bad/sys/msdk/msdk.h @@ -99,6 +99,7 @@ void GstMFXUnload (mfxLoader loader); #endif typedef struct _MsdkSession MsdkSession; +typedef struct _GstMsdkSurface GstMsdkSurface; struct _MsdkSession {