From 3062f1b6b03bccf8eba00efa4316d7ac619c72a9 Mon Sep 17 00:00:00 2001 From: Mengkejiergeli Ba Date: Tue, 23 Aug 2022 11:47:02 +0800 Subject: [PATCH] msdkdec: Apply the modified memory allocation logic We did several things to enable the new memory logic in msdkdec: (1) We always use video memory for decoder in linux path; (2) We give negotiated pool to alloc_pool stored in GstMsdkContext which will be used in callback mfxFrameAllocator:Alloc to alloc surfaces as MediaSDK needs, and this pool is also available for decoder itself; (3) We modify decide_allocation process, that is we make pool negotiaion before gst_msdk_init_decoder to ensure the pool is decided and ready for use in mfxFrameAllocator:Alloc callback; then we will consider the case when we need to do the gpu to cpu copy. (4) In gst_msdkdec_finish_task, we modify the way for copy following the logic in (3). Part-of: --- .../sys/msdk/gstmsdkallocator.h | 2 +- .../sys/msdk/gstmsdkallocator_libva.c | 282 +----------- .../gst-plugins-bad/sys/msdk/gstmsdkcontext.c | 21 - .../gst-plugins-bad/sys/msdk/gstmsdkdec.c | 433 ++++++------------ .../gst-plugins-bad/sys/msdk/gstmsdkdec.h | 2 +- 5 files changed, 164 insertions(+), 576 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h index 138c610aff..30bcaf3a6e 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h @@ -45,7 +45,7 @@ struct _GstMsdkMemoryID { mfxU32 fourcc; #ifndef _WIN32 - VASurfaceID *surface; + VASurfaceID surface; VAImage image; VADRMPRIMESurfaceDescriptor desc; #else diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c index 60427f5893..1c046b7409 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c @@ -44,222 +44,6 @@ mfxStatus gst_msdk_frame_alloc (mfxHDL pthis, mfxFrameAllocRequest * req, mfxFrameAllocResponse * resp) -{ - VAStatus va_status; - mfxStatus status = MFX_ERR_NONE; - gint i; - guint format; - guint va_fourcc = 0; - VASurfaceID *surfaces = NULL; - VASurfaceAttrib attribs[2]; - guint num_attribs = 0; - mfxMemId *mids = NULL; - GstMsdkContext *context = (GstMsdkContext *) pthis; - GstMsdkMemoryID *msdk_mids = NULL; - GstMsdkAllocResponse *msdk_resp = NULL; - mfxU32 fourcc = req->Info.FourCC; - mfxU16 surfaces_num = req->NumFrameSuggested; - - /* MFX_MAKEFOURCC('V','P','8','S') is used for MFX_FOURCC_VP9_SEGMAP surface - * in MSDK and this surface is an internal surface. The external allocator - * shouldn't be used for this surface allocation - * - * See https://github.com/Intel-Media-SDK/MediaSDK/issues/762 - */ - if (req->Type & MFX_MEMTYPE_INTERNAL_FRAME - && fourcc == MFX_MAKEFOURCC ('V', 'P', '8', 'S')) - return MFX_ERR_UNSUPPORTED; - - if (req->Type & MFX_MEMTYPE_EXTERNAL_FRAME) { - GstMsdkAllocResponse *cached = - gst_msdk_context_get_cached_alloc_responses_by_request (context, req); - if (cached) { - /* check if enough frames were allocated */ - if (req->NumFrameSuggested > cached->response.NumFrameActual) - return MFX_ERR_MEMORY_ALLOC; - - *resp = cached->response; - g_atomic_int_inc (&cached->refcount); - return MFX_ERR_NONE; - } - } - - /* The VA API does not define any surface types and the application can use either - * MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET or - * MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET to indicate data in video memory. - */ - if (!(req->Type & (MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET | - MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET))) - return MFX_ERR_UNSUPPORTED; - - va_fourcc = gst_msdk_get_va_fourcc_from_mfx_fourcc (fourcc); - - msdk_mids = - (GstMsdkMemoryID *) g_slice_alloc0 (surfaces_num * - sizeof (GstMsdkMemoryID)); - mids = (mfxMemId *) g_slice_alloc0 (surfaces_num * sizeof (mfxMemId)); - surfaces = - (VASurfaceID *) g_slice_alloc0 (surfaces_num * sizeof (VASurfaceID)); - msdk_resp = - (GstMsdkAllocResponse *) g_slice_alloc0 (sizeof (GstMsdkAllocResponse)); - - if (va_fourcc != VA_FOURCC_P208) { - attribs[0].type = VASurfaceAttribPixelFormat; - attribs[0].flags = VA_SURFACE_ATTRIB_SETTABLE; - attribs[0].value.type = VAGenericValueTypeInteger; - attribs[0].value.value.i = va_fourcc; - num_attribs = 1; - - /* set VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER flag for encoding */ -#if (MFX_VERSION >= 1025) - if ((req->Type & MFX_MEMTYPE_VIDEO_MEMORY_ENCODER_TARGET) && - (req->Type & MFX_MEMTYPE_FROM_ENCODE)) { - attribs[1].type = VASurfaceAttribUsageHint; - attribs[1].flags = VA_SURFACE_ATTRIB_SETTABLE; - attribs[1].value.type = VAGenericValueTypeInteger; - attribs[1].value.value.i = VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER; - num_attribs = 2; - } -#endif - - format = - gst_msdk_get_va_rt_format_from_mfx_rt_format (req->Info.ChromaFormat); - - if (format == VA_RT_FORMAT_YUV420 && va_fourcc == VA_FOURCC_P010) -#if VA_CHECK_VERSION(1, 2, 0) - format = VA_RT_FORMAT_YUV420_10; -#else - format = VA_RT_FORMAT_YUV420_10BPP; -#endif - -#if VA_CHECK_VERSION(1, 4, 1) - if (format == VA_RT_FORMAT_YUV444 && va_fourcc == VA_FOURCC_A2R10G10B10) - format = VA_RT_FORMAT_RGB32_10; -#endif - -#if ((MFX_VERSION >= 1027) && VA_CHECK_VERSION(1, 2, 0)) - if (format == VA_RT_FORMAT_YUV422 && va_fourcc == VA_FOURCC_Y210) - format = VA_RT_FORMAT_YUV422_10; - else if (format == VA_RT_FORMAT_YUV444 && va_fourcc == VA_FOURCC_Y410) - format = VA_RT_FORMAT_YUV444_10; -#endif - -#if ((MFX_VERSION >= 1031) && VA_CHECK_VERSION(1, 2, 0)) - if (format == VA_RT_FORMAT_YUV420 && va_fourcc == VA_FOURCC_P016) - format = VA_RT_FORMAT_YUV420_12; - - if (format == VA_RT_FORMAT_YUV422 && va_fourcc == VA_FOURCC_Y216) - format = VA_RT_FORMAT_YUV422_12; - - if (format == VA_RT_FORMAT_YUV444 && va_fourcc == VA_FOURCC_Y416) - format = VA_RT_FORMAT_YUV444_12; -#endif - -#if (MFX_VERSION >= 2004) - if (format == VA_RT_FORMAT_YUV444 && (va_fourcc == VA_FOURCC_RGBP - || va_fourcc == VA_FOURCC_BGRP)) - format = VA_RT_FORMAT_RGBP; -#endif - - va_status = vaCreateSurfaces (gst_msdk_context_get_handle (context), - format, - req->Info.Width, req->Info.Height, surfaces, surfaces_num, attribs, - num_attribs); - - status = gst_msdk_get_mfx_status_from_va_status (va_status); - if (status != MFX_ERR_NONE) { - GST_WARNING ("failed to create VA surface"); - return status; - } - - for (i = 0; i < surfaces_num; i++) { - /* Get dmabuf handle if MFX_MEMTYPE_EXPORT_FRAME */ - if (req->Type & MFX_MEMTYPE_EXPORT_FRAME) { - VADRMPRIMESurfaceDescriptor va_desc = { 0 }; - uint32_t export_flags = VA_EXPORT_SURFACE_SEPARATE_LAYERS | - VA_EXPORT_SURFACE_READ_WRITE; - - va_status = - vaExportSurfaceHandle (gst_msdk_context_get_handle (context), - surfaces[i], VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, export_flags, - &va_desc); - - status = gst_msdk_get_mfx_status_from_va_status (va_status); - - if (MFX_ERR_NONE != status) { - GST_ERROR ("Failed to export surface"); - return status; - } - - g_assert (va_desc.num_objects); - - /* This plugin supports single object only */ - if (va_desc.num_objects > 1) { - GST_ERROR ("Can not support multiple objects"); - return MFX_ERR_UNSUPPORTED; - } - - msdk_mids[i].desc = va_desc; - } - - /* Don't use image for DMABuf */ - msdk_mids[i].image.image_id = VA_INVALID_ID; - msdk_mids[i].image.buf = VA_INVALID_ID; - - msdk_mids[i].surface = &surfaces[i]; - mids[i] = (mfxMemId *) & msdk_mids[i]; - } - } else { - /* This is requested from the driver when h265 encoding. - * These buffers will be used inside the driver and released by - * gst_msdk_frame_free functions. Application doesn't need to handle these buffers. - * - * See https://github.com/Intel-Media-SDK/samples/issues/13 for more details. - */ - VAContextID context_id = req->AllocId; - gint width32 = 32 * ((req->Info.Width + 31) >> 5); - gint height32 = 32 * ((req->Info.Height + 31) >> 5); - guint64 codedbuf_size = (width32 * height32) * 400LL / (16 * 16); - - for (i = 0; i < surfaces_num; i++) { - VABufferID coded_buf; - - va_status = vaCreateBuffer (gst_msdk_context_get_handle (context), - context_id, VAEncCodedBufferType, codedbuf_size, 1, NULL, &coded_buf); - - status = gst_msdk_get_mfx_status_from_va_status (va_status); - if (status < MFX_ERR_NONE) { - GST_ERROR ("failed to create buffer"); - return status; - } - - surfaces[i] = coded_buf; - msdk_mids[i].surface = &surfaces[i]; - msdk_mids[i].fourcc = fourcc; - - /* Don't use image for P208 */ - msdk_mids[i].image.image_id = VA_INVALID_ID; - msdk_mids[i].image.buf = VA_INVALID_ID; - - mids[i] = (mfxMemId *) & msdk_mids[i]; - } - } - - resp->mids = mids; - resp->NumFrameActual = surfaces_num; - - msdk_resp->response = *resp; - msdk_resp->request = *req; - msdk_resp->refcount = 1; - - gst_msdk_context_add_alloc_response (context, msdk_resp); - - return status; -} - -mfxStatus -gst_msdk_frame_alloc_2 (mfxHDL pthis, mfxFrameAllocRequest * req, - mfxFrameAllocResponse * resp) { mfxStatus status = MFX_ERR_NONE; gint i; @@ -439,11 +223,6 @@ mfxStatus gst_msdk_frame_free (mfxHDL pthis, mfxFrameAllocResponse * resp) { GstMsdkContext *context = (GstMsdkContext *) pthis; - VAStatus va_status = VA_STATUS_SUCCESS; - mfxStatus status; - GstMsdkMemoryID *mem_id; - VADisplay dpy; - gint i; GstMsdkAllocResponse *cached = NULL; cached = gst_msdk_context_get_cached_alloc_responses (context, resp); @@ -457,45 +236,9 @@ gst_msdk_frame_free (mfxHDL pthis, mfxFrameAllocResponse * resp) if (!gst_msdk_context_remove_alloc_response (context, resp)) return MFX_ERR_NONE; - mem_id = resp->mids[0]; - dpy = gst_msdk_context_get_handle (context); - - if (mem_id->fourcc != MFX_FOURCC_P8) { - /* Make sure that all the vaImages are destroyed */ - for (i = 0; i < resp->NumFrameActual; i++) { - GstMsdkMemoryID *mem = resp->mids[i]; - - /* Release prime fd if used */ - if (mem->desc.num_objects) { - g_assert (mem->desc.num_objects == 1); - close (mem->desc.objects[0].fd); - mem->desc.num_objects = 0; - } - - if (mem->image.image_id != VA_INVALID_ID && - vaDestroyImage (dpy, mem->image.image_id) == VA_STATUS_SUCCESS) { - mem_id->image.image_id = VA_INVALID_ID; - mem_id->image.buf = VA_INVALID_ID; - } - } - - va_status = - vaDestroySurfaces (dpy, (VASurfaceID *) mem_id->surface, - resp->NumFrameActual); - } else { - VASurfaceID *surfaces = mem_id->surface; - - for (i = 0; i < resp->NumFrameActual; i++) { - va_status = vaDestroyBuffer (dpy, surfaces[i]); - } - } - - g_slice_free1 (resp->NumFrameActual * sizeof (VASurfaceID), mem_id->surface); - g_slice_free1 (resp->NumFrameActual * sizeof (GstMsdkMemoryID), mem_id); g_slice_free1 (resp->NumFrameActual * sizeof (mfxMemId), resp->mids); - status = gst_msdk_get_mfx_status_from_va_status (va_status); - return status; + return MFX_ERR_NONE; } mfxStatus @@ -505,7 +248,7 @@ gst_msdk_frame_lock (mfxHDL pthis, mfxMemId mid, mfxFrameData * data) VAStatus va_status; mfxStatus status; mfxU8 *buf = NULL; - VASurfaceID *va_surface; + VASurfaceID va_surface; VADisplay dpy; GstMsdkMemoryID *mem_id; @@ -519,7 +262,7 @@ gst_msdk_frame_lock (mfxHDL pthis, mfxMemId mid, mfxFrameData * data) } if (mem_id->fourcc != MFX_FOURCC_P8) { - va_status = vaDeriveImage (dpy, *va_surface, &mem_id->image); + va_status = vaDeriveImage (dpy, va_surface, &mem_id->image); status = gst_msdk_get_mfx_status_from_va_status (va_status); if (status != MFX_ERR_NONE) { @@ -647,7 +390,7 @@ gst_msdk_frame_lock (mfxHDL pthis, mfxMemId mid, mfxFrameData * data) } else { VACodedBufferSegment *coded_buffer_segment; va_status = - vaMapBuffer (dpy, *va_surface, (void **) (&coded_buffer_segment)); + vaMapBuffer (dpy, va_surface, (void **) (&coded_buffer_segment)); status = gst_msdk_get_mfx_status_from_va_status (va_status); if (MFX_ERR_NONE == status) data->Y = (mfxU8 *) coded_buffer_segment->buf; @@ -679,7 +422,7 @@ gst_msdk_frame_unlock (mfxHDL pthis, mfxMemId mid, mfxFrameData * ptr) mem_id->image.buf = VA_INVALID_ID; } } else { - va_status = vaUnmapBuffer (dpy, *(mem_id->surface)); + va_status = vaUnmapBuffer (dpy, mem_id->surface); } status = gst_msdk_get_mfx_status_from_va_status (va_status); @@ -696,7 +439,7 @@ gst_msdk_frame_get_hdl (mfxHDL pthis, mfxMemId mid, mfxHDL * hdl) return MFX_ERR_INVALID_HANDLE; mem_id = mid; - *hdl = mem_id->surface; + *hdl = &mem_id->surface; return MFX_ERR_NONE; } @@ -970,8 +713,7 @@ gst_msdk_import_to_msdk_surface (GstBuffer * buf, GstMsdkContext * msdk_context, mfx_surface = g_slice_new0 (mfxFrameSurface1); msdk_mid = g_slice_new0 (GstMsdkMemoryID); - msdk_mid->surface = g_slice_new0 (VASurfaceID); - *msdk_mid->surface = va_surface; + msdk_mid->surface = va_surface; mfx_surface->Data.MemId = (mfxMemId) msdk_mid; @@ -1003,7 +745,7 @@ gst_msdk_replace_mfx_memid (GstMsdkContext * context, { GstMsdkMemoryID *msdk_mid = NULL; VADisplay dpy; - VASurfaceID *old_surface_id; + VASurfaceID old_surface_id; VAStatus va_status; mfxStatus status = MFX_ERR_NONE; @@ -1024,14 +766,14 @@ gst_msdk_replace_mfx_memid (GstMsdkContext * context, /* Destroy the associated VASurface */ old_surface_id = msdk_mid->surface; - if (*old_surface_id != VA_INVALID_ID) { - va_status = vaDestroySurfaces (dpy, old_surface_id, 1); + if (old_surface_id != VA_INVALID_ID) { + va_status = vaDestroySurfaces (dpy, &old_surface_id, 1); status = gst_msdk_get_mfx_status_from_va_status (va_status); if (status != MFX_ERR_NONE) goto error_destroy_va_surface; } - *msdk_mid->surface = surface_id; + msdk_mid->surface = surface_id; return TRUE; @@ -1042,7 +784,7 @@ error_destroy_va_image: } error_destroy_va_surface: { - GST_ERROR ("Failed to Destroy the VASurfaceID %x", *old_surface_id); + GST_ERROR ("Failed to Destroy the VASurfaceID %x", old_surface_id); return FALSE; } } diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c index df7bc6f26b..eb44301e16 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c @@ -736,25 +736,6 @@ gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext * return NULL; } -static void -create_surfaces (GstMsdkContext * context, GstMsdkAllocResponse * resp) -{ - gint i; - mfxMemId *mem_id; - mfxFrameSurface1 *surface; - - for (i = 0; i < resp->response.NumFrameActual; i++) { - mem_id = resp->response.mids[i]; - surface = (mfxFrameSurface1 *) g_slice_new0 (mfxFrameSurface1); - if (!surface) { - GST_ERROR ("failed to allocate surface"); - break; - } - surface->Data.MemId = mem_id; - resp->surfaces_avail = g_list_prepend (resp->surfaces_avail, surface); - } -} - static void free_surface (gpointer surface) { @@ -775,8 +756,6 @@ gst_msdk_context_add_alloc_response (GstMsdkContext * context, { context->priv->cached_alloc_responses = g_list_prepend (context->priv->cached_alloc_responses, resp); - - create_surfaces (context, resp); } gboolean diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c index c95cf622b7..3ebbf30699 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c @@ -40,6 +40,7 @@ #include "gstmsdkvideomemory.h" #include "gstmsdksystemmemory.h" #include "gstmsdkcontextutil.h" +#include "gstmsdkallocator.h" #ifndef _WIN32 #include @@ -72,17 +73,9 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", #define gst_msdkdec_parent_class parent_class G_DEFINE_TYPE (GstMsdkDec, gst_msdkdec, GST_TYPE_VIDEO_DECODER); -typedef struct _MsdkSurface -{ - mfxFrameSurface1 *surface; - GstBuffer *buf; - GstVideoFrame data; - GstVideoFrame copy; -} MsdkSurface; - struct _MsdkDecTask { - MsdkSurface *surface; + GstMsdkSurface *surface; mfxSyncPoint sync_point; gboolean decode_only; @@ -155,39 +148,22 @@ gst_msdkdec_get_oldest_frame (GstVideoDecoder * decoder) } static inline void -free_surface (MsdkSurface * s) +free_surface (GstMsdkSurface * s) { gst_buffer_unref (s->buf); - g_slice_free (MsdkSurface, s); -} - -static void -unmap_frame (GstMsdkDec * thiz, MsdkSurface * s) -{ - if (s->copy.buffer) { - /* we allocate this buffer from down stream, we need ref-1 for it */ - gst_buffer_unref (s->copy.buffer); - gst_video_frame_unmap (&s->copy); - s->copy.buffer = NULL; - } - - if (s->data.buffer) { - gst_video_frame_unmap (&s->data); - s->data.buffer = NULL; - } + g_slice_free (GstMsdkSurface, s); } static void gst_msdkdec_free_unlocked_msdk_surfaces (GstMsdkDec * thiz) { GList *l; - MsdkSurface *surface; + GstMsdkSurface *surface; for (l = thiz->locked_msdk_surfaces; l;) { GList *next = l->next; surface = l->data; if (surface->surface->Data.Locked == 0) { - unmap_frame (thiz, surface); free_surface (surface); thiz->locked_msdk_surfaces = g_list_delete_link (thiz->locked_msdk_surfaces, l); @@ -196,101 +172,6 @@ gst_msdkdec_free_unlocked_msdk_surfaces (GstMsdkDec * thiz) } } -static GstFlowReturn -allocate_output_buffer (GstMsdkDec * thiz, GstBuffer ** buffer) -{ - GstFlowReturn flow; - GstVideoCodecFrame *frame; - GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz); - - frame = gst_msdkdec_get_oldest_frame (decoder); - if (!frame) { - if (GST_PAD_IS_FLUSHING (decoder->srcpad)) - return GST_FLOW_FLUSHING; - else - return GST_FLOW_CUSTOM_SUCCESS; - } - - if (!frame->output_buffer) { - /* Free un-unsed msdk surfaces firstly, hence the associated mfx - * surfaces will be moved from used list to available list */ - gst_msdkdec_free_unlocked_msdk_surfaces (thiz); - - flow = gst_video_decoder_allocate_output_frame (decoder, frame); - if (flow != GST_FLOW_OK) { - gst_video_codec_frame_unref (frame); - return flow; - } - } - - *buffer = gst_buffer_ref (frame->output_buffer); - gst_buffer_replace (&frame->output_buffer, NULL); - gst_video_codec_frame_unref (frame); - - return GST_FLOW_OK; -} - -static MsdkSurface * -get_surface (GstMsdkDec * thiz, GstBuffer * buffer) -{ - MsdkSurface *i; - GstVideoCodecState *output_state = NULL; - gboolean success; - - i = g_slice_new0 (MsdkSurface); - - if (gst_msdk_is_msdk_buffer (buffer)) { - i->surface = gst_msdk_get_surface_from_buffer (buffer); - i->buf = buffer; - } else { - /* Confirm to activate the side pool */ - if (!gst_buffer_pool_is_active (thiz->pool) && - !gst_buffer_pool_set_active (thiz->pool, TRUE)) { - g_slice_free (MsdkSurface, i); - return NULL; - } - - if (!gst_video_frame_map (&i->copy, &thiz->non_msdk_pool_info, buffer, - GST_MAP_WRITE)) - goto failed_unref_buffer; - - if (gst_buffer_pool_acquire_buffer (thiz->pool, &buffer, - NULL) != GST_FLOW_OK) - goto failed_unmap_copy; - - i->surface = gst_msdk_get_surface_from_buffer (buffer); - i->buf = buffer; - - output_state = - gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz)); - success = - gst_video_frame_map (&i->data, &output_state->info, buffer, - GST_MAP_READWRITE); - gst_video_codec_state_unref (output_state); - if (!success) - goto failed_unref_buffer2; - } - - if (!thiz->sfc) - gst_msdk_update_mfx_frame_info_from_mfx_video_param (&i->surface->Info, - &thiz->param); - - thiz->locked_msdk_surfaces = g_list_append (thiz->locked_msdk_surfaces, i); - return i; - -failed_unref_buffer2: - gst_buffer_unref (buffer); - buffer = i->data.buffer; -failed_unmap_copy: - gst_video_frame_unmap (&i->copy); -failed_unref_buffer: - gst_buffer_unref (buffer); - g_slice_free (MsdkSurface, i); - - GST_ERROR_OBJECT (thiz, "failed to handle buffer"); - return NULL; -} - static GstMsdkSurface * allocate_output_surface (GstMsdkDec * thiz) { @@ -697,7 +578,7 @@ static gboolean gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation) { GstVideoCodecState *output_state; - GstVideoInfo *vinfo; + GstVideoInfo vinfo; GstVideoAlignment align; GstCaps *allocation_caps = NULL; GstCaps *allowed_caps = NULL, *temp_caps; @@ -820,13 +701,13 @@ gst_msdkdec_set_src_caps (GstMsdkDec * thiz, gboolean need_allocation) /* Ensure output_state->caps and info have same width and height * Also, mandate 32 bit alignment */ - vinfo = &output_state->info; + vinfo = output_state->info; if (width == out_width || height == out_height) - gst_msdk_set_video_alignment (vinfo, 0, 0, &align); + gst_msdk_set_video_alignment (&vinfo, 0, 0, &align); else - gst_msdk_set_video_alignment (vinfo, alloc_w, alloc_h, &align); - gst_video_info_align (vinfo, &align); - output_state->caps = gst_video_info_to_caps (vinfo); + gst_msdk_set_video_alignment (&vinfo, alloc_w, alloc_h, &align); + gst_video_info_align (&vinfo, &align); + output_state->caps = gst_video_info_to_caps (&vinfo); #ifndef _WIN32 if (pad_accept_memory (thiz, GST_CAPS_FEATURE_MEMORY_VA, output_state->caps)) { gst_caps_set_features (output_state->caps, 0, @@ -906,7 +787,7 @@ gst_msdkdec_set_latency (GstMsdkDec * thiz) static gint _find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface) { - MsdkSurface *cached_surface = (MsdkSurface *) msdk_surface; + GstMsdkSurface *cached_surface = (GstMsdkSurface *) msdk_surface; mfxFrameSurface1 *_surface = (mfxFrameSurface1 *) comp_surface; return cached_surface ? cached_surface->surface != _surface : -1; @@ -915,11 +796,8 @@ _find_msdk_surface (gconstpointer msdk_surface, gconstpointer comp_surface) static void finish_task (GstMsdkDec * thiz, MsdkDecTask * task) { - MsdkSurface *surface = task->surface; + GstMsdkSurface *surface = task->surface; if (surface) { - if (G_UNLIKELY (surface->copy.buffer)) { - unmap_frame (thiz, surface); - } thiz->locked_msdk_surfaces = g_list_append (thiz->locked_msdk_surfaces, surface); } @@ -959,13 +837,76 @@ gst_msdkdec_frame_corruption_report (GstMsdkDec * thiz, mfxU16 corruption) ("[Corruption] Corrupted reference list!"), (NULL)); } +static gboolean +_copy_to_sys_mem (GstMsdkDec * thiz, GstMsdkSurface * surface, + GstVideoCodecFrame * frame) +{ + GstBuffer *buffer = NULL; + GstVideoFrame src_frame; + GstVideoFrame dst_frame; + GstVideoInfo *src_info; + GstVideoInfo dst_info; + GstVideoCodecState *output_state = + gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz)); + + src_info = &output_state->info; + gst_video_info_set_format (&dst_info, GST_VIDEO_INFO_FORMAT (src_info), + GST_VIDEO_INFO_WIDTH (src_info), GST_VIDEO_INFO_HEIGHT (src_info)); + + if (!gst_buffer_pool_is_active (thiz->other_pool) && + !gst_buffer_pool_set_active (thiz->other_pool, TRUE)) { + GST_ERROR_OBJECT (thiz, "Failed to activate buffer pool"); + goto error_active; + } + + if (gst_buffer_pool_acquire_buffer (thiz->other_pool, &buffer, NULL) + != GST_FLOW_OK) { + GST_ERROR ("Failed to acquire buffer from pool"); + goto error; + } + + if (!gst_video_frame_map (&src_frame, src_info, surface->buf, GST_MAP_READ)) { + GST_ERROR_OBJECT (thiz, "Failed to map buf to src frame"); + goto error; + } + + if (!gst_video_frame_map (&dst_frame, &dst_info, buffer, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (thiz, "Failed to map buf to dst frame"); + gst_video_frame_unmap (&src_frame); + goto error; + } + + if (!gst_video_frame_copy (&dst_frame, &src_frame)) { + GST_ERROR_OBJECT (thiz, "Failed to copy surface data"); + gst_video_frame_unmap (&src_frame); + gst_video_frame_unmap (&dst_frame); + goto error; + } + + frame->output_buffer = buffer; + gst_video_frame_unmap (&src_frame); + gst_video_frame_unmap (&dst_frame); + gst_video_codec_state_unref (output_state); + + return TRUE; + +error: + gst_buffer_unref (buffer); + gst_buffer_pool_set_active (thiz->other_pool, FALSE); + gst_object_unref (thiz->other_pool); + +error_active: + gst_video_codec_state_unref (output_state); + return FALSE; +} + static GstFlowReturn gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task) { GstVideoDecoder *decoder = GST_VIDEO_DECODER (thiz); GstFlowReturn flow; GstVideoCodecFrame *frame; - MsdkSurface *surface; + GstMsdkSurface *surface; mfxStatus status; guint64 pts = MFX_TIMESTAMP_UNKNOWN; @@ -1004,16 +945,17 @@ gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task) } if (G_LIKELY (frame)) { - if (G_LIKELY (surface->copy.buffer == NULL)) { + if (!thiz->do_copy) { /* gst_video_decoder_finish_frame will call gst_buffer_make_writable - * we need this to avoid copy buffer */ + * we need this to avoid copy buffer */ GST_MINI_OBJECT_FLAG_SET (surface->buf, GST_MINI_OBJECT_FLAG_LOCKABLE); frame->output_buffer = gst_buffer_ref (surface->buf); } else { - gst_video_frame_copy (&surface->copy, &surface->data); - frame->output_buffer = gst_buffer_ref (surface->copy.buffer); - unmap_frame (thiz, surface); + /* We need to do the copy from video memory to system memory */ + if (!_copy_to_sys_mem (thiz, surface, frame)) + return GST_FLOW_ERROR; } + GST_DEBUG_OBJECT (thiz, "surface %p TimeStamp: %" G_GUINT64_FORMAT " frame %p TimeStamp: %" G_GUINT64_FORMAT, surface->surface, (guint64) surface->surface->Data.TimeStamp, @@ -1035,6 +977,7 @@ gst_msdkdec_finish_task (GstMsdkDec * thiz, MsdkDecTask * task) return flow; } finish_task (thiz, task); + return GST_FLOW_OK; } @@ -1142,6 +1085,10 @@ gst_msdkdec_stop (GstVideoDecoder * decoder) gst_object_unref (thiz->pool); thiz->pool = NULL; } + if (thiz->other_pool) { + gst_object_unref (thiz->other_pool); + thiz->other_pool = NULL; + } gst_video_info_init (&thiz->non_msdk_pool_info); gst_msdkdec_close_decoder (thiz, TRUE); @@ -1177,13 +1124,12 @@ static void release_msdk_surfaces (GstMsdkDec * thiz) { GList *l; - MsdkSurface *surface; + GstMsdkSurface *surface; gint locked = 0; gst_msdkdec_free_unlocked_msdk_surfaces (thiz); for (l = thiz->locked_msdk_surfaces; l; l = l->next) { - surface = (MsdkSurface *) l->data; - unmap_frame (thiz, surface); + surface = (GstMsdkSurface *) l->data; free_surface (surface); locked++; } @@ -1283,7 +1229,7 @@ find_msdk_surface (GstMsdkDec * thiz, MsdkDecTask * task, GST_ERROR_OBJECT (thiz, "msdk return an invalid surface %p", out_surface); return FALSE; } - task->surface = (MsdkSurface *) l->data; + task->surface = (GstMsdkSurface *) l->data; thiz->locked_msdk_surfaces = g_list_delete_link (thiz->locked_msdk_surfaces, l); return TRUE; @@ -1356,11 +1302,11 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) GstMsdkDec *thiz = GST_MSDKDEC (decoder); GstMsdkDecClass *klass = GST_MSDKDEC_GET_CLASS (thiz); GstFlowReturn flow; - GstBuffer *buffer, *input_buffer = NULL; + GstBuffer *input_buffer = NULL; GstVideoInfo alloc_info; MsdkDecTask *task = NULL; mfxBitstream bitstream; - MsdkSurface *surface = NULL; + GstMsdkSurface *surface = NULL; mfxFrameSurface1 *out_surface = NULL; mfxSession session; mfxStatus status; @@ -1530,13 +1476,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) goto error; } if (!surface) { - flow = allocate_output_buffer (thiz, &buffer); - if (flow == GST_FLOW_CUSTOM_SUCCESS) { - flow = GST_FLOW_OK; - break; - } else if (flow != GST_FLOW_OK) - goto error; - surface = get_surface (thiz, buffer); + surface = allocate_output_surface (thiz); if (!surface) { /* Can't get a surface for some reason; finish tasks, then see if a surface becomes available. */ @@ -1546,7 +1486,7 @@ gst_msdkdec_handle_frame (GstVideoDecoder * decoder, GstVideoCodecFrame * frame) flow = gst_msdkdec_finish_task (thiz, task); if (flow != GST_FLOW_OK) goto error; - surface = get_surface (thiz, buffer); + surface = allocate_output_surface (thiz); if (surface) break; } @@ -1711,91 +1651,6 @@ gst_msdkdec_parse (GstVideoDecoder * decoder, GstVideoCodecFrame * frame, return ret; } -static GstBufferPool * -gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info, - guint num_buffers) -{ - GstBufferPool *pool = NULL; - GstStructure *config; - GstAllocator *allocator = NULL; - GstVideoAlignment align; - GstCaps *caps = NULL; - GstAllocationParams params = { 0, 31, 0, 0, }; - mfxFrameAllocResponse *alloc_resp = NULL; - - g_return_val_if_fail (info, NULL); - g_return_val_if_fail (GST_VIDEO_INFO_WIDTH (info) - && GST_VIDEO_INFO_HEIGHT (info), NULL); - - alloc_resp = &thiz->alloc_resp; - - pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp); - if (!pool) - goto error_no_pool; - - caps = gst_video_info_to_caps (info); - - /* allocators should use the same width/height/stride/height_alignment of - * negotiated output caps, which is what we configure in msdk_allocator */ - if (thiz->use_dmabuf) - allocator = gst_msdk_dmabuf_allocator_new (thiz->context, info, alloc_resp); - else if (thiz->use_video_memory) - allocator = gst_msdk_video_allocator_new (thiz->context, info, alloc_resp); - else - allocator = gst_msdk_system_allocator_new (info); - - if (!allocator) { - gst_caps_unref (caps); - goto error_no_allocator; - } - - config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool)); - /* we need register all bufffers when we create the msdk context, so the buffer pool is not resize able */ - gst_buffer_pool_config_set_params (config, caps, - GST_VIDEO_INFO_SIZE (info), num_buffers, num_buffers); - gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); - gst_caps_unref (caps); - - if (thiz->use_video_memory) { - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_MSDK_USE_VIDEO_MEMORY); - if (thiz->use_dmabuf) - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_MSDK_USE_DMABUF); - } - - - gst_buffer_pool_config_set_video_alignment (config, &align); - gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); - gst_object_unref (allocator); - - if (!gst_buffer_pool_set_config (pool, config)) - goto error_pool_config; - - return pool; - -error_no_pool: - { - GST_INFO_OBJECT (thiz, "failed to create bufferpool"); - return NULL; - } -error_no_allocator: - { - GST_INFO_OBJECT (thiz, "failed to create allocator"); - gst_object_unref (pool); - return NULL; - } -error_pool_config: - { - GST_INFO_OBJECT (thiz, "failed to set config"); - gst_object_unref (pool); - gst_object_unref (allocator); - return NULL; - } -} - #ifndef _WIN32 static GstBufferPool * gst_msdk_create_va_pool (GstMsdkDec * thiz, GstVideoInfo * info, @@ -1840,7 +1695,7 @@ gst_msdk_create_va_pool (GstMsdkDec * thiz, GstVideoInfo * info, #endif static GstBufferPool * -gst_msdkdec_create_buffer_pool2 (GstMsdkDec * thiz, GstVideoInfo * info, +gst_msdkdec_create_buffer_pool (GstMsdkDec * thiz, GstVideoInfo * info, guint num_buffers) { GstBufferPool *pool = NULL; @@ -1903,7 +1758,7 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) GstStructure *pool_config = NULL; GstCaps *pool_caps /*, *negotiated_caps */ ; guint size, min_buffers, max_buffers; - GstAllocator *allocator = NULL; + gboolean has_videometa, has_video_alignment; if (!thiz->param.mfx.FrameInfo.Width || !thiz->param.mfx.FrameInfo.Height) return FALSE; @@ -1918,6 +1773,11 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); pool_config = gst_buffer_pool_get_config (pool); + has_videometa = gst_query_find_allocation_meta + (query, GST_VIDEO_META_API_TYPE, NULL); + has_video_alignment = gst_buffer_pool_has_option + (pool, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + /* Get the caps of pool and increase the min and max buffers by async_depth. * We will always have that number of decode operations in-flight */ gst_buffer_pool_config_get_params (pool_config, &pool_caps, &size, @@ -1933,21 +1793,8 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) thiz->min_prealloc_buffers = min_buffers; if (_gst_caps_has_feature (pool_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) { - GST_INFO_OBJECT (decoder, "This MSDK decoder uses DMABuf memory"); - thiz->use_video_memory = thiz->use_dmabuf = TRUE; - } else if (thiz->sfc) - thiz->use_video_memory = TRUE; - - /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer, - * which requires information about frame allocation. - * No effect if already initialized. - */ - if (!gst_msdkdec_init_decoder (thiz)) - return FALSE; - - /* get the updated min_buffers, which account for the msdk requirement as well */ - min_buffers = thiz->min_prealloc_buffers; - + thiz->use_dmabuf = TRUE; + } /* Decoder always use its own pool. So we create a pool if msdk APIs * previously requested for allocation (do_realloc = TRUE) */ if (thiz->do_realloc || !thiz->pool) { @@ -1963,24 +1810,22 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) goto failed_to_create_pool; } } - - +#ifndef _WIN32 + GstAllocator *allocator = NULL; if (gst_query_get_n_allocation_params (query) > 0) { gst_query_parse_nth_allocation_param (query, 0, &allocator, NULL); - if (!(GST_IS_MSDK_VIDEO_ALLOCATOR (allocator) || - GST_IS_MSDK_DMABUF_ALLOCATOR (allocator) || - GST_IS_MSDK_SYSTEM_ALLOCATOR (allocator))) - thiz->ds_has_no_msdk_allocator = TRUE; + if (!(GST_IS_VA_ALLOCATOR (allocator) || + GST_IS_VA_DMABUF_ALLOCATOR (allocator))) + thiz->ds_has_known_allocator = FALSE; } +#endif - /* If downstream supports video meta and video alignment, - * or downstream doesn't have msdk_allocator, we can replace - * with our own msdk bufferpool and use it. + /* If downstream supports video meta and video alignment, or downstream + * doesn't have known allocator (known allocator refers to va allocator + * or d3d allocator), we replace with our own bufferpool and use it. */ - if ((gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL) - && gst_buffer_pool_has_option - (pool, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT)) - || thiz->ds_has_no_msdk_allocator) { + if ((has_videometa && has_video_alignment) + || !thiz->ds_has_known_allocator) { GstStructure *config; GstAllocator *allocator; @@ -1997,9 +1842,9 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) gst_query_set_nth_allocation_param (query, 0, allocator, NULL); gst_structure_free (config); } else { - /* Unfortunately, downstream doesn't have videometa or alignment support, - * we keep msdk pool as a side-pool that will be decoded into and - * then copied from. + /* When downstream doesn't have videometa or alignment support, + * or downstream pool is va/d3d pool,we will use downstream pool + * and keep decoder's own pool as side-pool. */ GstVideoCodecState *output_state = NULL; @@ -2023,9 +1868,33 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) GST_VIDEO_INFO_HEIGHT (&thiz->non_msdk_pool_info) = GST_VIDEO_INFO_HEIGHT (&output_state->info); - /* When downstream allocator is unknown and negotiaed caps is raw, - * we need to create other pool with system memory for copy use + gst_video_codec_state_unref (output_state); + } + + gst_msdk_context_set_alloc_pool (thiz->context, pool); + + /* Initialize MSDK decoder before new bufferpool tries to alloc each buffer, + * which requires information about frame allocation. + * No effect if already initialized. + */ + if (!gst_msdkdec_init_decoder (thiz)) + return FALSE; + + /* get the updated min_buffers, which account for the msdk requirement as well */ + min_buffers = thiz->min_prealloc_buffers; + + if (!has_videometa && !thiz->ds_has_known_allocator + && _gst_caps_has_feature (pool_caps, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY)) { + /* We need to create other pool with system memory for copy use under conditions: + * (1) downstream has no videometa; (2) downstream allocator is unknown; + * (3) negotiated caps is raw. */ + thiz->do_copy = TRUE; + GstVideoCodecState *output_state = + gst_video_decoder_get_output_state (GST_VIDEO_DECODER (thiz)); + thiz->other_pool = + gst_msdkdec_create_buffer_pool (thiz, &output_state->info, min_buffers); gst_video_codec_state_unref (output_state); } @@ -2035,7 +1904,6 @@ gst_msdkdec_decide_allocation (GstVideoDecoder * decoder, GstQuery * query) if (pool) gst_object_unref (pool); - return TRUE; failed_to_create_pool: @@ -2056,9 +1924,8 @@ gst_msdkdec_drain (GstVideoDecoder * decoder) { GstMsdkDec *thiz = GST_MSDKDEC (decoder); GstFlowReturn flow; - GstBuffer *buffer; MsdkDecTask *task; - MsdkSurface *surface = NULL; + GstMsdkSurface *surface = NULL; mfxFrameSurface1 *out_surface; mfxSession session; mfxStatus status; @@ -2078,10 +1945,7 @@ gst_msdkdec_drain (GstVideoDecoder * decoder) } if (!surface) { - flow = allocate_output_buffer (thiz, &buffer); - if (flow != GST_FLOW_OK) - return flow; - surface = get_surface (thiz, buffer); + surface = allocate_output_surface (thiz); if (!surface) return GST_FLOW_ERROR; } @@ -2384,9 +2248,12 @@ gst_msdkdec_init (GstMsdkDec * thiz) thiz->force_reset_on_res_change = TRUE; thiz->report_error = FALSE; thiz->sfc = FALSE; - thiz->ds_has_no_msdk_allocator = FALSE; + thiz->ds_has_known_allocator = TRUE; thiz->adapter = gst_adapter_new (); thiz->input_state = NULL; thiz->pool = NULL; thiz->context = NULL; +#ifndef _WIN32 + thiz->use_video_memory = TRUE; +#endif } diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.h index fa9ea0e176..2ff2f8d795 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.h @@ -79,7 +79,7 @@ struct _GstMsdkDec gboolean do_copy; gboolean initialized; gboolean sfc; - gboolean ds_has_no_msdk_allocator; + gboolean ds_has_known_allocator; /* for packetization */ GstAdapter *adapter;