From 248baef400e56da9b7aec9f6016d2ac6794896f2 Mon Sep 17 00:00:00 2001 From: He Junyan Date: Sat, 8 May 2021 17:54:34 +0800 Subject: [PATCH] MSDK: Import VA surface as encoder's input. We make all MSDK encoders declare "memory:VAMemory" feature. Then the pipeline such as: gst-launch-1.0 -vf filesrc location=xxx.h264 ! h264parse ! \ vah264dec ! msdkh265enc ! fakesink will choose VA memory caps between the VA decoder and MSDK encoder. Part-of: --- .../gst-plugins-bad/sys/msdk/gstmsdkenc.c | 88 ++++++++++++++++++- .../gst-plugins-bad/sys/msdk/gstmsdkenc.h | 1 + .../gst-plugins-bad/sys/msdk/gstmsdkh265enc.c | 3 +- .../gst-plugins-bad/sys/msdk/gstmsdkvp9enc.c | 3 +- subprojects/gst-plugins-bad/sys/msdk/msdk.h | 4 + 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c index 5ead853cfb..f25ae4ab5e 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c @@ -53,6 +53,7 @@ #ifndef _WIN32 #include "gstmsdkallocator_libva.h" +#include "gstmsdk_va.h" #endif static inline void * @@ -81,7 +82,8 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_MSDK_CAPS_STR - ("{ NV12, I420, YV12, YUY2, UYVY, BGRA }", "NV12")) + ("{ NV12, I420, YV12, YUY2, UYVY, BGRA }", "NV12") "; " + GST_MSDK_CAPS_MAKE_WITH_VA_FEATURE ("NV12")) ); #define PROP_HARDWARE_DEFAULT TRUE @@ -117,6 +119,8 @@ typedef struct { mfxFrameSurface1 *surface; GstBuffer *buf; + GstBuffer *buf_external; + VASurfaceID cache_surface; } MsdkSurface; void @@ -954,6 +958,17 @@ gst_msdkenc_create_surface (mfxFrameSurface1 * surface, GstBuffer * buf) static void gst_msdkenc_free_surface (MsdkSurface * surface) { + if (surface->buf_external) { + GstMsdkMemoryID *msdk_mid = NULL; + mfxFrameSurface1 *mfx_surface = NULL; + + mfx_surface = surface->surface; + msdk_mid = (GstMsdkMemoryID *) mfx_surface->Data.MemId; + *msdk_mid->surface = surface->cache_surface; + + gst_buffer_unref (surface->buf_external); + } + if (surface->buf) gst_buffer_unref (surface->buf); @@ -1437,6 +1452,17 @@ done: return ret; } +static gboolean +sinkpad_is_va (GstMsdkEnc * thiz) +{ + GstCapsFeatures *const features = + gst_caps_get_features (thiz->input_state->caps, 0); + if (gst_caps_features_contains (features, "memory:VAMemory")) + return TRUE; + + return FALSE; +} + static gboolean gst_msdkenc_set_format (GstVideoEncoder * encoder, GstVideoCodecState * state) { @@ -1460,6 +1486,8 @@ gst_msdkenc_set_format (GstVideoEncoder * encoder, GstVideoCodecState * state) */ #ifndef _WIN32 thiz->use_video_memory = TRUE; + if (sinkpad_is_va (thiz)) + thiz->use_va = TRUE; #else thiz->use_video_memory = FALSE; #endif @@ -1476,7 +1504,8 @@ gst_msdkenc_set_format (GstVideoEncoder * encoder, GstVideoCodecState * state) * based pipeline usage. Ideally we should have dmabuf support even with * raw-caps negotiation, but we don't have dmabuf-import support in msdk * plugin yet */ - if (sinkpad_can_dmabuf (thiz)) { + /* If VA is set, we do not fallback to DMA. */ + if (!thiz->use_va && sinkpad_can_dmabuf (thiz)) { thiz->input_state->caps = gst_caps_make_writable (thiz->input_state->caps); gst_caps_set_features (thiz->input_state->caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL)); @@ -1640,6 +1669,54 @@ import_dmabuf_to_msdk_surface (GstMsdkEnc * thiz, GstBuffer * buf, return TRUE; } + +static gboolean +import_va_surface_to_msdk (GstMsdkEnc * thiz, GstBuffer * buf, + MsdkSurface * msdk_surface) +{ + GstVideoInfo vinfo; + GstVideoMeta *vmeta; + GstMsdkMemoryID *msdk_mid = NULL; + mfxFrameSurface1 *mfx_surface = NULL; + VASurfaceID surface; + gint i; + + surface = gst_msdk_va_peek_buffer_surface (buf); + if (surface == VA_INVALID_SURFACE) + return FALSE; + + vinfo = thiz->input_state->info; + /* Update offset/stride/size if there is VideoMeta attached to + * the buffer */ + vmeta = gst_buffer_get_video_meta (buf); + if (vmeta) { + if (GST_VIDEO_INFO_FORMAT (&vinfo) != vmeta->format || + GST_VIDEO_INFO_WIDTH (&vinfo) != vmeta->width || + GST_VIDEO_INFO_HEIGHT (&vinfo) != vmeta->height || + GST_VIDEO_INFO_N_PLANES (&vinfo) != vmeta->n_planes) { + GST_ERROR_OBJECT (thiz, "VideoMeta attached to buffer is not matching" + "the negotiated width/height/format"); + return FALSE; + } + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vinfo); ++i) { + GST_VIDEO_INFO_PLANE_OFFSET (&vinfo, i) = vmeta->offset[i]; + GST_VIDEO_INFO_PLANE_STRIDE (&vinfo, i) = vmeta->stride[i]; + } + GST_VIDEO_INFO_SIZE (&vinfo) = gst_buffer_get_size (buf); + } + + if (GST_VIDEO_INFO_SIZE (&vinfo) < GST_VIDEO_INFO_SIZE (&thiz->aligned_info)) + return FALSE; + + msdk_surface->buf_external = gst_buffer_ref (buf); + + mfx_surface = msdk_surface->surface; + msdk_mid = (GstMsdkMemoryID *) mfx_surface->Data.MemId; + + msdk_surface->cache_surface = *msdk_mid->surface; + *msdk_mid->surface = surface; + return TRUE; +} #endif static MsdkSurface * @@ -1668,6 +1745,13 @@ gst_msdkenc_get_surface_from_frame (GstMsdkEnc * thiz, goto error; #ifndef _WIN32 + /************** VA import *****************/ + if (thiz->use_va) { + if (import_va_surface_to_msdk (thiz, inbuf, msdk_surface)) { + return msdk_surface; + } + } + /************ dmabuf-import ************* */ /* if upstream provided a dmabuf backed memory, but not an msdk * buffer, we could try to export the dmabuf to underlined vasurface */ diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.h index 496c60d1f6..35e54b298c 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.h @@ -133,6 +133,7 @@ struct _GstMsdkEnc GstVideoInfo aligned_info; gboolean use_video_memory; gboolean use_dmabuf; + gboolean use_va; gboolean initialized; /* element properties */ diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkh265enc.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkh265enc.c index a68e5d302f..fe679e9a27 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkh265enc.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkh265enc.c @@ -134,7 +134,8 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_MSDK_CAPS_STR (COMMON_FORMAT, - "{ NV12, P010_10LE }"))); + "{ NV12, P010_10LE }") "; " + GST_MSDK_CAPS_MAKE_WITH_VA_FEATURE ("NV12"))); static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvp9enc.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvp9enc.c index a74dd16fc6..8014666f8a 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvp9enc.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvp9enc.c @@ -71,7 +71,8 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_MSDK_CAPS_STR (COMMON_FORMAT, - "{ NV12, P010_10LE }"))); + "{ NV12, P010_10LE }") "; " + GST_MSDK_CAPS_MAKE_WITH_VA_FEATURE ("NV12"))); static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, diff --git a/subprojects/gst-plugins-bad/sys/msdk/msdk.h b/subprojects/gst-plugins-bad/sys/msdk/msdk.h index 10e44b4ccc..c47107aa8b 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/msdk.h +++ b/subprojects/gst-plugins-bad/sys/msdk/msdk.h @@ -75,8 +75,12 @@ G_BEGIN_DECLS #define GST_MSDK_CAPS_MAKE_WITH_DMABUF_FEATURE(dmaformat) \ GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_DMABUF, dmaformat) ", " \ "interlace-mode = (string) progressive" +#define GST_MSDK_CAPS_MAKE_WITH_VA_FEATURE(vaformat) \ + GST_VIDEO_CAPS_MAKE_WITH_FEATURES("memory:VAMemory", vaformat) ", " \ + "interlace-mode = (string) progressive" #else #define GST_MSDK_CAPS_MAKE_WITH_DMABUF_FEATURE(dmaformat) "" +#define GST_MSDK_CAPS_MAKE_WITH_VA_FEATURE(vaformat) "" #endif #define GST_MSDK_CAPS_STR(format,dmaformat) \