From b5ed0eb4b0eeacd44cc8b40e97028fd90bc0e586 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Fri, 25 Mar 2022 22:04:05 +0900 Subject: [PATCH] qsvencoder: Add support for VA memory Use VA allocator and buffer pool implementation for zero-copy encoding with upstream VA elements Part-of: --- .../sys/qsv/gstqsvallocator_va.cpp | 64 ++++++++++- .../gst-plugins-bad/sys/qsv/gstqsvencoder.cpp | 108 +++++++++++++----- .../gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp | 9 +- .../gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp | 9 +- .../gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp | 9 +- 5 files changed, 164 insertions(+), 35 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_va.cpp b/subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_va.cpp index 8d2ad8bd04..e30e537dd3 100644 --- a/subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_va.cpp +++ b/subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_va.cpp @@ -87,9 +87,69 @@ static GstBuffer * gst_qsv_va_allocator_upload (GstQsvAllocator * allocator, const GstVideoInfo * info, GstBuffer * buffer, GstBufferPool * pool) { - GST_ERROR_OBJECT (allocator, "Not implemented"); + GstVideoFrame src_frame, dst_frame; + VASurfaceID surface; + GstBuffer *dst_buf; + GstFlowReturn ret; - return nullptr; + /* TODO: handle buffer from different VA display */ + surface = gst_va_buffer_get_surface (buffer); + if (surface != VA_INVALID_ID) + return gst_buffer_ref (buffer); + + ret = gst_buffer_pool_acquire_buffer (pool, &dst_buf, nullptr); + if (ret != GST_FLOW_OK) { + GST_WARNING_OBJECT (allocator, "Failed to acquire buffer"); + return nullptr; + } + + if (!gst_video_frame_map (&src_frame, info, buffer, GST_MAP_READ)) { + GST_WARNING_OBJECT (allocator, "Failed to map src frame"); + gst_buffer_unref (dst_buf); + return nullptr; + } + + if (!gst_video_frame_map (&dst_frame, info, dst_buf, GST_MAP_WRITE)) { + GST_WARNING_OBJECT (allocator, "Failed to map src frame"); + gst_video_frame_unmap (&src_frame); + gst_buffer_unref (dst_buf); + return nullptr; + } + + for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) { + guint src_width_in_bytes, src_height; + guint dst_width_in_bytes, dst_height; + guint width_in_bytes, height; + guint src_stride, dst_stride; + guint8 *src_data, *dst_data; + + src_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&src_frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i); + src_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i); + src_stride = GST_VIDEO_FRAME_COMP_STRIDE (&src_frame, i); + + dst_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&dst_frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i); + dst_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i); + dst_stride = GST_VIDEO_FRAME_COMP_STRIDE (&dst_frame, i); + + width_in_bytes = MIN (src_width_in_bytes, dst_width_in_bytes); + height = MIN (src_height, dst_height); + + src_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&src_frame, i); + dst_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&dst_frame, i); + + for (guint j = 0; j < height; j++) { + memcpy (dst_data, src_data, width_in_bytes); + dst_data += dst_stride; + src_data += src_stride; + } + } + + gst_video_frame_unmap (&dst_frame); + gst_video_frame_unmap (&src_frame); + + return dst_buf; } static GstBuffer * diff --git a/subprojects/gst-plugins-bad/sys/qsv/gstqsvencoder.cpp b/subprojects/gst-plugins-bad/sys/qsv/gstqsvencoder.cpp index 6f1b18a559..213b56500f 100644 --- a/subprojects/gst-plugins-bad/sys/qsv/gstqsvencoder.cpp +++ b/subprojects/gst-plugins-bad/sys/qsv/gstqsvencoder.cpp @@ -854,8 +854,6 @@ gst_qsv_encoder_prepare_d3d11_pool (GstQsvEncoder * self, GstD3D11AllocationParams *params; GstD3D11Device *device = GST_D3D11_DEVICE_CAST (priv->device); - GST_DEBUG_OBJECT (self, "Use d3d11 memory pool"); - priv->internal_pool = gst_d3d11_buffer_pool_new (device); config = gst_buffer_pool_get_config (priv->internal_pool); params = gst_d3d11_allocation_params_new (device, aligned_info, @@ -870,34 +868,57 @@ gst_qsv_encoder_prepare_d3d11_pool (GstQsvEncoder * self, return TRUE; } -#endif - +#else static gboolean -gst_qsv_encoder_prepare_system_pool (GstQsvEncoder * self, +gst_qsv_encoder_prepare_va_pool (GstQsvEncoder * self, GstCaps * caps, GstVideoInfo * aligned_info) { GstQsvEncoderPrivate *priv = self->priv; + GstAllocator *allocator; GstStructure *config; + GArray *formats; + GstAllocationParams params; + GstVaDisplay *display = GST_VA_DISPLAY (priv->device); - GST_DEBUG_OBJECT (self, "Use system memory pool"); + formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat)); + g_array_append_val (formats, GST_VIDEO_INFO_FORMAT (aligned_info)); + + allocator = gst_va_allocator_new (display, formats); + if (!allocator) { + GST_ERROR_OBJECT (self, "Failed to create allocator"); + return FALSE; + } + + gst_allocation_params_init (¶ms); + + priv->internal_pool = gst_va_pool_new_with_config (caps, + GST_VIDEO_INFO_SIZE (aligned_info), 0, 0, + VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER, GST_VA_FEATURE_DISABLED, + allocator, ¶ms); + gst_object_unref (allocator); + + + if (!priv->internal_pool) { + GST_ERROR_OBJECT (self, "Failed to create va pool"); + return FALSE; + } - priv->internal_pool = gst_video_buffer_pool_new (); config = gst_buffer_pool_get_config (priv->internal_pool); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - gst_buffer_pool_config_set_params (config, - caps, GST_VIDEO_INFO_SIZE (aligned_info), 0, 0); - + gst_buffer_pool_config_set_params (config, caps, + GST_VIDEO_INFO_SIZE (aligned_info), 0, 0); gst_buffer_pool_set_config (priv->internal_pool, config); gst_buffer_pool_set_active (priv->internal_pool, TRUE); return TRUE; } +#endif /* Prepare internal pool, which is used to allocate fallback buffer * when upstream buffer is not directly accessible by QSV */ static gboolean gst_qsv_encoder_prepare_pool (GstQsvEncoder * self, GstCaps * caps, - GstVideoInfo * aligned_info, mfxU16 * io_pattern) + GstVideoInfo * aligned_info) { GstQsvEncoderPrivate *priv = self->priv; gboolean ret = FALSE; @@ -910,21 +931,12 @@ gst_qsv_encoder_prepare_pool (GstQsvEncoder * self, GstCaps * caps, aligned_caps = gst_video_info_to_caps (aligned_info); - /* TODO: Add Linux video memory (VA/DMABuf) support */ #ifdef G_OS_WIN32 - priv->mem_type = GST_QSV_VIDEO_MEMORY | GST_QSV_ENCODER_IN_MEMORY; - *io_pattern = MFX_IOPATTERN_IN_VIDEO_MEMORY; - ret = gst_qsv_encoder_prepare_d3d11_pool (self, aligned_caps, aligned_info); +#else + ret = gst_qsv_encoder_prepare_va_pool (self, aligned_caps, aligned_info); #endif - if (!ret) { - priv->mem_type = GST_QSV_SYSTEM_MEMORY | GST_QSV_ENCODER_IN_MEMORY; - *io_pattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; - - ret = gst_qsv_encoder_prepare_system_pool (self, - aligned_caps, aligned_info); - } gst_caps_unref (aligned_caps); return ret; @@ -976,8 +988,10 @@ gst_qsv_encoder_init_encode_session (GstQsvEncoder * self) GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_INTERLACE_MODE (info), frame_info->Width, frame_info->Height); - if (!gst_qsv_encoder_prepare_pool (self, caps, &priv->aligned_info, - ¶m.IOPattern)) { + /* Always video memory, even when upstream is non-hardware element */ + priv->mem_type = GST_QSV_VIDEO_MEMORY | GST_QSV_ENCODER_IN_MEMORY; + param.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY; + if (!gst_qsv_encoder_prepare_pool (self, caps, &priv->aligned_info)) { GST_ERROR_OBJECT (self, "Failed to prepare pool"); goto error; } @@ -1428,18 +1442,20 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query) return TRUE; } #else -/* TODO: Add support VA/DMABuf */ static gboolean gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query) { GstQsvEncoder *self = GST_QSV_ENCODER (encoder); GstQsvEncoderPrivate *priv = self->priv; GstVideoInfo info; + GstAllocator *allocator = nullptr; GstBufferPool *pool; GstCaps *caps; guint size; GstStructure *config; GstVideoAlignment align; + GstAllocationParams params; + GArray *formats; gst_query_parse_allocation (query, &caps, nullptr); if (!caps) { @@ -1452,7 +1468,31 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query) return FALSE; } - pool = gst_video_buffer_pool_new (); + gst_allocation_params_init (¶ms); + + formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat)); + g_array_append_val (formats, GST_VIDEO_INFO_FORMAT (&info)); + + allocator = gst_va_allocator_new (GST_VA_DISPLAY (priv->device), formats); + if (!allocator) { + GST_ERROR_OBJECT (self, "Failed to create allocator"); + return FALSE; + } + + /* Will not use derived image + * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1110 + */ + pool = gst_va_pool_new_with_config (caps, + GST_VIDEO_INFO_SIZE (&info), priv->surface_pool->len, 0, + VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER, GST_VA_FEATURE_DISABLED, + allocator, ¶ms); + + if (!pool) { + GST_ERROR_OBJECT (self, "Failed to create va pool"); + gst_object_unref (allocator); + + return FALSE; + } gst_video_alignment_reset (&align); align.padding_right = GST_VIDEO_INFO_WIDTH (&priv->aligned_info) - @@ -1464,21 +1504,29 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query) gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); - gst_video_info_align (&info, &align); gst_buffer_pool_config_set_video_alignment (config, &align); - size = GST_VIDEO_INFO_SIZE (&info); gst_buffer_pool_config_set_params (config, - caps, size, priv->surface_pool->len, 0); + caps, GST_VIDEO_INFO_SIZE (&info), priv->surface_pool->len, 0); if (!gst_buffer_pool_set_config (pool, config)) { - GST_WARNING_OBJECT (self, "Failed to set pool config"); + GST_ERROR_OBJECT (self, "Failed to set pool config"); + gst_clear_object (&allocator); gst_object_unref (pool); return FALSE; } + if (allocator) + gst_query_add_allocation_param (query, allocator, ¶ms); + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr); + gst_structure_free (config); + gst_query_add_allocation_pool (query, pool, size, priv->surface_pool->len, 0); gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, nullptr); + + gst_clear_object (&allocator); gst_object_unref (pool); return TRUE; diff --git a/subprojects/gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp b/subprojects/gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp index 1a6b3af194..be948c93eb 100644 --- a/subprojects/gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp +++ b/subprojects/gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp @@ -32,7 +32,7 @@ #ifdef G_OS_WIN32 #include #else -#include +#include #endif GST_DEBUG_CATEGORY_EXTERN (gst_qsv_h264_enc_debug); @@ -1829,6 +1829,13 @@ gst_qsv_h264_enc_register (GstPlugin * plugin, guint rank, guint impl_index, gst_caps_set_features_simple (d3d11_caps, caps_features); gst_caps_append (d3d11_caps, sink_caps); sink_caps = d3d11_caps; +#else + GstCaps *va_caps = gst_caps_copy (sink_caps); + GstCapsFeatures *caps_features = + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr); + gst_caps_set_features_simple (va_caps, caps_features); + gst_caps_append (va_caps, sink_caps); + sink_caps = va_caps; #endif std::string src_caps_str = "video/x-h264"; diff --git a/subprojects/gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp b/subprojects/gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp index 7324952c7a..d90c1e5bd1 100644 --- a/subprojects/gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp +++ b/subprojects/gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp @@ -31,7 +31,7 @@ #ifdef G_OS_WIN32 #include #else -#include +#include #endif GST_DEBUG_CATEGORY_EXTERN (gst_qsv_h265_enc_debug); @@ -1410,6 +1410,13 @@ gst_qsv_h265_enc_register (GstPlugin * plugin, guint rank, guint impl_index, gst_caps_set_features_simple (d3d11_caps, caps_features); gst_caps_append (d3d11_caps, sink_caps); sink_caps = d3d11_caps; +#else + GstCaps *va_caps = gst_caps_copy (sink_caps); + GstCapsFeatures *caps_features = + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr); + gst_caps_set_features_simple (va_caps, caps_features); + gst_caps_append (va_caps, sink_caps); + sink_caps = va_caps; #endif std::string src_caps_str = "video/x-h265"; diff --git a/subprojects/gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp b/subprojects/gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp index 173f486a54..a992450e1f 100644 --- a/subprojects/gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp +++ b/subprojects/gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp @@ -30,7 +30,7 @@ #ifdef G_OS_WIN32 #include #else -#include +#include #endif GST_DEBUG_CATEGORY_EXTERN (gst_qsv_vp9_enc_debug); @@ -948,6 +948,13 @@ gst_qsv_vp9_enc_register (GstPlugin * plugin, guint rank, guint impl_index, gst_caps_set_features_simple (d3d11_caps, caps_features); gst_caps_append (d3d11_caps, sink_caps); sink_caps = d3d11_caps; +#else + GstCaps *va_caps = gst_caps_copy (sink_caps); + GstCapsFeatures *caps_features = + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr); + gst_caps_set_features_simple (va_caps, caps_features); + gst_caps_append (va_caps, sink_caps); + sink_caps = va_caps; #endif std::string src_caps_str = "video/x-vp9";