qsvencoder: Add support for VA memory

Use VA allocator and buffer pool implementation for zero-copy
encoding with upstream VA elements

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2030>

subprojects/gst-plugins-bad/sys/qsv/gstqsvallocator_va.cpp

@@ -87,11 +87,71 @@ static GstBuffer *
 gst_qsv_va_allocator_upload (GstQsvAllocator * allocator,
     const GstVideoInfo * info, GstBuffer * buffer, GstBufferPool * pool)
 {
-  GST_ERROR_OBJECT (allocator, "Not implemented");
+  GstVideoFrame src_frame, dst_frame;
+  VASurfaceID surface;
+  GstBuffer *dst_buf;
+  GstFlowReturn ret;
 
+  /* TODO: handle buffer from different VA display */
+  surface = gst_va_buffer_get_surface (buffer);
+  if (surface != VA_INVALID_ID)
+    return gst_buffer_ref (buffer);
+
+  ret = gst_buffer_pool_acquire_buffer (pool, &dst_buf, nullptr);
+  if (ret != GST_FLOW_OK) {
+    GST_WARNING_OBJECT (allocator, "Failed to acquire buffer");
     return nullptr;
   }
 
+  if (!gst_video_frame_map (&src_frame, info, buffer, GST_MAP_READ)) {
+    GST_WARNING_OBJECT (allocator, "Failed to map src frame");
+    gst_buffer_unref (dst_buf);
+    return nullptr;
+  }
+
+  if (!gst_video_frame_map (&dst_frame, info, dst_buf, GST_MAP_WRITE)) {
+    GST_WARNING_OBJECT (allocator, "Failed to map src frame");
+    gst_video_frame_unmap (&src_frame);
+    gst_buffer_unref (dst_buf);
+    return nullptr;
+  }
+
+  for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) {
+    guint src_width_in_bytes, src_height;
+    guint dst_width_in_bytes, dst_height;
+    guint width_in_bytes, height;
+    guint src_stride, dst_stride;
+    guint8 *src_data, *dst_data;
+
+    src_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&src_frame, i) *
+        GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i);
+    src_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i);
+    src_stride = GST_VIDEO_FRAME_COMP_STRIDE (&src_frame, i);
+
+    dst_width_in_bytes = GST_VIDEO_FRAME_COMP_WIDTH (&dst_frame, i) *
+        GST_VIDEO_FRAME_COMP_PSTRIDE (&src_frame, i);
+    dst_height = GST_VIDEO_FRAME_COMP_HEIGHT (&src_frame, i);
+    dst_stride = GST_VIDEO_FRAME_COMP_STRIDE (&dst_frame, i);
+
+    width_in_bytes = MIN (src_width_in_bytes, dst_width_in_bytes);
+    height = MIN (src_height, dst_height);
+
+    src_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&src_frame, i);
+    dst_data = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (&dst_frame, i);
+
+    for (guint j = 0; j < height; j++) {
+      memcpy (dst_data, src_data, width_in_bytes);
+      dst_data += dst_stride;
+      src_data += src_stride;
+    }
+  }
+
+  gst_video_frame_unmap (&dst_frame);
+  gst_video_frame_unmap (&src_frame);
+
+  return dst_buf;
+}
+
 static GstBuffer *
 gst_qsv_va_allocator_download (GstQsvAllocator * allocator,
     const GstVideoInfo * info, gboolean force_copy, GstQsvFrame * frame,

subprojects/gst-plugins-bad/sys/qsv/gstqsvencoder.cpp

@@ -854,8 +854,6 @@ gst_qsv_encoder_prepare_d3d11_pool (GstQsvEncoder * self,
   GstD3D11AllocationParams *params;
   GstD3D11Device *device = GST_D3D11_DEVICE_CAST (priv->device);
 
-  GST_DEBUG_OBJECT (self, "Use d3d11 memory pool");
-
   priv->internal_pool = gst_d3d11_buffer_pool_new (device);
   config = gst_buffer_pool_get_config (priv->internal_pool);
   params = gst_d3d11_allocation_params_new (device, aligned_info,
@@ -870,34 +868,57 @@ gst_qsv_encoder_prepare_d3d11_pool (GstQsvEncoder * self,
 
   return TRUE;
 }
-#endif
-
+#else
 static gboolean
-gst_qsv_encoder_prepare_system_pool (GstQsvEncoder * self,
+gst_qsv_encoder_prepare_va_pool (GstQsvEncoder * self,
     GstCaps * caps, GstVideoInfo * aligned_info)
 {
   GstQsvEncoderPrivate *priv = self->priv;
+  GstAllocator *allocator;
   GstStructure *config;
+  GArray *formats;
+  GstAllocationParams params;
+  GstVaDisplay *display = GST_VA_DISPLAY (priv->device);
 
-  GST_DEBUG_OBJECT (self, "Use system memory pool");
+  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+  g_array_append_val (formats, GST_VIDEO_INFO_FORMAT (aligned_info));
+
+  allocator = gst_va_allocator_new (display, formats);
+  if (!allocator) {
+    GST_ERROR_OBJECT (self, "Failed to create allocator");
+    return FALSE;
+  }
+
+  gst_allocation_params_init (&params);
+
+  priv->internal_pool = gst_va_pool_new_with_config (caps,
+      GST_VIDEO_INFO_SIZE (aligned_info), 0, 0,
+      VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER, GST_VA_FEATURE_DISABLED,
+      allocator, &params);
+  gst_object_unref (allocator);
+
+
+  if (!priv->internal_pool) {
+    GST_ERROR_OBJECT (self, "Failed to create va pool");
+    return FALSE;
+  }
 
-  priv->internal_pool = gst_video_buffer_pool_new ();
   config = gst_buffer_pool_get_config (priv->internal_pool);
   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
-  gst_buffer_pool_config_set_params (config,
-      caps, GST_VIDEO_INFO_SIZE (aligned_info), 0, 0);
-
+  gst_buffer_pool_config_set_params (config, caps,
+      GST_VIDEO_INFO_SIZE (aligned_info), 0, 0);
   gst_buffer_pool_set_config (priv->internal_pool, config);
   gst_buffer_pool_set_active (priv->internal_pool, TRUE);
 
   return TRUE;
 }
+#endif
 
 /* Prepare internal pool, which is used to allocate fallback buffer
  * when upstream buffer is not directly accessible by QSV */
 static gboolean
 gst_qsv_encoder_prepare_pool (GstQsvEncoder * self, GstCaps * caps,
-    GstVideoInfo * aligned_info, mfxU16 * io_pattern)
+    GstVideoInfo * aligned_info)
 {
   GstQsvEncoderPrivate *priv = self->priv;
   gboolean ret = FALSE;
@@ -910,21 +931,12 @@ gst_qsv_encoder_prepare_pool (GstQsvEncoder * self, GstCaps * caps,
 
   aligned_caps = gst_video_info_to_caps (aligned_info);
 
-  /* TODO: Add Linux video memory (VA/DMABuf) support */
 #ifdef G_OS_WIN32
-  priv->mem_type = GST_QSV_VIDEO_MEMORY | GST_QSV_ENCODER_IN_MEMORY;
-  *io_pattern = MFX_IOPATTERN_IN_VIDEO_MEMORY;
-
   ret = gst_qsv_encoder_prepare_d3d11_pool (self, aligned_caps, aligned_info);
+#else
+  ret = gst_qsv_encoder_prepare_va_pool (self, aligned_caps, aligned_info);
 #endif
 
-  if (!ret) {
-    priv->mem_type = GST_QSV_SYSTEM_MEMORY | GST_QSV_ENCODER_IN_MEMORY;
-    *io_pattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY;
-
-    ret = gst_qsv_encoder_prepare_system_pool (self,
-        aligned_caps, aligned_info);
-  }
   gst_caps_unref (aligned_caps);
 
   return ret;
@@ -976,8 +988,10 @@ gst_qsv_encoder_init_encode_session (GstQsvEncoder * self)
       GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_INTERLACE_MODE (info),
       frame_info->Width, frame_info->Height);
 
-  if (!gst_qsv_encoder_prepare_pool (self, caps, &priv->aligned_info,
-          &param.IOPattern)) {
+  /* Always video memory, even when upstream is non-hardware element */
+  priv->mem_type = GST_QSV_VIDEO_MEMORY | GST_QSV_ENCODER_IN_MEMORY;
+  param.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY;
+  if (!gst_qsv_encoder_prepare_pool (self, caps, &priv->aligned_info)) {
     GST_ERROR_OBJECT (self, "Failed to prepare pool");
     goto error;
   }
@@ -1428,18 +1442,20 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query)
   return TRUE;
 }
 #else
-/* TODO: Add support VA/DMABuf */
 static gboolean
 gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query)
 {
   GstQsvEncoder *self = GST_QSV_ENCODER (encoder);
   GstQsvEncoderPrivate *priv = self->priv;
   GstVideoInfo info;
+  GstAllocator *allocator = nullptr;
   GstBufferPool *pool;
   GstCaps *caps;
   guint size;
   GstStructure *config;
   GstVideoAlignment align;
+  GstAllocationParams params;
+  GArray *formats;
 
   gst_query_parse_allocation (query, &caps, nullptr);
   if (!caps) {
@@ -1452,7 +1468,31 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query)
     return FALSE;
   }
 
-  pool = gst_video_buffer_pool_new ();
+  gst_allocation_params_init (&params);
+
+  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+  g_array_append_val (formats, GST_VIDEO_INFO_FORMAT (&info));
+
+  allocator = gst_va_allocator_new (GST_VA_DISPLAY (priv->device), formats);
+  if (!allocator) {
+    GST_ERROR_OBJECT (self, "Failed to create allocator");
+    return FALSE;
+  }
+
+  /* Will not use derived image
+   * https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1110
+   */
+  pool = gst_va_pool_new_with_config (caps,
+      GST_VIDEO_INFO_SIZE (&info), priv->surface_pool->len, 0,
+      VA_SURFACE_ATTRIB_USAGE_HINT_ENCODER, GST_VA_FEATURE_DISABLED,
+      allocator, &params);
+
+  if (!pool) {
+    GST_ERROR_OBJECT (self, "Failed to create va pool");
+    gst_object_unref (allocator);
+
+    return FALSE;
+  }
 
   gst_video_alignment_reset (&align);
   align.padding_right = GST_VIDEO_INFO_WIDTH (&priv->aligned_info) -
@@ -1464,21 +1504,29 @@ gst_qsv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query)
   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
   gst_buffer_pool_config_add_option (config,
       GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
-  gst_video_info_align (&info, &align);
   gst_buffer_pool_config_set_video_alignment (config, &align);
 
-  size = GST_VIDEO_INFO_SIZE (&info);
   gst_buffer_pool_config_set_params (config,
-      caps, size, priv->surface_pool->len, 0);
+      caps, GST_VIDEO_INFO_SIZE (&info), priv->surface_pool->len, 0);
 
   if (!gst_buffer_pool_set_config (pool, config)) {
-    GST_WARNING_OBJECT (self, "Failed to set pool config");
+    GST_ERROR_OBJECT (self, "Failed to set pool config");
+    gst_clear_object (&allocator);
     gst_object_unref (pool);
     return FALSE;
   }
 
+  if (allocator)
+    gst_query_add_allocation_param (query, allocator, &params);
+
+  config = gst_buffer_pool_get_config (pool);
+  gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr);
+  gst_structure_free (config);
+
   gst_query_add_allocation_pool (query, pool, size, priv->surface_pool->len, 0);
   gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, nullptr);
+
+  gst_clear_object (&allocator);
   gst_object_unref (pool);
 
   return TRUE;

subprojects/gst-plugins-bad/sys/qsv/gstqsvh264enc.cpp

@@ -32,7 +32,7 @@
 #ifdef G_OS_WIN32
 #include <gst/d3d11/gstd3d11.h>
 #else
-#include <gst/va/gstvadisplay_drm.h>
+#include <gst/va/gstva.h>
 #endif
 
 GST_DEBUG_CATEGORY_EXTERN (gst_qsv_h264_enc_debug);
@@ -1829,6 +1829,13 @@ gst_qsv_h264_enc_register (GstPlugin * plugin, guint rank, guint impl_index,
   gst_caps_set_features_simple (d3d11_caps, caps_features);
   gst_caps_append (d3d11_caps, sink_caps);
   sink_caps = d3d11_caps;
+#else
+  GstCaps *va_caps = gst_caps_copy (sink_caps);
+  GstCapsFeatures *caps_features =
+      gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr);
+  gst_caps_set_features_simple (va_caps, caps_features);
+  gst_caps_append (va_caps, sink_caps);
+  sink_caps = va_caps;
 #endif
 
   std::string src_caps_str = "video/x-h264";

subprojects/gst-plugins-bad/sys/qsv/gstqsvh265enc.cpp

@@ -31,7 +31,7 @@
 #ifdef G_OS_WIN32
 #include <gst/d3d11/gstd3d11.h>
 #else
-#include <gst/va/gstvadisplay_drm.h>
+#include <gst/va/gstva.h>
 #endif
 
 GST_DEBUG_CATEGORY_EXTERN (gst_qsv_h265_enc_debug);
@@ -1410,6 +1410,13 @@ gst_qsv_h265_enc_register (GstPlugin * plugin, guint rank, guint impl_index,
   gst_caps_set_features_simple (d3d11_caps, caps_features);
   gst_caps_append (d3d11_caps, sink_caps);
   sink_caps = d3d11_caps;
+#else
+  GstCaps *va_caps = gst_caps_copy (sink_caps);
+  GstCapsFeatures *caps_features =
+      gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr);
+  gst_caps_set_features_simple (va_caps, caps_features);
+  gst_caps_append (va_caps, sink_caps);
+  sink_caps = va_caps;
 #endif
 
   std::string src_caps_str = "video/x-h265";

subprojects/gst-plugins-bad/sys/qsv/gstqsvvp9enc.cpp

@@ -30,7 +30,7 @@
 #ifdef G_OS_WIN32
 #include <gst/d3d11/gstd3d11.h>
 #else
-#include <gst/va/gstvadisplay_drm.h>
+#include <gst/va/gstva.h>
 #endif
 
 GST_DEBUG_CATEGORY_EXTERN (gst_qsv_vp9_enc_debug);
@@ -948,6 +948,13 @@ gst_qsv_vp9_enc_register (GstPlugin * plugin, guint rank, guint impl_index,
   gst_caps_set_features_simple (d3d11_caps, caps_features);
   gst_caps_append (d3d11_caps, sink_caps);
   sink_caps = d3d11_caps;
+#else
+  GstCaps *va_caps = gst_caps_copy (sink_caps);
+  GstCapsFeatures *caps_features =
+      gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_VA, nullptr);
+  gst_caps_set_features_simple (va_caps, caps_features);
+  gst_caps_append (va_caps, sink_caps);
+  sink_caps = va_caps;
 #endif
 
   std::string src_caps_str = "video/x-vp9";