msdk: Add help functions to get mfxFrameSurface1 from GstBuffer and wrap it as GstMsdkSurface

Note that the memory abstraction for system memory is for windows path.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2498>

subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.c

@@ -0,0 +1,151 @@
+/*
+ * GStreamer Intel MSDK plugin
+ * Copyright (c) 2022 Intel Corporation. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "gstmsdkallocator.h"
+
+static gboolean
+map_data (GstBuffer * buffer, mfxFrameSurface1 * mfx_surface, GstVideoInfo info)
+{
+  guint stride;
+  GstVideoFrame frame;
+
+  if (!gst_video_frame_map (&frame, &info, buffer, GST_MAP_READWRITE))
+    return FALSE;
+
+  stride = GST_VIDEO_FRAME_PLANE_STRIDE (&frame, 0);
+
+  switch (GST_VIDEO_INFO_FORMAT (&info)) {
+    case GST_VIDEO_FORMAT_NV12:
+    case GST_VIDEO_FORMAT_P010_10LE:
+    case GST_VIDEO_FORMAT_P012_LE:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.UV = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1);
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_YV12:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 2);
+      mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1);
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_I420:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 1);
+      mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 2);
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_YUY2:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = mfx_surface->Data.Y + 1;
+      mfx_surface->Data.V = mfx_surface->Data.Y + 3;
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_UYVY:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = mfx_surface->Data.Y;
+      mfx_surface->Data.V = mfx_surface->Data.U + 2;
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_VUYA:
+      mfx_surface->Data.V = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = mfx_surface->Data.V + 1;
+      mfx_surface->Data.Y = mfx_surface->Data.V + 2;
+      mfx_surface->Data.A = mfx_surface->Data.V + 3;
+      mfx_surface->Data.PitchHigh = (mfxU16) (stride / (1 << 16));
+      mfx_surface->Data.PitchLow = (mfxU16) (stride % (1 << 16));
+      break;
+    case GST_VIDEO_FORMAT_BGRA:
+    case GST_VIDEO_FORMAT_BGRx:
+      mfx_surface->Data.B = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.G = mfx_surface->Data.B + 1;
+      mfx_surface->Data.R = mfx_surface->Data.B + 2;
+      mfx_surface->Data.A = mfx_surface->Data.B + 3;
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_Y210:
+    case GST_VIDEO_FORMAT_Y212_LE:
+      mfx_surface->Data.Y = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.U = mfx_surface->Data.Y + 2;
+      mfx_surface->Data.V = mfx_surface->Data.Y + 6;
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    case GST_VIDEO_FORMAT_Y410:
+      mfx_surface->Data.Y410 =
+          (mfxY410 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.Pitch = stride;
+      break;
+    case GST_VIDEO_FORMAT_Y412_LE:
+      mfx_surface->Data.U = (mfxU8 *) GST_VIDEO_FRAME_PLANE_DATA (&frame, 0);
+      mfx_surface->Data.Y = mfx_surface->Data.Y + 2;
+      mfx_surface->Data.V = mfx_surface->Data.Y + 4;
+      mfx_surface->Data.A = mfx_surface->Data.Y + 6;
+      mfx_surface->Data.Pitch = (mfxU16) stride;
+      break;
+    default:
+      g_assert_not_reached ();
+      break;
+  }
+
+  gst_video_frame_unmap (&frame);
+  return TRUE;
+}
+
+GstMsdkSurface *
+gst_msdk_import_sys_mem_to_msdk_surface (GstBuffer * buf, GstVideoInfo info)
+{
+  GstMsdkSurface *msdk_surface = NULL;
+  GstMapInfo map_info;
+  mfxFrameInfo frame_info = { 0, };
+  mfxFrameSurface1 *mfx_surface = NULL;
+
+  if (!gst_buffer_map (buf, &map_info, GST_MAP_READ)) {
+    GST_ERROR ("Failed to map buffer");
+    return msdk_surface;
+  }
+
+  mfx_surface = g_slice_new0 (mfxFrameSurface1);
+  mfx_surface->Data.MemId = (mfxMemId) map_info.data;
+
+  if (!map_data (buf, mfx_surface, info)) {
+    g_slice_free (mfxFrameSurface1, mfx_surface);
+    return msdk_surface;
+  }
+
+  gst_buffer_unmap (buf, &map_info);
+
+  gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, &info);
+  mfx_surface->Info = frame_info;
+
+  msdk_surface = g_slice_new0 (GstMsdkSurface);
+  msdk_surface->surface = mfx_surface;
+
+  return msdk_surface;
+}

subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator.h

@@ -39,6 +39,7 @@
 G_BEGIN_DECLS
 
 typedef struct _GstMsdkMemoryID GstMsdkMemoryID;
+typedef struct _GstMsdkSurface GstMsdkSurface;
 
 struct _GstMsdkMemoryID {
   mfxU32 fourcc;
@@ -56,6 +57,15 @@ struct _GstMsdkMemoryID {
 #endif
 };
 
+struct _GstMsdkSurface
+{
+  mfxFrameSurface1 *surface;
+  GstBuffer *buf;
+};
+
+GstMsdkSurface *
+gst_msdk_import_sys_mem_to_msdk_surface (GstBuffer * buf, GstVideoInfo info);
+
 mfxStatus gst_msdk_frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req, mfxFrameAllocResponse *resp);
 mfxStatus gst_msdk_frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp);
 mfxStatus gst_msdk_frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);

subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.c

@@ -37,6 +37,21 @@
 #include "gstmsdkallocator_libva.h"
 #include "msdk_libva.h"
 
+#include <gst/va/gstvaallocator.h>
+
+#define GST_MSDK_FRAME_SURFACE gst_msdk_frame_surface_quark_get ()
+static GQuark
+gst_msdk_frame_surface_quark_get (void)
+{
+  static gsize g_quark;
+
+  if (g_once_init_enter (&g_quark)) {
+    gsize quark = (gsize) g_quark_from_static_string ("GstMsdkFrameSurface");
+    g_once_init_leave (&g_quark, quark);
+  }
+  return g_quark;
+}
+
 mfxStatus
 gst_msdk_frame_alloc (mfxHDL pthis, mfxFrameAllocRequest * req,
     mfxFrameAllocResponse * resp)
@@ -704,6 +719,105 @@ error_create_surface:
   }
 }
 
+static VASurfaceID
+_get_va_surface (GstBuffer * buf, GstVideoInfo * info,
+    GstMsdkContext * msdk_context)
+{
+  VASurfaceID va_surface = VA_INVALID_ID;
+
+  if (!info) {
+    va_surface = gst_va_buffer_get_surface (buf);
+  } else {
+    /* Update offset/stride/size if there is VideoMeta attached to
+     * the dma buffer, which is then used to get vasurface */
+    GstMemory *mem;
+    gint i, fd;
+    GstVideoMeta *vmeta;
+
+    vmeta = gst_buffer_get_video_meta (buf);
+    if (vmeta) {
+      if (GST_VIDEO_INFO_FORMAT (info) != vmeta->format ||
+          GST_VIDEO_INFO_WIDTH (info) != vmeta->width ||
+          GST_VIDEO_INFO_HEIGHT (info) != vmeta->height ||
+          GST_VIDEO_INFO_N_PLANES (info) != vmeta->n_planes) {
+        GST_ERROR ("VideoMeta attached to buffer is not matching"
+            "the negotiated width/height/format");
+        return va_surface;
+      }
+      for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); ++i) {
+        GST_VIDEO_INFO_PLANE_OFFSET (info, i) = vmeta->offset[i];
+        GST_VIDEO_INFO_PLANE_STRIDE (info, i) = vmeta->stride[i];
+      }
+      GST_VIDEO_INFO_SIZE (info) = gst_buffer_get_size (buf);
+    }
+
+    mem = gst_buffer_peek_memory (buf, 0);
+    fd = gst_dmabuf_memory_get_fd (mem);
+    if (fd < 0)
+      return va_surface;
+    /* export dmabuf to vasurface */
+    if (!gst_msdk_export_dmabuf_to_vasurface (msdk_context, info, fd,
+            &va_surface))
+      return VA_INVALID_ID;
+  }
+
+  return va_surface;
+}
+
+GstMsdkSurface *
+gst_msdk_import_to_msdk_surface (GstBuffer * buf, GstMsdkContext * msdk_context,
+    GstVideoInfo * vinfo)
+{
+  VASurfaceID va_surface = VA_INVALID_ID;
+  GstMemory *mem = NULL;
+  mfxFrameInfo frame_info = { 0, };
+  GstMsdkSurface *msdk_surface = NULL;
+  mfxFrameSurface1 *mfx_surface = NULL;
+  GstMsdkMemoryID *msdk_mid = NULL;
+
+  mem = gst_buffer_peek_memory (buf, 0);
+  msdk_surface = g_slice_new0 (GstMsdkSurface);
+
+  /* If buffer has qdata pointing to mfxFrameSurface1, directly extract it */
+  if ((mfx_surface = gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem),
+              GST_MSDK_FRAME_SURFACE))) {
+    msdk_surface->surface = mfx_surface;
+    return msdk_surface;
+  }
+
+  if (gst_msdk_is_va_mem (mem)) {
+    va_surface = _get_va_surface (buf, NULL, NULL);
+  } else if (gst_is_dmabuf_memory (mem)) {
+    /* For dma memory, videoinfo is used with dma fd to create va surface. */
+    GstVideoInfo info = *vinfo;
+    va_surface = _get_va_surface (buf, &info, msdk_context);
+  }
+
+  if (va_surface == VA_INVALID_ID) {
+    g_slice_free (GstMsdkSurface, msdk_surface);
+    return NULL;
+  }
+
+  mfx_surface = g_slice_new0 (mfxFrameSurface1);
+  msdk_mid = g_slice_new0 (GstMsdkMemoryID);
+
+  msdk_mid->surface = g_slice_new0 (VASurfaceID);
+  *msdk_mid->surface = va_surface;
+
+  mfx_surface->Data.MemId = (mfxMemId) msdk_mid;
+
+  gst_msdk_set_mfx_frame_info_from_video_info (&frame_info, vinfo);
+  mfx_surface->Info = frame_info;
+
+  /* Set mfxFrameSurface1 as qdata in buffer */
+  gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem),
+      GST_MSDK_FRAME_SURFACE, mfx_surface, NULL);
+
+  msdk_surface->surface = mfx_surface;
+
+  return msdk_surface;
+}
+
 /**
  * gst_msdk_replace_mfx_memid:
  * This method replace the internal VA Suface in mfxSurface with a new one

subprojects/gst-plugins-bad/sys/msdk/gstmsdkallocator_libva.h

@@ -45,6 +45,10 @@ gboolean
 gst_msdk_export_dmabuf_to_vasurface (GstMsdkContext *context,
     GstVideoInfo *vinfo, gint fd, VASurfaceID *surface_id);
 
+GstMsdkSurface *
+gst_msdk_import_to_msdk_surface (GstBuffer * buf, GstMsdkContext * msdk_context,
+    GstVideoInfo * vinfo);
+
 gboolean
 gst_msdk_replace_mfx_memid (GstMsdkContext *context,
     mfxFrameSurface1 *mfx_surface, VASurfaceID surface_id);

subprojects/gst-plugins-bad/sys/msdk/meson.build

@@ -1,5 +1,6 @@
 msdk_sources = [
   'gstmsdk.c',
+  'gstmsdkallocator.c',
   'gstmsdkbufferpool.c',
   'gstmsdkcontext.c',
   'gstmsdkcontextutil.c',

subprojects/gst-plugins-bad/sys/msdk/msdk.h

@@ -99,6 +99,7 @@ void GstMFXUnload (mfxLoader loader);
 #endif
 
 typedef struct _MsdkSession MsdkSession;
+typedef struct _GstMsdkSurface GstMsdkSurface;
 
 struct _MsdkSession
 {