MSDK: Introduce d3d11 device to MSDK context

Currently MSDK context does not support d3d11va. Now introduce d3d11va
device to MSDK context, making it able to create msdk session with d3d11
device and to easily share with upstream and donwstream.

Add environment variable to enable user to choose GPU device in multi-GPU
environment. This variable is only valid when there's no context
returned by upstream or downstream. Otherwise it will use the device
that created by upstream or downstream.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3231>

subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c

@@ -38,6 +38,8 @@
 #include <va/va_drm.h>
 #include <gudev/gudev.h>
 #include <gst/va/gstvadisplay_drm.h>
+#else
+#include <gst/d3d11/gstd3d11.h>
 #endif
 
 GST_DEBUG_CATEGORY_STATIC (gst_debug_msdkcontext);
@@ -56,6 +58,8 @@ struct _GstMsdkContextPrivate
   GstMsdkContext *parent_context;
 #ifndef _WIN32
   GstVaDisplay *display;
+#else
+  GstD3D11Device *device;
 #endif
 };
 
@@ -201,6 +205,101 @@ failed:
 
   return FALSE;
 }
+#else
+static GstD3D11Device *
+get_device_by_index (IDXGIFactory1 * factory, guint idx)
+{
+  HRESULT hr;
+  IDXGIAdapter1 *adapter;
+  ID3D11Device *device_handle;
+  ID3D10Multithread *multi_thread;
+  DXGI_ADAPTER_DESC desc;
+  GstD3D11Device *device = NULL;
+  gint64 luid;
+
+  hr = IDXGIFactory1_EnumAdapters1 (factory, idx, &adapter);
+  if (FAILED (hr)) {
+    return NULL;
+  }
+
+  hr = IDXGIAdapter1_GetDesc (adapter, &desc);
+  if (FAILED (hr)) {
+    IDXGIAdapter1_Release (adapter);
+    return NULL;
+  }
+
+  if (desc.VendorId != 0x8086) {
+    IDXGIAdapter1_Release (adapter);
+    return NULL;
+  }
+
+  luid = gst_d3d11_luid_to_int64 (&desc.AdapterLuid);
+  device = gst_d3d11_device_new_for_adapter_luid (luid,
+      D3D11_CREATE_DEVICE_BGRA_SUPPORT);
+  IDXGIAdapter1_Release (adapter);
+
+  device_handle = gst_d3d11_device_get_device_handle (device);
+  hr = ID3D11Device_QueryInterface (device_handle,
+      &IID_ID3D10Multithread, (void **) &multi_thread);
+  if (FAILED (hr)) {
+    gst_object_unref (device);
+    return NULL;
+  }
+
+  hr = ID3D10Multithread_SetMultithreadProtected (multi_thread, TRUE);
+  ID3D10Multithread_Release (multi_thread);
+
+  return device;
+}
+
+static gboolean
+gst_msdk_context_use_d3d11 (GstMsdkContext * context)
+{
+  HRESULT hr;
+  IDXGIFactory1 *factory = NULL;
+  GstD3D11Device *device = NULL;
+  ID3D11Device *device_handle;
+  GstMsdkContextPrivate *priv = context->priv;
+  mfxStatus status;
+  guint idx = 0;
+  gint user_idx = -1;
+  const gchar *user_choice = g_getenv ("GST_MSDK_DEVICE");
+
+  hr = CreateDXGIFactory1 (&IID_IDXGIFactory1, (void **) &factory);
+  if (FAILED (hr)) {
+    GST_ERROR ("Couldn't create DXGI factory");
+    return FALSE;
+  }
+
+  if (user_choice) {
+    user_idx = atoi (user_choice);
+    if (!(device = get_device_by_index (factory, user_idx)))
+      GST_WARNING
+          ("Failed to get device by user index, try to pick the first available device");
+  }
+
+  /* Pick the first available device */
+  while (!device) {
+    device = get_device_by_index (factory, idx++);
+  }
+
+  IDXGIFactory1_Release (factory);
+  device_handle = gst_d3d11_device_get_device_handle (device);
+
+  status =
+      MFXVideoCORE_SetHandle (priv->session.session, MFX_HANDLE_D3D11_DEVICE,
+      gst_d3d11_device_get_device_handle (device));
+  if (status != MFX_ERR_NONE) {
+    GST_ERROR ("Setting D3D11VA handle failed (%s)",
+        msdk_status_to_string (status));
+    gst_object_unref (device);
+    return FALSE;
+  }
+
+  priv->device = device;
+
+  return TRUE;
+}
 #endif
 
 static gboolean
@@ -210,12 +309,18 @@ gst_msdk_context_open (GstMsdkContext * context, gboolean hardware,
   mfxU16 codename;
   GstMsdkContextPrivate *priv = context->priv;
   MsdkSession msdk_session;
+  mfxIMPL impl;
 
   priv->job_type = job_type;
   priv->hardware = hardware;
 
-  msdk_session =
-      msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE);
+  impl = hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE;
+
+#ifdef _WIN32
+  impl |= MFX_IMPL_VIA_D3D11;
+#endif
+
+  msdk_session = msdk_open_session (impl);
   priv->session = msdk_session;
   if (!priv->session.session)
     goto failed;
@@ -225,6 +330,11 @@ gst_msdk_context_open (GstMsdkContext * context, gboolean hardware,
     if (!gst_msdk_context_use_vaapi (context))
       goto failed;
   }
+#else
+  if (hardware) {
+    if (!gst_msdk_context_use_d3d11 (context))
+      goto failed;
+  }
 #endif
 
   codename = msdk_get_platform_codename (priv->session.session);
@@ -281,6 +391,9 @@ gst_msdk_context_finalize (GObject * obj)
 #ifndef _WIN32
   if (priv->display)
     gst_object_unref (priv->display);
+#else
+  if (priv->device)
+    gst_object_unref (priv->device);
 #endif
 
 done:
@@ -336,6 +449,8 @@ gst_msdk_context_new_with_parent (GstMsdkContext * parent)
 
   if (MFX_IMPL_VIA_VAAPI == (0x0f00 & (impl)))
     handle_type = MFX_HANDLE_VA_DISPLAY;
+  else if (MFX_IMPL_VIA_D3D11 == (0x0f00 & (impl)))
+    handle_type = MFX_HANDLE_D3D11_DEVICE;
 
   if (handle_type) {
     status =
@@ -396,19 +511,21 @@ gst_msdk_context_new_with_parent (GstMsdkContext * parent)
       g_list_prepend (parent_priv->child_session_list, priv->session.session);
 #ifndef _WIN32
   priv->display = parent_priv->display;
+#else
+  priv->device = parent_priv->device;
 #endif
   priv->parent_context = gst_object_ref (parent);
 
   return obj;
 }
 
+#ifndef _WIN32
 GstMsdkContext *
 gst_msdk_context_new_with_va_display (GstObject * display_obj,
     gboolean hardware, GstMsdkContextJobType job_type)
 {
   GstMsdkContext *obj = NULL;
 
-#ifndef _WIN32
   GstMsdkContextPrivate *priv;
   mfxU16 codename;
   mfxStatus status;
@@ -451,10 +568,71 @@ gst_msdk_context_new_with_va_display (GstObject * display_obj,
   else
     GST_WARNING ("Unknown MFX platform");
 
-#endif
-
   return obj;
 }
+#else
+GstMsdkContext *
+gst_msdk_context_new_with_d3d11_device (GstD3D11Device * device,
+    gboolean hardware, GstMsdkContextJobType job_type)
+{
+  GstMsdkContext *obj = NULL;
+  GstMsdkContextPrivate *priv;
+  mfxU16 codename;
+  mfxStatus status;
+  ID3D10Multithread *multi_thread;
+  ID3D11Device *device_handle;
+  HRESULT hr;
+
+  obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL);
+
+  priv = obj->priv;
+  priv->device = gst_object_ref (device);
+
+  priv->job_type = job_type;
+  priv->hardware = hardware;
+  priv->session =
+      msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE);
+  if (!priv->session.session) {
+    goto failed;
+  }
+
+  device_handle = gst_d3d11_device_get_device_handle (device);
+  hr = ID3D11Device_QueryInterface (device_handle,
+      &IID_ID3D10Multithread, (void **) &multi_thread);
+  if (FAILED (hr)) {
+    GST_ERROR ("ID3D10Multithread interface is unavailable");
+    goto failed;
+  }
+
+  hr = ID3D10Multithread_SetMultithreadProtected (multi_thread, TRUE);
+  ID3D10Multithread_Release (multi_thread);
+
+  if (hardware) {
+    status =
+        MFXVideoCORE_SetHandle (priv->session.session, MFX_HANDLE_D3D11_DEVICE,
+        device_handle);
+    if (status != MFX_ERR_NONE) {
+      GST_ERROR ("Setting D3D11VA handle failed (%s)",
+          msdk_status_to_string (status));
+      goto failed;
+    }
+  }
+
+  codename = msdk_get_platform_codename (priv->session.session);
+
+  if (codename != MFX_PLATFORM_UNKNOWN)
+    GST_INFO ("Detected MFX platform with device code %d", codename);
+  else
+    GST_WARNING ("Unknown MFX platform");
+
+  return obj;
+
+failed:
+  gst_object_unref (obj);
+  gst_object_unref (device);
+  return NULL;
+}
+#endif
 
 mfxSession
 gst_msdk_context_get_session (GstMsdkContext * context)
@@ -472,15 +650,23 @@ gst_msdk_context_get_handle (GstMsdkContext * context)
 #endif
 }
 
-GstObject *
-gst_msdk_context_get_display (GstMsdkContext * context)
-{
 #ifndef _WIN32
+GstObject *
+gst_msdk_context_get_va_display (GstMsdkContext * context)
+{
   if (context->priv->display)
     return gst_object_ref (GST_OBJECT_CAST (context->priv->display));
-#endif
   return NULL;
 }
+#else
+GstD3D11Device *
+gst_msdk_context_get_d3d11_device (GstMsdkContext * context)
+{
+  if (context->priv->device)
+    return gst_object_ref (context->priv->device);
+  return NULL;
+}
+#endif
 
 static gint
 _find_response (gconstpointer resp, gconstpointer comp_resp)

subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.h

@@ -37,6 +37,8 @@
 #ifndef _WIN32
 #include <va/va.h>
 #include <va/va_drmcommon.h>
+#else
+#include <gst/d3d11/gstd3d11_fwd.h>
 #endif
 
 G_BEGIN_DECLS
@@ -87,12 +89,21 @@ GType gst_msdk_context_get_type (void);
 
 GstMsdkContext * gst_msdk_context_new (gboolean hardware, GstMsdkContextJobType job_type);
 GstMsdkContext * gst_msdk_context_new_with_parent (GstMsdkContext * parent);
+#ifndef _WIN32
 GstMsdkContext * gst_msdk_context_new_with_va_display (GstObject * display_obj,
     gboolean hardware, GstMsdkContextJobType job_type);
+#else
+GstMsdkContext * gst_msdk_context_new_with_d3d11_device (GstD3D11Device * device,
+    gboolean hardware, GstMsdkContextJobType job_type);
+#endif
 mfxSession gst_msdk_context_get_session (GstMsdkContext * context);
 
 gpointer gst_msdk_context_get_handle (GstMsdkContext * context);
-GstObject * gst_msdk_context_get_display (GstMsdkContext * context);
+#ifndef _WIN32
+GstObject * gst_msdk_context_get_va_display (GstMsdkContext * context);
+#else
+GstD3D11Device * gst_msdk_context_get_d3d11_device (GstMsdkContext * context);
+#endif
 
 /* GstMsdkContext contains mfxFrameAllocResponses,
  * if app calls MFXVideoCORE_SetFrameAllocator.

subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.c

@@ -34,6 +34,8 @@
 #ifndef _WIN32
 #include <gst/va/gstvadisplay.h>
 #include <gst/va/gstvautils.h>
+#else
+#include <gst/d3d11/gstd3d11device.h>
 #endif
 
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_CONTEXT);
@@ -236,7 +238,7 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware,
 
 #ifndef _WIN32
   /* 2) Query the neighbour the VA display. If already a valid VA display,
-     using it by gst_msdk_context_from_external_display() in set_context(). */
+     using it by gst_msdk_context_from_external_va_display() in set_context(). */
   gst_va_context_query (element, GST_VA_DISPLAY_HANDLE_CONTEXT_TYPE_STR);
   msdk_context = g_atomic_pointer_get (context_ptr);
   if (msdk_context) {
@@ -245,9 +247,23 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware,
     ret = TRUE;
     goto done;
   }
+#else
+  /* 2) Query the neighbour the D3D11 device. If already a valid D3D11 device,
+     using it by gst_msdk_context_from_external_d3d11_device() in set_context(). */
+  _context_query (element, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE);
+  msdk_context = g_atomic_pointer_get (context_ptr);
+  if (msdk_context) {
+    gst_object_ref (msdk_context);
+    propagate_display = FALSE;
+    ret = TRUE;
+    goto done;
+  }
 #endif
 
-  /* 3) Create a MSDK context from scratch. */
+  /* 3) Create a MSDK context from scratch. Currently we use environment variable
+     to enable user to choose GPU device in multi-GPU environment. This variable
+     is only valid when there's no context returned by upstream or downstream.
+     Otherwise it will use the device that created by upstream or downstream. */
   msdk_context = gst_msdk_context_new (hardware, job);
   if (!msdk_context) {
     GST_ERROR_OBJECT (element, "Context creation failed");
@@ -264,7 +280,7 @@ done:
   if (propagate_display) {
 #ifndef _WIN32
     GstVaDisplay *display =
-        (GstVaDisplay *) gst_msdk_context_get_display (msdk_context);
+        (GstVaDisplay *) gst_msdk_context_get_va_display (msdk_context);
     gst_va_element_propagate_display_context (element, display);
     gst_clear_object (&display);
 #endif
@@ -276,11 +292,12 @@ done:
   return ret;
 }
 
-gboolean
-gst_msdk_context_from_external_display (GstContext * context, gboolean hardware,
-    GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context)
-{
 #ifndef _WIN32
+gboolean
+gst_msdk_context_from_external_va_display (GstContext * context,
+    gboolean hardware, GstMsdkContextJobType job_type,
+    GstMsdkContext ** msdk_context)
+{
   GstObject *va_display = NULL;
   const gchar *type;
   const GstStructure *s;
@@ -309,10 +326,50 @@ gst_msdk_context_from_external_display (GstContext * context, gboolean hardware,
   if (ctx)
     return TRUE;
 
-#endif
+  return FALSE;
+}
+#else
+gboolean
+gst_msdk_context_from_external_d3d11_device (GstContext * context,
+    gboolean hardware, GstMsdkContextJobType job_type,
+    GstMsdkContext ** msdk_context)
+{
+  GstD3D11Device *d3d11_device = NULL;
+  const gchar *type;
+  const GstStructure *s;
+  GstMsdkContext *ctx = NULL;
+  guint vendor_id = 0;
+
+  _init_context_debug ();
+
+  type = gst_context_get_context_type (context);
+  if (g_strcmp0 (type, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE))
+    return FALSE;
+
+  s = gst_context_get_structure (context);
+  if (gst_structure_get (s, "device", GST_TYPE_D3D11_DEVICE, &d3d11_device,
+          NULL)) {
+    g_object_get (d3d11_device, "vendor-id", &vendor_id, NULL);
+    if (vendor_id != 0x8086) {
+      GST_ERROR ("Not an Intel device");
+      gst_clear_object (&d3d11_device);
+      return FALSE;
+    }
+    ctx =
+        gst_msdk_context_new_with_d3d11_device (d3d11_device, hardware,
+        job_type);
+    if (ctx)
+      *msdk_context = ctx;
+
+    gst_clear_object (&d3d11_device);
+  }
+
+  if (ctx)
+    return TRUE;
 
   return FALSE;
 }
+#endif
 
 gboolean
 gst_msdk_handle_context_query (GstElement * element, GstQuery * query,
@@ -346,7 +403,7 @@ gst_msdk_handle_context_query (GstElement * element, GstQuery * query,
 #ifndef _WIN32
   if (g_strcmp0 (context_type, GST_VA_DISPLAY_HANDLE_CONTEXT_TYPE_STR) == 0) {
     GstStructure *s;
-    GstObject *display = gst_msdk_context_get_display (msdk_context);
+    GstObject *display = gst_msdk_context_get_va_display (msdk_context);
 
     if (display) {
       GST_CAT_LOG (GST_CAT_CONTEXT,
@@ -360,6 +417,22 @@ gst_msdk_handle_context_query (GstElement * element, GstQuery * query,
       ret = TRUE;
     }
   } else
+#else
+  if (g_strcmp0 (context_type, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE) == 0) {
+    GstStructure *s;
+    GstD3D11Device *device = gst_msdk_context_get_d3d11_device (msdk_context);
+
+    if (device) {
+      GST_CAT_LOG (GST_CAT_CONTEXT,
+          "setting GstD3D11Device (%" GST_PTR_FORMAT ") on context (%"
+          GST_PTR_FORMAT ")", device, ctxt);
+
+      s = gst_context_writable_structure (ctxt);
+      gst_structure_set (s, "device", GST_TYPE_D3D11_DEVICE, device, NULL);
+      gst_object_unref (device);
+      ret = TRUE;
+    }
+  } else
 #endif
   if (g_strcmp0 (context_type, GST_MSDK_CONTEXT_TYPE_NAME) == 0) {
     GstStructure *s;

subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.h

@@ -52,9 +52,15 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware, GstMsdkCon
 gboolean
 gst_msdk_context_get_context (GstContext * context, GstMsdkContext ** msdk_context);
 
+#ifndef _WIN32
 gboolean
-gst_msdk_context_from_external_display (GstContext * context, gboolean hardware,
+gst_msdk_context_from_external_va_display (GstContext * context, gboolean hardware,
     GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context);
+#else
+gboolean
+gst_msdk_context_from_external_d3d11_device (GstContext * context, gboolean hardware,
+    GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context);
+#endif
 
 gboolean
 gst_msdk_handle_context_query (GstElement * element, GstQuery * query, GstMsdkContext * msdk_context);

subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c

@@ -323,13 +323,24 @@ gst_msdkdec_set_context (GstElement * element, GstContext * context)
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
-  } else if (gst_msdk_context_from_external_display (context,
+  } else
+#ifndef _WIN32
+    if (gst_msdk_context_from_external_va_display (context,
           thiz->hardware, 0 /* GST_MSDK_JOB_DECODER will be set later */ ,
           &msdk_context)) {
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
   }
+#else
+    if (gst_msdk_context_from_external_d3d11_device (context,
+          thiz->hardware, 0 /* GST_MSDK_JOB_DECODER will be set later */ ,
+          &msdk_context)) {
+    gst_object_replace ((GstObject **) & thiz->context,
+        (GstObject *) msdk_context);
+    gst_object_unref (msdk_context);
+  }
+#endif
 
   GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
 }

subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c

@@ -146,13 +146,24 @@ gst_msdkenc_set_context (GstElement * element, GstContext * context)
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
-  } else if (gst_msdk_context_from_external_display (context,
+  } else
+#ifndef _WIN32
+    if (gst_msdk_context_from_external_va_display (context,
           thiz->hardware, 0 /* GST_MSDK_JOB_ENCODER will be set later */ ,
           &msdk_context)) {
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
   }
+#else
+    if (gst_msdk_context_from_external_d3d11_device (context,
+          thiz->hardware, 0 /* GST_MSDK_JOB_ENCODER will be set later */ ,
+          &msdk_context)) {
+    gst_object_replace ((GstObject **) & thiz->context,
+        (GstObject *) msdk_context);
+    gst_object_unref (msdk_context);
+  }
+#endif
 
   GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
 }
@@ -1299,7 +1310,7 @@ gst_msdk_create_va_pool (GstMsdkEnc * thiz, GstCaps * caps, guint num_buffers)
   GstVaDisplay *display = NULL;
   GstVideoInfo info = thiz->input_state->info;
 
-  display = (GstVaDisplay *) gst_msdk_context_get_display (thiz->context);
+  display = (GstVaDisplay *) gst_msdk_context_get_va_display (thiz->context);
 
   if (thiz->use_dmabuf) {
     allocator = gst_va_dmabuf_allocator_new (display);

subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c

@@ -420,7 +420,7 @@ gst_msdk_create_va_pool (GstVideoInfo * info, GstMsdkContext * msdk_context,
   GstVaDisplay *display = NULL;
   GstCaps *aligned_caps = NULL;
 
-  display = (GstVaDisplay *) gst_msdk_context_get_display (msdk_context);
+  display = (GstVaDisplay *) gst_msdk_context_get_va_display (msdk_context);
 
   if (use_dmabuf)
     allocator = gst_va_dmabuf_allocator_new (display);
@@ -1661,13 +1661,24 @@ gst_msdkvpp_set_context (GstElement * element, GstContext * context)
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
-  } else if (gst_msdk_context_from_external_display (context,
+  } else
+#ifndef _WIN32
+    if (gst_msdk_context_from_external_va_display (context,
           thiz->hardware, 0 /* GST_MSDK_JOB_VPP will be set later */ ,
           &msdk_context)) {
     gst_object_replace ((GstObject **) & thiz->context,
         (GstObject *) msdk_context);
     gst_object_unref (msdk_context);
   }
+#else
+    if (gst_msdk_context_from_external_d3d11_device (context,
+          thiz->hardware, 0 /* GST_MSDK_JOB_VPP will be set later */ ,
+          &msdk_context)) {
+    gst_object_replace ((GstObject **) & thiz->context,
+        (GstObject *) msdk_context);
+    gst_object_unref (msdk_context);
+  }
+#endif
 
   GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
 }