mfvideoenc: Add support for Direct3D11 texture

Initial support for d3d11 texture so that encoder can copy
upstream d3d11 texture into encoder's own texture pool without
downloading memory.

This implementation requires MFTEnum2() API for creating
MFT (Media Foundation Transform) object for specific GPU but
the API is Windows 10 desktop only. So UWP is not target
of this change.
See also https://docs.microsoft.com/en-us/windows/win32/api/mfapi/nf-mfapi-mftenum2

Note that, for MF plugin to be able to support old OS versions
without breakage, this commit will load MFTEnum2() symbol
by using g_module_open()

Summary of required system environment:
- Needs Windows 10 (probably at least RS 1 update)
- GPU should support ExtendedNV12SharedTextureSupported feature
- Desktop application only (UWP is not supported yet)

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1903>

sys/mediafoundation/gstmfh264enc.cpp

@@ -35,12 +35,18 @@
 #include "config.h"
 #endif
 
+#include "gstmfconfig.h"
+
 #include <gst/gst.h>
 #include <gst/pbutils/pbutils.h>
 #include "gstmfvideoenc.h"
 #include "gstmfh264enc.h"
 #include <wrl.h>
 
+#if GST_MF_HAVE_D3D11
+#include <gst/d3d11/gstd3d11.h>
+#endif
+
 using namespace Microsoft::WRL;
 
 GST_DEBUG_CATEGORY (gst_mf_h264_enc_debug);
@@ -154,6 +160,8 @@ enum
   PROP_QP_P,
   PROP_QP_B,
   PROP_REF,
+  PROP_D3D11_AWARE,
+  PROP_ADAPTER,
 };
 
 #define DEFAULT_BITRATE (2 * 1024)
@@ -456,6 +464,21 @@ gst_mf_h264_enc_class_init (GstMFH264EncClass * klass, gpointer data)
             G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
   }
 
+  g_object_class_install_property (gobject_class, PROP_D3D11_AWARE,
+      g_param_spec_boolean ("d3d11-aware", "D3D11 Aware",
+          "Whether device can support Direct3D11 interop",
+          device_caps->d3d11_aware,
+          (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+
+  if (device_caps->d3d11_aware) {
+    g_object_class_install_property (gobject_class, PROP_ADAPTER,
+        g_param_spec_uint ("adapter", "Adapter",
+            "DXGI Adapter index for creating device",
+            0, G_MAXUINT32, device_caps->adapter,
+            (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
+            G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+  }
+
   long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name);
   classification = g_strdup_printf ("Codec/Encoder/Video%s",
       (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ?
@@ -519,6 +542,7 @@ gst_mf_h264_enc_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec)
 {
   GstMFH264Enc *self = (GstMFH264Enc *) (object);
+  GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object);
 
   switch (prop_id) {
     case PROP_BITRATE:
@@ -587,6 +611,12 @@ gst_mf_h264_enc_get_property (GObject * object, guint prop_id,
     case PROP_REF:
       g_value_set_uint (value, self->max_num_ref);
       break;
+    case PROP_D3D11_AWARE:
+      g_value_set_boolean (value, klass->device_caps.d3d11_aware);
+      break;
+    case PROP_ADAPTER:
+      g_value_set_uint (value, klass->device_caps.adapter);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -984,7 +1014,8 @@ gst_mf_h264_enc_set_src_caps (GstMFVideoEnc * mfenc,
 }
 
 void
-gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank)
+gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank,
+    GList * d3d11_device)
 {
   GTypeInfo type_info = {
     sizeof (GstMFH264EncClass),
@@ -1001,5 +1032,5 @@ gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank)
 
   GST_DEBUG_CATEGORY_INIT (gst_mf_h264_enc_debug, "mfh264enc", 0, "mfh264enc");
 
-  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info);
+  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device);
 }

sys/mediafoundation/gstmfh264enc.h

@@ -26,7 +26,8 @@
 G_BEGIN_DECLS
 
 void gst_mf_h264_enc_plugin_init (GstPlugin * plugin,
-                                  guint rank);
+                                  guint rank,
+                                  GList * d3d11_device);
 
 G_END_DECLS
 

sys/mediafoundation/gstmfh265enc.cpp

@@ -115,6 +115,8 @@ enum
   PROP_QP_P,
   PROP_QP_B,
   PROP_REF,
+  PROP_D3D11_AWARE,
+  PROP_ADAPTER,
 };
 
 #define DEFAULT_BITRATE (2 * 1024)
@@ -353,6 +355,21 @@ gst_mf_h265_enc_class_init (GstMFH265EncClass * klass, gpointer data)
             G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
   }
 
+  g_object_class_install_property (gobject_class, PROP_D3D11_AWARE,
+      g_param_spec_boolean ("d3d11-aware", "D3D11 Aware",
+          "Whether device can support Direct3D11 interop",
+          device_caps->d3d11_aware,
+          (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+
+  if (device_caps->d3d11_aware) {
+    g_object_class_install_property (gobject_class, PROP_ADAPTER,
+        g_param_spec_uint ("adapter", "Adapter",
+            "DXGI Adapter index for creating device",
+            0, G_MAXUINT32, device_caps->adapter,
+            (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
+            G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+  }
+
   long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name);
   classification = g_strdup_printf ("Codec/Encoder/Video%s",
       (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ?
@@ -411,6 +428,7 @@ gst_mf_h265_enc_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec)
 {
   GstMFH265Enc *self = (GstMFH265Enc *) (object);
+  GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object);
 
   switch (prop_id) {
     case PROP_BITRATE:
@@ -464,6 +482,12 @@ gst_mf_h265_enc_get_property (GObject * object, guint prop_id,
     case PROP_REF:
       g_value_set_uint (value, self->max_num_ref);
       break;
+    case PROP_D3D11_AWARE:
+      g_value_set_boolean (value, klass->device_caps.d3d11_aware);
+      break;
+    case PROP_ADAPTER:
+      g_value_set_uint (value, klass->device_caps.adapter);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -753,7 +777,8 @@ gst_mf_h265_enc_set_src_caps (GstMFVideoEnc * mfenc,
 }
 
 void
-gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank)
+gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank,
+    GList * d3d11_device)
 {
   GTypeInfo type_info = {
     sizeof (GstMFH265EncClass),
@@ -770,5 +795,5 @@ gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank)
 
   GST_DEBUG_CATEGORY_INIT (gst_mf_h265_enc_debug, "mfh265enc", 0, "mfh265enc");
 
-  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info);
+  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device);
 }

sys/mediafoundation/gstmfh265enc.h

@@ -26,7 +26,8 @@
 G_BEGIN_DECLS
 
 void gst_mf_h265_enc_plugin_init (GstPlugin * plugin,
-                                  guint rank);
+                                  guint rank,
+                                  GList * d3d11_device);
 
 G_END_DECLS
 

sys/mediafoundation/gstmftransform.cpp

@@ -22,7 +22,10 @@
 #include "config.h"
 #endif
 
+#include "gstmfconfig.h"
+
 #include <gst/gst.h>
+#include <gmodule.h>
 #include "gstmftransform.h"
 #include "gstmfutils.h"
 #include <string.h>
@@ -37,6 +40,43 @@ GST_DEBUG_CATEGORY_EXTERN (gst_mf_transform_debug);
 
 G_END_DECLS
 
+static GModule *mf_plat_module = NULL;
+typedef HRESULT (__stdcall *pMFTEnum2) (GUID guidCategory,
+                                        UINT32 Flags,
+                                        const MFT_REGISTER_TYPE_INFO * pInputType,
+                                        const MFT_REGISTER_TYPE_INFO * pOutputType,
+                                        IMFAttributes * pAttributes,
+                                        IMFActivate *** pppMFTActivate,
+                                        UINT32 * pnumMFTActivate);
+static pMFTEnum2 GstMFTEnum2Func = NULL;
+
+gboolean
+gst_mf_transform_load_library (void)
+{
+#if GST_MF_HAVE_D3D11
+  static volatile gsize _init = 0;
+  if (g_once_init_enter (&_init)) {
+    mf_plat_module = g_module_open ("mfplat.dll", G_MODULE_BIND_LAZY);
+
+    if (mf_plat_module) {
+      if (!g_module_symbol (mf_plat_module, "MFTEnum2",
+              (gpointer *) & GstMFTEnum2Func)) {
+        GST_WARNING ("Cannot load MFTEnum2 symbol");
+        g_module_close (mf_plat_module);
+        mf_plat_module = NULL;
+        GstMFTEnum2Func = NULL;
+      } else {
+        GST_INFO ("MFTEnum2 symbol is available");
+      }
+    }
+
+    g_once_init_leave (&_init, 1);
+  }
+#endif
+
+  return ! !GstMFTEnum2Func;
+}
+
 typedef HRESULT (*GstMFTransformAsyncCallbackOnEvent) (MediaEventType event,
     GstObject * client);
 
@@ -221,6 +261,7 @@ enum
   PROP_DEVICE_NAME,
   PROP_HARDWARE,
   PROP_ENUM_PARAMS,
+  PROP_D3D11_AWARE,
 };
 
 struct _GstMFTransform
@@ -232,6 +273,7 @@ struct _GstMFTransform
 
   gchar *device_name;
   gboolean hardware;
+  gboolean d3d11_aware;
 
   IMFActivate *activate;
   IMFTransform *transform;
@@ -299,6 +341,10 @@ gst_mf_transform_class_init (GstMFTransformClass * klass)
           "GstMFTransformEnumParams for MFTEnumEx",
           (GParamFlags) (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY |
           G_PARAM_STATIC_STRINGS)));
+  g_object_class_install_property (gobject_class, PROP_D3D11_AWARE,
+      g_param_spec_boolean ("d3d11-aware", "D3D11 Aware",
+          "Whether Direct3D11 supports Direct3D11", FALSE,
+          (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
 }
 
 static void
@@ -382,6 +428,9 @@ gst_mf_transform_get_property (GObject * object, guint prop_id,
     case PROP_HARDWARE:
       g_value_set_boolean (value, self->hardware);
       break;
+    case PROP_D3D11_AWARE:
+      g_value_set_boolean (value, self->d3d11_aware);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -404,6 +453,7 @@ gst_mf_transform_set_property (GObject * object, guint prop_id,
       self->enum_params.category = params->category;
       self->enum_params.enum_flags = params->enum_flags;
       self->enum_params.device_index = params->device_index;
+      self->enum_params.adapter_luid = params->adapter_luid;
       if (params->input_typeinfo) {
         self->enum_params.input_typeinfo = g_new0 (MFT_REGISTER_TYPE_INFO, 1);
         memcpy (self->enum_params.input_typeinfo, params->input_typeinfo,
@@ -438,7 +488,7 @@ gst_mf_transform_main_loop_running_cb (GstMFTransform * self)
 static gpointer
 gst_mf_transform_thread_func (GstMFTransform * self)
 {
-  HRESULT hr;
+  HRESULT hr = S_OK;
   IMFActivate **devices = NULL;
   UINT32 num_devices, i;
   LPWSTR name = NULL;
@@ -454,9 +504,43 @@ gst_mf_transform_thread_func (GstMFTransform * self)
   g_source_attach (source, self->context);
   g_source_unref (source);
 
-  hr = MFTEnumEx (self->enum_params.category, self->enum_params.enum_flags,
-      self->enum_params.input_typeinfo, self->enum_params.output_typeinfo,
-      &devices, &num_devices);
+  /* NOTE: MFTEnum2 is desktop only and requires Windows 10 */
+#if GST_MF_HAVE_D3D11
+  if (GstMFTEnum2Func && self->enum_params.adapter_luid &&
+      (self->enum_params.enum_flags & MFT_ENUM_FLAG_HARDWARE) != 0) {
+    ComPtr<IMFAttributes> attr;
+    LUID luid;
+
+    hr = MFCreateAttributes (&attr, 1);
+    if (!gst_mf_result (hr)) {
+      GST_ERROR_OBJECT (self, "Couldn't create IMFAttributes");
+      goto run_loop;
+    }
+
+    GST_INFO_OBJECT (self,
+        "Enumerating MFT for adapter-luid %" G_GINT64_FORMAT,
+        self->enum_params.adapter_luid);
+
+    luid.LowPart = (DWORD) (self->enum_params.adapter_luid & 0xffffffff);
+    luid.HighPart = (LONG) (self->enum_params.adapter_luid >> 32);
+
+    hr = attr->SetBlob (GST_GUID_MFT_ENUM_ADAPTER_LUID, (BYTE *) &luid,
+        sizeof (LUID));
+    if (!gst_mf_result (hr)) {
+      GST_ERROR_OBJECT (self, "Couldn't set MFT_ENUM_ADAPTER_LUID");
+      goto run_loop;
+    }
+
+    hr = GstMFTEnum2Func (self->enum_params.category,
+        self->enum_params.enum_flags, self->enum_params.input_typeinfo,
+        self->enum_params.output_typeinfo, attr.Get (), &devices, &num_devices);
+  } else
+#endif
+  {
+    hr = MFTEnumEx (self->enum_params.category, self->enum_params.enum_flags,
+        self->enum_params.input_typeinfo, self->enum_params.output_typeinfo,
+        &devices, &num_devices);
+  }
 
   if (!gst_mf_result (hr)) {
     GST_WARNING_OBJECT (self, "MFTEnumEx failure");
@@ -833,6 +917,7 @@ gst_mf_transform_open_internal (GstMFTransformOpenData * data)
 
   if (object->hardware) {
     ComPtr<IMFAttributes> attr;
+    UINT32 supports_d3d11 = 0;
 
     hr = object->transform->GetAttributes (attr.GetAddressOf ());
     if (!gst_mf_result (hr)) {
@@ -846,6 +931,12 @@ gst_mf_transform_open_internal (GstMFTransformOpenData * data)
       goto done;
     }
 
+    hr = attr->GetUINT32 (GST_GUID_MF_SA_D3D11_AWARE, &supports_d3d11);
+    if (gst_mf_result (hr) && supports_d3d11 != 0) {
+      GST_DEBUG_OBJECT (object, "MFT supports direct3d11");
+      object->d3d11_aware = TRUE;
+    }
+
     /* Create our IMFAsyncCallback object so that listen METransformNeedInput
      * and METransformHaveOutput events. The event callback will be called from
      * Media Foundation's worker queue thread */
@@ -909,6 +1000,29 @@ gst_mf_transform_open (GstMFTransform * object)
   return data.ret;
 }
 
+gboolean
+gst_mf_transform_set_device_manager (GstMFTransform * object,
+    IMFDXGIDeviceManager * manager)
+{
+  HRESULT hr;
+
+  g_return_val_if_fail (GST_IS_MF_TRANSFORM (object), FALSE);
+
+  if (!object->transform) {
+    GST_ERROR_OBJECT (object, "IMFTransform is not configured yet");
+    return FALSE;
+  }
+
+  hr = object->transform->ProcessMessage (MFT_MESSAGE_SET_D3D_MANAGER,
+      (ULONG_PTR) manager);
+  if (!gst_mf_result (hr)) {
+    GST_ERROR_OBJECT (object, "Couldn't set device manager");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
 void
 gst_mf_transform_set_new_sample_callback (GstMFTransform * object,
     GstMFTransformNewSampleCallback callback, gpointer user_data)

sys/mediafoundation/gstmftransform.h

@@ -34,6 +34,26 @@ G_DECLARE_FINAL_TYPE (GstMFTransform, gst_mf_transform,
 
 #define GST_MF_TRANSFORM_FLOW_NEED_DATA GST_FLOW_CUSTOM_SUCCESS
 
+/* NOTE: This GUID is defined in mfapi.h header but it's available only for
+ * at least Windows 10 RS1. So defining the GUID here again so that
+ * make use even if build target (e.g., WINVER) wasn't for Windows 10 */
+DEFINE_GUID(GST_GUID_MFT_ENUM_ADAPTER_LUID,
+    0x1d39518c, 0xe220, 0x4da8, 0xa0, 0x7f, 0xba, 0x17, 0x25, 0x52, 0xd6, 0xb1);
+
+/* below GUIDs are defined in mftransform.h for Windows 8 or greater
+ * FIXME: remove below defines when we bump minimum supported OS version to
+ * Windows 10 */
+DEFINE_GUID(GST_GUID_MF_SA_D3D11_AWARE,
+    0x206b4fc8, 0xfcf9, 0x4c51, 0xaf, 0xe3, 0x97, 0x64, 0x36, 0x9e, 0x33, 0xa0);
+DEFINE_GUID(GST_GUID_MF_SA_BUFFERS_PER_SAMPLE,
+    0x873c5171, 0x1e3d, 0x4e25, 0x98, 0x8d, 0xb4, 0x33, 0xce, 0x04, 0x19, 0x83);
+DEFINE_GUID(GST_GUID_MF_SA_D3D11_USAGE,
+    0xe85fe442, 0x2ca3, 0x486e, 0xa9, 0xc7, 0x10, 0x9d, 0xda, 0x60, 0x98, 0x80);
+DEFINE_GUID(GST_GUID_MF_SA_D3D11_SHARED_WITHOUT_MUTEX,
+    0x39dbd44d, 0x2e44, 0x4931, 0xa4, 0xc8, 0x35, 0x2d, 0x3d, 0xc4, 0x21, 0x15);
+DEFINE_GUID(GST_GUID_MF_SA_D3D11_BINDFLAGS,
+    0xeacf97ad, 0x065c, 0x4408, 0xbe, 0xe3, 0xfd, 0xcb, 0xfd, 0x12, 0x8b, 0xe2);
+
 typedef struct _GstMFTransformEnumParams
 {
   GUID category;
@@ -42,16 +62,22 @@ typedef struct _GstMFTransformEnumParams
   MFT_REGISTER_TYPE_INFO *output_typeinfo;
 
   guint device_index;
+  gint64 adapter_luid;
 } GstMFTransformEnumParams;
 
 typedef HRESULT (*GstMFTransformNewSampleCallback) (GstMFTransform * object,
                                                     IMFSample * sample,
                                                     gpointer user_data);
 
+gboolean         gst_mf_transform_load_library    (void);
+
 GstMFTransform * gst_mf_transform_new             (GstMFTransformEnumParams * params);
 
 gboolean        gst_mf_transform_open             (GstMFTransform * object);
 
+gboolean        gst_mf_transform_set_device_manager (GstMFTransform * object,
+                                                     IMFDXGIDeviceManager * manager);
+
 void            gst_mf_transform_set_new_sample_callback (GstMFTransform * object,
                                                           GstMFTransformNewSampleCallback callback,
                                                           gpointer user_data);

sys/mediafoundation/gstmfvideoenc.h

@@ -21,11 +21,17 @@
 #ifndef __GST_MF_VIDEO_ENC_H__
 #define __GST_MF_VIDEO_ENC_H__
 
+#include "gstmfconfig.h"
+
 #include <gst/gst.h>
 #include <gst/video/video.h>
 #include "gstmfutils.h"
 #include "gstmftransform.h"
 
+#if GST_MF_HAVE_D3D11
+#include <gst/d3d11/gstd3d11.h>
+#endif
+
 G_BEGIN_DECLS
 
 #define GST_TYPE_MF_VIDEO_ENC           (gst_mf_video_enc_get_type())
@@ -66,6 +72,11 @@ struct _GstMFVideoEncDeviceCaps
   gboolean max_num_ref;   /* AVEncVideoMaxNumRefFrame */
   guint max_num_ref_high;
   guint max_num_ref_low;
+
+  /* TRUE if MFT support d3d11 and also we can use d3d11 interop */
+  gboolean d3d11_aware;
+  /* DXGI adapter index to use, ignored if d3d11-unaware  */
+  guint adapter;
 };
 
 struct _GstMFVideoEncClassData
@@ -88,6 +99,15 @@ struct _GstMFVideoEnc
   GstFlowReturn last_ret;
 
   GstVideoCodecState *input_state;
+
+#if GST_MF_HAVE_D3D11
+  /* For D3D11 interop. */
+  GstD3D11Device *other_d3d11_device;
+  GstD3D11Device *d3d11_device;
+  IMFDXGIDeviceManager *device_manager;
+  UINT reset_token;
+  IMFVideoSampleAllocatorEx *mf_allocator;
+#endif
 };
 
 struct _GstMFVideoEncClass
@@ -114,7 +134,8 @@ GType gst_mf_video_enc_get_type (void);
 void  gst_mf_video_enc_register (GstPlugin * plugin,
                                  guint rank,
                                  GUID * subtype,
-                                 GTypeInfo * type_info);
+                                 GTypeInfo * type_info,
+                                 GList * d3d11_device);
 
 G_END_DECLS
 

sys/mediafoundation/gstmfvp9enc.cpp

@@ -105,6 +105,8 @@ enum
   PROP_THREADS,
   PROP_CONTENT_TYPE,
   PROP_LOW_LATENCY,
+  PROP_D3D11_AWARE,
+  PROP_ADAPTER,
 };
 
 #define DEFAULT_BITRATE (2 * 1024)
@@ -250,6 +252,21 @@ gst_mf_vp9_enc_class_init (GstMFVP9EncClass * klass, gpointer data)
             G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
   }
 
+  g_object_class_install_property (gobject_class, PROP_D3D11_AWARE,
+      g_param_spec_boolean ("d3d11-aware", "D3D11 Aware",
+          "Whether device can support Direct3D11 interop",
+          device_caps->d3d11_aware,
+          (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+
+  if (device_caps->d3d11_aware) {
+    g_object_class_install_property (gobject_class, PROP_ADAPTER,
+        g_param_spec_uint ("adapter", "Adapter",
+            "DXGI Adapter index for creating device",
+            0, G_MAXUINT32, device_caps->adapter,
+            (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
+            G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+  }
+
   long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name);
   classification = g_strdup_printf ("Codec/Encoder/Video%s",
       (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ?
@@ -300,6 +317,7 @@ gst_mf_vp9_enc_get_property (GObject * object, guint prop_id,
     GValue * value, GParamSpec * pspec)
 {
   GstMFVP9Enc *self = (GstMFVP9Enc *) (object);
+  GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object);
 
   switch (prop_id) {
     case PROP_BITRATE:
@@ -326,6 +344,12 @@ gst_mf_vp9_enc_get_property (GObject * object, guint prop_id,
     case PROP_LOW_LATENCY:
       g_value_set_boolean (value, self->low_latency);
       break;
+    case PROP_D3D11_AWARE:
+      g_value_set_boolean (value, klass->device_caps.d3d11_aware);
+      break;
+    case PROP_ADAPTER:
+      g_value_set_uint (value, klass->device_caps.adapter);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -529,7 +553,8 @@ gst_mf_vp9_enc_set_src_caps (GstMFVideoEnc * mfenc,
 }
 
 void
-gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank)
+gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank,
+    GList * d3d11_device)
 {
   GTypeInfo type_info = {
     sizeof (GstMFVP9EncClass),
@@ -546,5 +571,5 @@ gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank)
 
   GST_DEBUG_CATEGORY_INIT (gst_mf_vp9_enc_debug, "mfvp9enc", 0, "mfvp9enc");
 
-  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info);
+  gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device);
 }

sys/mediafoundation/gstmfvp9enc.h

@@ -25,7 +25,8 @@
 G_BEGIN_DECLS
 
 void gst_mf_vp9_enc_plugin_init (GstPlugin * plugin,
-                                 guint rank);
+                                 guint rank,
+                                 GList * d3d11_device);
 
 G_END_DECLS
 

sys/mediafoundation/meson.build

@@ -37,8 +37,11 @@ mf_header_deps = [
 winapi_desktop = false
 winapi_app = false
 have_capture_engine = false
+have_mf_d3d11 = false
 mf_lib_deps = []
 mf_config = configuration_data()
+extra_c_args = ['-DCOBJMACROS']
+extra_cpp_args = []
 
 mf_option = get_option('mediafoundation')
 if host_system != 'windows' or mf_option.disabled()
@@ -120,10 +123,22 @@ endif
 
 if winapi_desktop
   mf_sources += mf_desktop_sources
+  # We need d3d11_4.h header for querying "ExtendedNV12SharedTextureSupported"
+  # Since MFTEnum2 is desktop only we don't support d3d11 interop for UWP build
+  # And because MFTEnum2 is Windows 10 API, we will load MFTEnum2 symbol
+  # by using g_module_open() so that keep supporting old OS versions
+  if gstd3d11_dep.found() and cc.has_header('d3d11_4.h')
+    have_mf_d3d11 = true
+    mf_lib_deps += [gstd3d11_dep, gmodule_dep]
+    extra_c_args += ['-DGST_USE_UNSTABLE_API']
+    extra_cpp_args += ['-DGST_USE_UNSTABLE_API']
+    message ('Enable D3D11 interop for MediaFoundation plugin')
+  endif
 endif
 
 mf_config.set10('GST_MF_WINAPI_APP', winapi_app)
 mf_config.set10('GST_MF_WINAPI_DESKTOP', winapi_desktop)
+mf_config.set10('GST_MF_HAVE_D3D11', have_mf_d3d11)
 
 configure_file(
   output: 'gstmfconfig.h',
@@ -132,8 +147,8 @@ configure_file(
 
 gstmediafoundation = library('gstmediafoundation',
   mf_sources,
-  c_args : gst_plugins_bad_args + ['-DCOBJMACROS'],
-  cpp_args : gst_plugins_bad_args,
+  c_args : gst_plugins_bad_args + extra_c_args,
+  cpp_args : gst_plugins_bad_args + extra_cpp_args,
   include_directories : [configinc],
   dependencies : [gstbase_dep, gstvideo_dep, gstaudio_dep, gstpbutils_dep] + mf_lib_deps,
   install : true,

sys/mediafoundation/plugin.c

@@ -27,6 +27,7 @@
 #include <winapifamily.h>
 
 #include <gst/gst.h>
+#include <gst/video/video.h>
 #include "gstmfvideosrc.h"
 #include "gstmfdevice.h"
 #include "gstmfutils.h"
@@ -35,6 +36,11 @@
 #include "gstmfvp9enc.h"
 #include "gstmfaacenc.h"
 #include "gstmfmp3enc.h"
+#include "gstmftransform.h"
+
+#if GST_MF_HAVE_D3D11
+#include <gst/d3d11/gstd3d11.h>
+#endif
 
 GST_DEBUG_CATEGORY (gst_mf_debug);
 GST_DEBUG_CATEGORY (gst_mf_utils_debug);
@@ -51,11 +57,103 @@ plugin_deinit (gpointer data)
   MFShutdown ();
 }
 
+#if GST_MF_HAVE_D3D11
+static GList *
+get_d3d11_devices (void)
+{
+  GList *ret = NULL;
+  guint i;
+  HRESULT hr;
+  IMFVideoSampleAllocatorEx *allocator = NULL;
+
+  /* Check whether we can use IMFVideoSampleAllocatorEx interface */
+  hr = MFCreateVideoSampleAllocatorEx (&IID_IMFVideoSampleAllocatorEx,
+      &allocator);
+  if (!gst_mf_result (hr)) {
+    GST_DEBUG ("IMFVideoSampleAllocatorEx interface is unavailable");
+    return NULL;
+  } else {
+    IMFVideoSampleAllocatorEx_Release (allocator);
+  }
+
+  /* AMD seems supporting up to 12 cards, and 8 for NVIDIA */
+  for (i = 0; i < 12; i++) {
+    GstD3D11Device *device;
+    gboolean is_hardware = FALSE;
+    const GstD3D11Format *d3d11_format;
+    ID3D11Device *device_handle;
+    D3D11_FEATURE_DATA_D3D11_OPTIONS4 options = { 0, };
+    UINT supported = 0;
+
+    device = gst_d3d11_device_new (i, D3D11_CREATE_DEVICE_VIDEO_SUPPORT);
+
+    if (!device)
+      break;
+
+    g_object_get (device, "hardware", &is_hardware, NULL);
+
+    if (!is_hardware) {
+      GST_DEBUG_OBJECT (device, "Given d3d11 device is not for hardware");
+      gst_object_unref (device);
+      continue;
+    }
+
+    /* device can support NV12 format? */
+    d3d11_format =
+        gst_d3d11_device_format_from_gst (device, GST_VIDEO_FORMAT_NV12);
+    if (!d3d11_format || d3d11_format->dxgi_format != DXGI_FORMAT_NV12) {
+      GST_DEBUG_OBJECT (device,
+          "Given d3d11 device cannot support NV12 format");
+      gst_object_unref (device);
+      continue;
+    }
+
+    /* device can support ExtendedNV12SharedTextureSupported?
+     *
+     * NOTE: we will make use of per encoder object d3d11 device without
+     * sharing it in a pipeline because MF needs D3D11_CREATE_DEVICE_VIDEO_SUPPORT
+     * but the flag doesn't used for the other our use cases.
+     * So we need texture sharing feature so that we can copy d3d11 texture into
+     * MF specific texture pool without download texture */
+
+    device_handle = gst_d3d11_device_get_device_handle (device);
+    hr = ID3D11Device_CheckFeatureSupport (device_handle,
+        D3D11_FEATURE_D3D11_OPTIONS4, &options, sizeof (options));
+    if (!gst_d3d11_result (hr, device) ||
+        !options.ExtendedNV12SharedTextureSupported) {
+      GST_DEBUG_OBJECT (device,
+          "Given d3d11 device cannot support NV12 format for shared texture");
+      gst_object_unref (device);
+      continue;
+    }
+
+    /* can we bind NV12 texture for encoder? */
+    hr = ID3D11Device_CheckFormatSupport (device_handle,
+        DXGI_FORMAT_NV12, &supported);
+
+    if (!gst_d3d11_result (hr, device)) {
+      GST_DEBUG_OBJECT (device, "Couldn't query format support");
+      gst_object_unref (device);
+      continue;
+    } else if ((supported & D3D11_FORMAT_SUPPORT_VIDEO_ENCODER) == 0) {
+      GST_DEBUG_OBJECT (device, "We cannot bind NV12 format for encoding");
+      gst_object_unref (device);
+      continue;
+    }
+
+    ret = g_list_append (ret, device);
+  }
+
+  return ret;
+}
+#endif
+
 static gboolean
 plugin_init (GstPlugin * plugin)
 {
   HRESULT hr;
   GstRank rank = GST_RANK_SECONDARY;
+  GList *device_list = NULL;
 
   /**
    * plugin-mediafoundation:
@@ -86,13 +184,25 @@ plugin_init (GstPlugin * plugin)
   rank = GST_RANK_PRIMARY + 1;
 #endif
 
+  /* FIXME: In order to create MFT for a specific GPU, MFTEnum2() API is
+   * required API but it's desktop only.
+   * So, resulting MFT and D3D11 might not be compatible in case of multi-GPU
+   * environment on UWP. */
+#if GST_MF_HAVE_D3D11
+  if (gst_mf_transform_load_library ())
+    device_list = get_d3d11_devices ();
+#endif
+
   gst_element_register (plugin, "mfvideosrc", rank, GST_TYPE_MF_VIDEO_SRC);
   gst_device_provider_register (plugin, "mfdeviceprovider",
       rank, GST_TYPE_MF_DEVICE_PROVIDER);
 
-  gst_mf_h264_enc_plugin_init (plugin, GST_RANK_SECONDARY);
-  gst_mf_h265_enc_plugin_init (plugin, GST_RANK_SECONDARY);
-  gst_mf_vp9_enc_plugin_init (plugin, GST_RANK_SECONDARY);
+  gst_mf_h264_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
+  gst_mf_h265_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
+  gst_mf_vp9_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list);
+
+  if (device_list)
+    g_list_free_full (device_list, gst_object_unref);
 
   gst_mf_aac_enc_plugin_init (plugin, GST_RANK_SECONDARY);
   gst_mf_mp3_enc_plugin_init (plugin, GST_RANK_SECONDARY);