From 84db4b68fe1347c790843af753de7cffbb2646fd Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Mon, 21 Dec 2020 05:11:03 +0900 Subject: [PATCH] mfvideoenc: Add support for Direct3D11 texture Initial support for d3d11 texture so that encoder can copy upstream d3d11 texture into encoder's own texture pool without downloading memory. This implementation requires MFTEnum2() API for creating MFT (Media Foundation Transform) object for specific GPU but the API is Windows 10 desktop only. So UWP is not target of this change. See also https://docs.microsoft.com/en-us/windows/win32/api/mfapi/nf-mfapi-mftenum2 Note that, for MF plugin to be able to support old OS versions without breakage, this commit will load MFTEnum2() symbol by using g_module_open() Summary of required system environment: - Needs Windows 10 (probably at least RS 1 update) - GPU should support ExtendedNV12SharedTextureSupported feature - Desktop application only (UWP is not supported yet) Part-of: --- sys/mediafoundation/gstmfh264enc.cpp | 35 +- sys/mediafoundation/gstmfh264enc.h | 3 +- sys/mediafoundation/gstmfh265enc.cpp | 29 +- sys/mediafoundation/gstmfh265enc.h | 3 +- sys/mediafoundation/gstmftransform.cpp | 122 +++- sys/mediafoundation/gstmftransform.h | 26 + sys/mediafoundation/gstmfvideoenc.cpp | 879 +++++++++++++++++++++---- sys/mediafoundation/gstmfvideoenc.h | 23 +- sys/mediafoundation/gstmfvp9enc.cpp | 29 +- sys/mediafoundation/gstmfvp9enc.h | 3 +- sys/mediafoundation/meson.build | 19 +- sys/mediafoundation/plugin.c | 116 +++- 12 files changed, 1129 insertions(+), 158 deletions(-) diff --git a/sys/mediafoundation/gstmfh264enc.cpp b/sys/mediafoundation/gstmfh264enc.cpp index c672eb3fb3..a9ea77dd70 100644 --- a/sys/mediafoundation/gstmfh264enc.cpp +++ b/sys/mediafoundation/gstmfh264enc.cpp @@ -35,12 +35,18 @@ #include "config.h" #endif +#include "gstmfconfig.h" + #include #include #include "gstmfvideoenc.h" #include "gstmfh264enc.h" #include +#if GST_MF_HAVE_D3D11 +#include +#endif + using namespace Microsoft::WRL; GST_DEBUG_CATEGORY (gst_mf_h264_enc_debug); @@ -154,6 +160,8 @@ enum PROP_QP_P, PROP_QP_B, PROP_REF, + PROP_D3D11_AWARE, + PROP_ADAPTER, }; #define DEFAULT_BITRATE (2 * 1024) @@ -456,6 +464,21 @@ gst_mf_h264_enc_class_init (GstMFH264EncClass * klass, gpointer data) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } + g_object_class_install_property (gobject_class, PROP_D3D11_AWARE, + g_param_spec_boolean ("d3d11-aware", "D3D11 Aware", + "Whether device can support Direct3D11 interop", + device_caps->d3d11_aware, + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + if (device_caps->d3d11_aware) { + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_uint ("adapter", "Adapter", + "DXGI Adapter index for creating device", + 0, G_MAXUINT32, device_caps->adapter, + (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE | + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + } + long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name); classification = g_strdup_printf ("Codec/Encoder/Video%s", (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ? @@ -519,6 +542,7 @@ gst_mf_h264_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstMFH264Enc *self = (GstMFH264Enc *) (object); + GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object); switch (prop_id) { case PROP_BITRATE: @@ -587,6 +611,12 @@ gst_mf_h264_enc_get_property (GObject * object, guint prop_id, case PROP_REF: g_value_set_uint (value, self->max_num_ref); break; + case PROP_D3D11_AWARE: + g_value_set_boolean (value, klass->device_caps.d3d11_aware); + break; + case PROP_ADAPTER: + g_value_set_uint (value, klass->device_caps.adapter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -984,7 +1014,8 @@ gst_mf_h264_enc_set_src_caps (GstMFVideoEnc * mfenc, } void -gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank) +gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank, + GList * d3d11_device) { GTypeInfo type_info = { sizeof (GstMFH264EncClass), @@ -1001,5 +1032,5 @@ gst_mf_h264_enc_plugin_init (GstPlugin * plugin, guint rank) GST_DEBUG_CATEGORY_INIT (gst_mf_h264_enc_debug, "mfh264enc", 0, "mfh264enc"); - gst_mf_video_enc_register (plugin, rank, &subtype, &type_info); + gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device); } diff --git a/sys/mediafoundation/gstmfh264enc.h b/sys/mediafoundation/gstmfh264enc.h index 00e6de0c01..bacc42d3cf 100644 --- a/sys/mediafoundation/gstmfh264enc.h +++ b/sys/mediafoundation/gstmfh264enc.h @@ -26,7 +26,8 @@ G_BEGIN_DECLS void gst_mf_h264_enc_plugin_init (GstPlugin * plugin, - guint rank); + guint rank, + GList * d3d11_device); G_END_DECLS diff --git a/sys/mediafoundation/gstmfh265enc.cpp b/sys/mediafoundation/gstmfh265enc.cpp index 95c87b675c..cb59d40b36 100644 --- a/sys/mediafoundation/gstmfh265enc.cpp +++ b/sys/mediafoundation/gstmfh265enc.cpp @@ -115,6 +115,8 @@ enum PROP_QP_P, PROP_QP_B, PROP_REF, + PROP_D3D11_AWARE, + PROP_ADAPTER, }; #define DEFAULT_BITRATE (2 * 1024) @@ -353,6 +355,21 @@ gst_mf_h265_enc_class_init (GstMFH265EncClass * klass, gpointer data) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } + g_object_class_install_property (gobject_class, PROP_D3D11_AWARE, + g_param_spec_boolean ("d3d11-aware", "D3D11 Aware", + "Whether device can support Direct3D11 interop", + device_caps->d3d11_aware, + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + if (device_caps->d3d11_aware) { + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_uint ("adapter", "Adapter", + "DXGI Adapter index for creating device", + 0, G_MAXUINT32, device_caps->adapter, + (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE | + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + } + long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name); classification = g_strdup_printf ("Codec/Encoder/Video%s", (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ? @@ -411,6 +428,7 @@ gst_mf_h265_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstMFH265Enc *self = (GstMFH265Enc *) (object); + GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object); switch (prop_id) { case PROP_BITRATE: @@ -464,6 +482,12 @@ gst_mf_h265_enc_get_property (GObject * object, guint prop_id, case PROP_REF: g_value_set_uint (value, self->max_num_ref); break; + case PROP_D3D11_AWARE: + g_value_set_boolean (value, klass->device_caps.d3d11_aware); + break; + case PROP_ADAPTER: + g_value_set_uint (value, klass->device_caps.adapter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -753,7 +777,8 @@ gst_mf_h265_enc_set_src_caps (GstMFVideoEnc * mfenc, } void -gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank) +gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank, + GList * d3d11_device) { GTypeInfo type_info = { sizeof (GstMFH265EncClass), @@ -770,5 +795,5 @@ gst_mf_h265_enc_plugin_init (GstPlugin * plugin, guint rank) GST_DEBUG_CATEGORY_INIT (gst_mf_h265_enc_debug, "mfh265enc", 0, "mfh265enc"); - gst_mf_video_enc_register (plugin, rank, &subtype, &type_info); + gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device); } \ No newline at end of file diff --git a/sys/mediafoundation/gstmfh265enc.h b/sys/mediafoundation/gstmfh265enc.h index 92e16114a8..990064e0ad 100644 --- a/sys/mediafoundation/gstmfh265enc.h +++ b/sys/mediafoundation/gstmfh265enc.h @@ -26,7 +26,8 @@ G_BEGIN_DECLS void gst_mf_h265_enc_plugin_init (GstPlugin * plugin, - guint rank); + guint rank, + GList * d3d11_device); G_END_DECLS diff --git a/sys/mediafoundation/gstmftransform.cpp b/sys/mediafoundation/gstmftransform.cpp index 46da00158f..c6424535eb 100644 --- a/sys/mediafoundation/gstmftransform.cpp +++ b/sys/mediafoundation/gstmftransform.cpp @@ -22,7 +22,10 @@ #include "config.h" #endif +#include "gstmfconfig.h" + #include +#include #include "gstmftransform.h" #include "gstmfutils.h" #include @@ -37,6 +40,43 @@ GST_DEBUG_CATEGORY_EXTERN (gst_mf_transform_debug); G_END_DECLS +static GModule *mf_plat_module = NULL; +typedef HRESULT (__stdcall *pMFTEnum2) (GUID guidCategory, + UINT32 Flags, + const MFT_REGISTER_TYPE_INFO * pInputType, + const MFT_REGISTER_TYPE_INFO * pOutputType, + IMFAttributes * pAttributes, + IMFActivate *** pppMFTActivate, + UINT32 * pnumMFTActivate); +static pMFTEnum2 GstMFTEnum2Func = NULL; + +gboolean +gst_mf_transform_load_library (void) +{ +#if GST_MF_HAVE_D3D11 + static volatile gsize _init = 0; + if (g_once_init_enter (&_init)) { + mf_plat_module = g_module_open ("mfplat.dll", G_MODULE_BIND_LAZY); + + if (mf_plat_module) { + if (!g_module_symbol (mf_plat_module, "MFTEnum2", + (gpointer *) & GstMFTEnum2Func)) { + GST_WARNING ("Cannot load MFTEnum2 symbol"); + g_module_close (mf_plat_module); + mf_plat_module = NULL; + GstMFTEnum2Func = NULL; + } else { + GST_INFO ("MFTEnum2 symbol is available"); + } + } + + g_once_init_leave (&_init, 1); + } +#endif + + return ! !GstMFTEnum2Func; +} + typedef HRESULT (*GstMFTransformAsyncCallbackOnEvent) (MediaEventType event, GstObject * client); @@ -221,6 +261,7 @@ enum PROP_DEVICE_NAME, PROP_HARDWARE, PROP_ENUM_PARAMS, + PROP_D3D11_AWARE, }; struct _GstMFTransform @@ -232,6 +273,7 @@ struct _GstMFTransform gchar *device_name; gboolean hardware; + gboolean d3d11_aware; IMFActivate *activate; IMFTransform *transform; @@ -299,6 +341,10 @@ gst_mf_transform_class_init (GstMFTransformClass * klass) "GstMFTransformEnumParams for MFTEnumEx", (GParamFlags) (G_PARAM_WRITABLE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property (gobject_class, PROP_D3D11_AWARE, + g_param_spec_boolean ("d3d11-aware", "D3D11 Aware", + "Whether Direct3D11 supports Direct3D11", FALSE, + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); } static void @@ -382,6 +428,9 @@ gst_mf_transform_get_property (GObject * object, guint prop_id, case PROP_HARDWARE: g_value_set_boolean (value, self->hardware); break; + case PROP_D3D11_AWARE: + g_value_set_boolean (value, self->d3d11_aware); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -404,6 +453,7 @@ gst_mf_transform_set_property (GObject * object, guint prop_id, self->enum_params.category = params->category; self->enum_params.enum_flags = params->enum_flags; self->enum_params.device_index = params->device_index; + self->enum_params.adapter_luid = params->adapter_luid; if (params->input_typeinfo) { self->enum_params.input_typeinfo = g_new0 (MFT_REGISTER_TYPE_INFO, 1); memcpy (self->enum_params.input_typeinfo, params->input_typeinfo, @@ -438,7 +488,7 @@ gst_mf_transform_main_loop_running_cb (GstMFTransform * self) static gpointer gst_mf_transform_thread_func (GstMFTransform * self) { - HRESULT hr; + HRESULT hr = S_OK; IMFActivate **devices = NULL; UINT32 num_devices, i; LPWSTR name = NULL; @@ -454,9 +504,43 @@ gst_mf_transform_thread_func (GstMFTransform * self) g_source_attach (source, self->context); g_source_unref (source); - hr = MFTEnumEx (self->enum_params.category, self->enum_params.enum_flags, - self->enum_params.input_typeinfo, self->enum_params.output_typeinfo, - &devices, &num_devices); + /* NOTE: MFTEnum2 is desktop only and requires Windows 10 */ +#if GST_MF_HAVE_D3D11 + if (GstMFTEnum2Func && self->enum_params.adapter_luid && + (self->enum_params.enum_flags & MFT_ENUM_FLAG_HARDWARE) != 0) { + ComPtr attr; + LUID luid; + + hr = MFCreateAttributes (&attr, 1); + if (!gst_mf_result (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create IMFAttributes"); + goto run_loop; + } + + GST_INFO_OBJECT (self, + "Enumerating MFT for adapter-luid %" G_GINT64_FORMAT, + self->enum_params.adapter_luid); + + luid.LowPart = (DWORD) (self->enum_params.adapter_luid & 0xffffffff); + luid.HighPart = (LONG) (self->enum_params.adapter_luid >> 32); + + hr = attr->SetBlob (GST_GUID_MFT_ENUM_ADAPTER_LUID, (BYTE *) &luid, + sizeof (LUID)); + if (!gst_mf_result (hr)) { + GST_ERROR_OBJECT (self, "Couldn't set MFT_ENUM_ADAPTER_LUID"); + goto run_loop; + } + + hr = GstMFTEnum2Func (self->enum_params.category, + self->enum_params.enum_flags, self->enum_params.input_typeinfo, + self->enum_params.output_typeinfo, attr.Get (), &devices, &num_devices); + } else +#endif + { + hr = MFTEnumEx (self->enum_params.category, self->enum_params.enum_flags, + self->enum_params.input_typeinfo, self->enum_params.output_typeinfo, + &devices, &num_devices); + } if (!gst_mf_result (hr)) { GST_WARNING_OBJECT (self, "MFTEnumEx failure"); @@ -833,6 +917,7 @@ gst_mf_transform_open_internal (GstMFTransformOpenData * data) if (object->hardware) { ComPtr attr; + UINT32 supports_d3d11 = 0; hr = object->transform->GetAttributes (attr.GetAddressOf ()); if (!gst_mf_result (hr)) { @@ -846,6 +931,12 @@ gst_mf_transform_open_internal (GstMFTransformOpenData * data) goto done; } + hr = attr->GetUINT32 (GST_GUID_MF_SA_D3D11_AWARE, &supports_d3d11); + if (gst_mf_result (hr) && supports_d3d11 != 0) { + GST_DEBUG_OBJECT (object, "MFT supports direct3d11"); + object->d3d11_aware = TRUE; + } + /* Create our IMFAsyncCallback object so that listen METransformNeedInput * and METransformHaveOutput events. The event callback will be called from * Media Foundation's worker queue thread */ @@ -909,6 +1000,29 @@ gst_mf_transform_open (GstMFTransform * object) return data.ret; } +gboolean +gst_mf_transform_set_device_manager (GstMFTransform * object, + IMFDXGIDeviceManager * manager) +{ + HRESULT hr; + + g_return_val_if_fail (GST_IS_MF_TRANSFORM (object), FALSE); + + if (!object->transform) { + GST_ERROR_OBJECT (object, "IMFTransform is not configured yet"); + return FALSE; + } + + hr = object->transform->ProcessMessage (MFT_MESSAGE_SET_D3D_MANAGER, + (ULONG_PTR) manager); + if (!gst_mf_result (hr)) { + GST_ERROR_OBJECT (object, "Couldn't set device manager"); + return FALSE; + } + + return TRUE; +} + void gst_mf_transform_set_new_sample_callback (GstMFTransform * object, GstMFTransformNewSampleCallback callback, gpointer user_data) diff --git a/sys/mediafoundation/gstmftransform.h b/sys/mediafoundation/gstmftransform.h index f9abdee7f3..8e2402ed28 100644 --- a/sys/mediafoundation/gstmftransform.h +++ b/sys/mediafoundation/gstmftransform.h @@ -34,6 +34,26 @@ G_DECLARE_FINAL_TYPE (GstMFTransform, gst_mf_transform, #define GST_MF_TRANSFORM_FLOW_NEED_DATA GST_FLOW_CUSTOM_SUCCESS +/* NOTE: This GUID is defined in mfapi.h header but it's available only for + * at least Windows 10 RS1. So defining the GUID here again so that + * make use even if build target (e.g., WINVER) wasn't for Windows 10 */ +DEFINE_GUID(GST_GUID_MFT_ENUM_ADAPTER_LUID, + 0x1d39518c, 0xe220, 0x4da8, 0xa0, 0x7f, 0xba, 0x17, 0x25, 0x52, 0xd6, 0xb1); + +/* below GUIDs are defined in mftransform.h for Windows 8 or greater + * FIXME: remove below defines when we bump minimum supported OS version to + * Windows 10 */ +DEFINE_GUID(GST_GUID_MF_SA_D3D11_AWARE, + 0x206b4fc8, 0xfcf9, 0x4c51, 0xaf, 0xe3, 0x97, 0x64, 0x36, 0x9e, 0x33, 0xa0); +DEFINE_GUID(GST_GUID_MF_SA_BUFFERS_PER_SAMPLE, + 0x873c5171, 0x1e3d, 0x4e25, 0x98, 0x8d, 0xb4, 0x33, 0xce, 0x04, 0x19, 0x83); +DEFINE_GUID(GST_GUID_MF_SA_D3D11_USAGE, + 0xe85fe442, 0x2ca3, 0x486e, 0xa9, 0xc7, 0x10, 0x9d, 0xda, 0x60, 0x98, 0x80); +DEFINE_GUID(GST_GUID_MF_SA_D3D11_SHARED_WITHOUT_MUTEX, + 0x39dbd44d, 0x2e44, 0x4931, 0xa4, 0xc8, 0x35, 0x2d, 0x3d, 0xc4, 0x21, 0x15); +DEFINE_GUID(GST_GUID_MF_SA_D3D11_BINDFLAGS, + 0xeacf97ad, 0x065c, 0x4408, 0xbe, 0xe3, 0xfd, 0xcb, 0xfd, 0x12, 0x8b, 0xe2); + typedef struct _GstMFTransformEnumParams { GUID category; @@ -42,16 +62,22 @@ typedef struct _GstMFTransformEnumParams MFT_REGISTER_TYPE_INFO *output_typeinfo; guint device_index; + gint64 adapter_luid; } GstMFTransformEnumParams; typedef HRESULT (*GstMFTransformNewSampleCallback) (GstMFTransform * object, IMFSample * sample, gpointer user_data); +gboolean gst_mf_transform_load_library (void); + GstMFTransform * gst_mf_transform_new (GstMFTransformEnumParams * params); gboolean gst_mf_transform_open (GstMFTransform * object); +gboolean gst_mf_transform_set_device_manager (GstMFTransform * object, + IMFDXGIDeviceManager * manager); + void gst_mf_transform_set_new_sample_callback (GstMFTransform * object, GstMFTransformNewSampleCallback callback, gpointer user_data); diff --git a/sys/mediafoundation/gstmfvideoenc.cpp b/sys/mediafoundation/gstmfvideoenc.cpp index 372b946805..c6edba583c 100644 --- a/sys/mediafoundation/gstmfvideoenc.cpp +++ b/sys/mediafoundation/gstmfvideoenc.cpp @@ -41,6 +41,9 @@ G_END_DECLS G_DEFINE_ABSTRACT_TYPE (GstMFVideoEnc, gst_mf_video_enc, GST_TYPE_VIDEO_ENCODER); +static void gst_mf_video_enc_dispose (GObject * object); +static void gst_mf_video_enc_set_context (GstElement * element, + GstContext * context); static gboolean gst_mf_video_enc_open (GstVideoEncoder * enc); static gboolean gst_mf_video_enc_close (GstVideoEncoder * enc); static gboolean gst_mf_video_enc_set_format (GstVideoEncoder * enc, @@ -49,6 +52,12 @@ static GstFlowReturn gst_mf_video_enc_handle_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame); static GstFlowReturn gst_mf_video_enc_finish (GstVideoEncoder * enc); static gboolean gst_mf_video_enc_flush (GstVideoEncoder * enc); +static gboolean gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc, + GstQuery * query); +static gboolean gst_mf_video_enc_sink_query (GstVideoEncoder * enc, + GstQuery * query); +static gboolean gst_mf_video_enc_src_query (GstVideoEncoder * enc, + GstQuery * query); static HRESULT gst_mf_video_on_new_sample (GstMFTransform * object, IMFSample * sample, GstMFVideoEnc * self); @@ -56,8 +65,14 @@ static HRESULT gst_mf_video_on_new_sample (GstMFTransform * object, static void gst_mf_video_enc_class_init (GstMFVideoEncClass * klass) { + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GstVideoEncoderClass *videoenc_class = GST_VIDEO_ENCODER_CLASS (klass); + gobject_class->dispose = gst_mf_video_enc_dispose; + + element_class->set_context = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_context); + videoenc_class->open = GST_DEBUG_FUNCPTR (gst_mf_video_enc_open); videoenc_class->close = GST_DEBUG_FUNCPTR (gst_mf_video_enc_close); videoenc_class->set_format = GST_DEBUG_FUNCPTR (gst_mf_video_enc_set_format); @@ -65,6 +80,12 @@ gst_mf_video_enc_class_init (GstMFVideoEncClass * klass) GST_DEBUG_FUNCPTR (gst_mf_video_enc_handle_frame); videoenc_class->finish = GST_DEBUG_FUNCPTR (gst_mf_video_enc_finish); videoenc_class->flush = GST_DEBUG_FUNCPTR (gst_mf_video_enc_flush); + videoenc_class->propose_allocation = + GST_DEBUG_FUNCPTR (gst_mf_video_enc_propose_allocation); + videoenc_class->sink_query = + GST_DEBUG_FUNCPTR (gst_mf_video_enc_sink_query); + videoenc_class->src_query = + GST_DEBUG_FUNCPTR (gst_mf_video_enc_src_query); gst_type_mark_as_plugin_api (GST_TYPE_MF_VIDEO_ENC, (GstPluginAPIFlags) 0); } @@ -74,15 +95,89 @@ gst_mf_video_enc_init (GstMFVideoEnc * self) { } +static void +gst_mf_video_enc_dispose (GObject * object) +{ +#if GST_MF_HAVE_D3D11 + GstMFVideoEnc *self = GST_MF_VIDEO_ENC (object); + + gst_clear_object (&self->d3d11_device); + gst_clear_object (&self->other_d3d11_device); +#endif + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_mf_video_enc_set_context (GstElement * element, GstContext * context) +{ +#if GST_MF_HAVE_D3D11 + GstMFVideoEnc *self = GST_MF_VIDEO_ENC (element); + + gst_d3d11_handle_set_context (element, context, 0, &self->other_d3d11_device); +#endif + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + static gboolean gst_mf_video_enc_open (GstVideoEncoder * enc) { GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc); GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (enc); + GstMFVideoEncDeviceCaps *device_caps = &klass->device_caps; GstMFTransformEnumParams enum_params = { 0, }; + gint64 adapter_luid = 0; MFT_REGISTER_TYPE_INFO output_type; gboolean ret; +#if GST_MF_HAVE_D3D11 + if (device_caps->d3d11_aware) { + HRESULT hr; + ID3D11Device *device_handle; + GstD3D11Device *device; + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), + device_caps->adapter, &self->other_d3d11_device)) { + GST_ERROR_OBJECT (self, "Other d3d11 device is unavailable"); + return FALSE; + } + + /* Create our own device with D3D11_CREATE_DEVICE_VIDEO_SUPPORT */ + self->d3d11_device = gst_d3d11_device_new (device_caps->adapter, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT); + if (!self->d3d11_device) { + GST_ERROR_OBJECT (self, "Couldn't create internal d3d11 device"); + gst_clear_object (&self->other_d3d11_device); + return FALSE; + } + + device = self->d3d11_device; + + hr = MFCreateDXGIDeviceManager (&self->reset_token, + &self->device_manager); + if (!gst_mf_result (hr)) { + GST_ERROR_OBJECT (self, "Couldn't create DXGI device manager"); + gst_clear_object (&self->other_d3d11_device); + gst_clear_object (&self->d3d11_device); + return FALSE; + } + + device_handle = gst_d3d11_device_get_device_handle (device); + hr = self->device_manager->ResetDevice ((IUnknown *) device_handle, + self->reset_token); + if (!gst_mf_result (hr)) { + GST_ERROR_OBJECT (self, + "Couldn't reset device with given d3d11 device"); + gst_clear_object (&self->other_d3d11_device); + gst_clear_object (&self->d3d11_device); + return FALSE; + } + + g_object_get (self->d3d11_device, "adapter-luid", &adapter_luid, NULL); + } +#endif + output_type.guidMajorType = MFMediaType_Video; output_type.guidSubtype = klass->codec_id; @@ -91,14 +186,21 @@ gst_mf_video_enc_open (GstVideoEncoder * enc) enum_params.output_typeinfo = &output_type; enum_params.device_index = klass->device_index; - GST_DEBUG_OBJECT (self, "Create MFT with enum flags 0x%x, device index %d", - klass->enum_flags, klass->device_index); + if (device_caps->d3d11_aware) + enum_params.adapter_luid = adapter_luid; + + GST_DEBUG_OBJECT (self, + "Create MFT with enum flags: 0x%x, device index: %d, d3d11 aware: %d, " + "adapter-luid %" G_GINT64_FORMAT, klass->enum_flags, klass->device_index, + device_caps->d3d11_aware, adapter_luid); self->transform = gst_mf_transform_new (&enum_params); ret = !!self->transform; - if (!ret) + if (!ret) { GST_ERROR_OBJECT (self, "Cannot create MFT object"); + return FALSE; + } /* In case of hardware MFT, it will be running on async mode. * And new output sample callback will be called from Media Foundation's @@ -129,6 +231,22 @@ gst_mf_video_enc_close (GstVideoEncoder * enc) self->input_state = NULL; } +#if GST_MF_HAVE_D3D11 + if (self->device_manager) { + self->device_manager->Release (); + self->device_manager = nullptr; + } + + if (self->mf_allocator) { + self->mf_allocator->UninitializeSampleAllocator (); + self->mf_allocator->Release (); + self->mf_allocator = NULL; + } + + gst_clear_object (&self->other_d3d11_device); + gst_clear_object (&self->d3d11_device); +#endif + return TRUE; } @@ -158,6 +276,16 @@ gst_mf_video_enc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state) return FALSE; } + if (self->device_manager) { + if (!gst_mf_transform_set_device_manager (self->transform, + self->device_manager)) { + GST_ERROR_OBJECT (self, "Couldn't set device manager"); + return FALSE; + } else { + GST_DEBUG_OBJECT (self, "set device manager done"); + } + } + hr = MFCreateMediaType (out_type.GetAddressOf ()); if (!gst_mf_result (hr)) return FALSE; @@ -294,6 +422,84 @@ gst_mf_video_enc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state) return FALSE; } +#if GST_MF_HAVE_D3D11 + if (self->mf_allocator) { + self->mf_allocator->UninitializeSampleAllocator (); + self->mf_allocator->Release (); + self->mf_allocator = NULL; + } + + /* Check whether upstream is d3d11 element */ + if (state->caps) { + GstCapsFeatures *features; + ComPtr allocator; + + features = gst_caps_get_features (state->caps, 0); + + if (features && + gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + GST_DEBUG_OBJECT (self, "found D3D11 memory feature"); + + hr = MFCreateVideoSampleAllocatorEx (IID_PPV_ARGS (&allocator)); + if (!gst_mf_result (hr)) + GST_WARNING_OBJECT (self, "IMFVideoSampleAllocatorEx interface is unavailable"); + } + + if (allocator) { + do { + ComPtr attr; + + hr = MFCreateAttributes (&attr, 4); + if (!gst_mf_result (hr)) + break; + + /* Only one buffer per sample + * (multiple sample is usually for multi-view things) */ + hr = attr->SetUINT32 (GST_GUID_MF_SA_BUFFERS_PER_SAMPLE, 1); + if (!gst_mf_result (hr)) + break; + + hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_USAGE, D3D11_USAGE_DEFAULT); + if (!gst_mf_result (hr)) + break; + + /* TODO: Check if we need to use keyed-mutex */ + hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_SHARED_WITHOUT_MUTEX, TRUE); + if (!gst_mf_result (hr)) + break; + + hr = attr->SetUINT32 (GST_GUID_MF_SA_D3D11_BINDFLAGS, + D3D11_BIND_VIDEO_ENCODER); + if (!gst_mf_result (hr)) + break; + + hr = allocator->SetDirectXManager (self->device_manager); + if (!gst_mf_result (hr)) + break; + + hr = allocator->InitializeSampleAllocatorEx ( + /* min samples, since we are running on async mode, + * at least 2 samples would be required */ + 2, + /* max samples, why 16 + 2? it's just magic number + * (H264 max dpb size 16 + our min sample size 2) */ + 16 + 2, + attr.Get (), + in_type.Get () + ); + + if (!gst_mf_result (hr)) + break; + + GST_DEBUG_OBJECT (self, "IMFVideoSampleAllocatorEx is initialized"); + + self->mf_allocator = allocator.Detach (); + } while (0); + } + } +#endif + return TRUE; } @@ -397,89 +603,13 @@ typedef struct static gboolean gst_mf_video_enc_process_input (GstMFVideoEnc * self, - GstVideoCodecFrame * frame) + GstVideoCodecFrame * frame, IMFSample * sample) { GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (self); HRESULT hr; - ComPtr sample; - ComPtr media_buffer; - ComPtr video_buffer; - GstVideoInfo *info = &self->input_state->info; - gint i, j; - GstVideoFrame *vframe = NULL; gboolean unset_force_keyframe = FALSE; GstMFVideoEncFrameData *frame_data = NULL; - BYTE *data = NULL; - gboolean need_copy; - gboolean res = FALSE; - - vframe = g_new0 (GstVideoFrame, 1); - - if (!gst_video_frame_map (vframe, info, frame->input_buffer, GST_MAP_READ)) { - GST_ERROR_OBJECT (self, "Couldn't map input frame"); - g_free (vframe); - return FALSE; - } - - hr = MFCreateSample (&sample); - if (!gst_mf_result (hr)) - goto error; - - /* Check if we can forward this memory to Media Foundation without copy */ - need_copy = gst_mf_video_enc_frame_needs_copy (vframe); - if (need_copy) { - GST_TRACE_OBJECT (self, "Copy input buffer into Media Foundation memory"); - hr = MFCreateMemoryBuffer (GST_VIDEO_INFO_SIZE (info), &media_buffer); - } else { - GST_TRACE_OBJECT (self, "Can use input buffer without copy"); - hr = IGstMFVideoBuffer::CreateInstanceWrapped (&vframe->info, - (BYTE *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0), - GST_VIDEO_INFO_SIZE (&vframe->info), &media_buffer); - } - - if (!gst_mf_result (hr)) - goto error; - - if (!need_copy) { - hr = media_buffer.As (&video_buffer); - if (!gst_mf_result (hr)) - goto error; - } else { - hr = media_buffer->Lock (&data, NULL, NULL); - if (!gst_mf_result (hr)) - goto error; - - for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) { - guint8 *src, *dst; - gint src_stride, dst_stride; - gint width; - - src = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, i); - dst = data + GST_VIDEO_INFO_PLANE_OFFSET (info, i); - - src_stride = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, i); - dst_stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i); - - width = GST_VIDEO_INFO_COMP_WIDTH (info, i) - * GST_VIDEO_INFO_COMP_PSTRIDE (info, i); - - for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (info, i); j++) { - memcpy (dst, src, width); - src += src_stride; - dst += dst_stride; - } - } - - media_buffer->Unlock (); - } - - hr = media_buffer->SetCurrentLength (GST_VIDEO_INFO_SIZE (info)); - if (!gst_mf_result (hr)) - goto error; - - hr = sample->AddBuffer (media_buffer.Get ()); - if (!gst_mf_result (hr)) - goto error; + gboolean res; frame_data = g_new0 (GstMFVideoEncFrameData, 1); frame_data->mf_pts = frame->pts / 100; @@ -489,12 +619,12 @@ gst_mf_video_enc_process_input (GstMFVideoEnc * self, hr = sample->SetSampleTime (frame_data->mf_pts); if (!gst_mf_result (hr)) - goto error; + return FALSE; hr = sample->SetSampleDuration ( GST_CLOCK_TIME_IS_VALID (frame->duration) ? frame->duration / 100 : 0); if (!gst_mf_result (hr)) - goto error; + return FALSE; if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (frame)) { if (klass->device_caps.force_keyframe) { @@ -506,19 +636,6 @@ gst_mf_video_enc_process_input (GstMFVideoEnc * self, } } - if (!need_copy) { - /* IGstMFVideoBuffer will hold GstVideoFrame (+ GstBuffer), then it will be - * cleared when it's no more referenced by Media Foundation internals */ - hr = video_buffer->SetUserData ((gpointer) vframe, - (GDestroyNotify) gst_mf_video_buffer_free); - if (!gst_mf_result (hr)) - goto error; - } else { - gst_video_frame_unmap (vframe); - g_free (vframe); - vframe = NULL; - } - /* Unlock temporary so that we can output frame from Media Foundation's * worker thread. * While we are processing input, MFT might notify @@ -527,7 +644,7 @@ gst_mf_video_enc_process_input (GstMFVideoEnc * self, * not from streaming (this) thread */ if (self->async_mft) GST_VIDEO_ENCODER_STREAM_UNLOCK (self); - res = gst_mf_transform_process_input (self->transform, sample.Get ()); + res = gst_mf_transform_process_input (self->transform, sample); if (self->async_mft) GST_VIDEO_ENCODER_STREAM_LOCK (self); @@ -538,18 +655,10 @@ gst_mf_video_enc_process_input (GstMFVideoEnc * self, if (!res) { GST_ERROR_OBJECT (self, "Failed to process input"); - goto error; + return FALSE; } return TRUE; - -error: - if (vframe) { - gst_video_frame_unmap (vframe); - g_free (vframe); - } - - return FALSE; } static GstVideoCodecFrame * @@ -697,14 +806,286 @@ gst_mf_video_enc_process_output (GstMFVideoEnc * self) return self->last_ret; } +static gboolean +gst_mf_video_enc_create_input_sample (GstMFVideoEnc * self, + GstVideoCodecFrame * frame, IMFSample ** sample) +{ + HRESULT hr; + ComPtr new_sample; + ComPtr media_buffer; + ComPtr video_buffer; + GstVideoInfo *info = &self->input_state->info; + gint i, j; + GstVideoFrame *vframe = NULL; + BYTE *data = NULL; + gboolean need_copy; + + vframe = g_new0 (GstVideoFrame, 1); + + if (!gst_video_frame_map (vframe, info, frame->input_buffer, GST_MAP_READ)) { + GST_ERROR_OBJECT (self, "Couldn't map input frame"); + g_free (vframe); + return FALSE; + } + + hr = MFCreateSample (&new_sample); + if (!gst_mf_result (hr)) + goto error; + + /* Check if we can forward this memory to Media Foundation without copy */ + need_copy = gst_mf_video_enc_frame_needs_copy (vframe); + if (need_copy) { + GST_TRACE_OBJECT (self, "Copy input buffer into Media Foundation memory"); + hr = MFCreateMemoryBuffer (GST_VIDEO_INFO_SIZE (info), &media_buffer); + } else { + GST_TRACE_OBJECT (self, "Can use input buffer without copy"); + hr = IGstMFVideoBuffer::CreateInstanceWrapped (&vframe->info, + (BYTE *) GST_VIDEO_FRAME_PLANE_DATA (vframe, 0), + GST_VIDEO_INFO_SIZE (&vframe->info), &media_buffer); + } + + if (!gst_mf_result (hr)) + goto error; + + if (!need_copy) { + hr = media_buffer.As (&video_buffer); + if (!gst_mf_result (hr)) + goto error; + } else { + hr = media_buffer->Lock (&data, NULL, NULL); + if (!gst_mf_result (hr)) + goto error; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) { + guint8 *src, *dst; + gint src_stride, dst_stride; + gint width; + + src = (guint8 *) GST_VIDEO_FRAME_PLANE_DATA (vframe, i); + dst = data + GST_VIDEO_INFO_PLANE_OFFSET (info, i); + + src_stride = GST_VIDEO_FRAME_PLANE_STRIDE (vframe, i); + dst_stride = GST_VIDEO_INFO_PLANE_STRIDE (info, i); + + width = GST_VIDEO_INFO_COMP_WIDTH (info, i) + * GST_VIDEO_INFO_COMP_PSTRIDE (info, i); + + for (j = 0; j < GST_VIDEO_INFO_COMP_HEIGHT (info, i); j++) { + memcpy (dst, src, width); + src += src_stride; + dst += dst_stride; + } + } + + media_buffer->Unlock (); + } + + hr = media_buffer->SetCurrentLength (GST_VIDEO_INFO_SIZE (info)); + if (!gst_mf_result (hr)) + goto error; + + hr = new_sample->AddBuffer (media_buffer.Get ()); + if (!gst_mf_result (hr)) + goto error; + + if (!need_copy) { + /* IGstMFVideoBuffer will hold GstVideoFrame (+ GstBuffer), then it will be + * cleared when it's no more referenced by Media Foundation internals */ + hr = video_buffer->SetUserData ((gpointer) vframe, + (GDestroyNotify) gst_mf_video_buffer_free); + if (!gst_mf_result (hr)) + goto error; + } else { + gst_video_frame_unmap (vframe); + g_free (vframe); + vframe = NULL; + } + + *sample = new_sample.Detach (); + + return TRUE; + +error: + if (vframe) { + gst_video_frame_unmap (vframe); + g_free (vframe); + } + + return FALSE; +} + +#if GST_MF_HAVE_D3D11 +static gboolean +gst_mf_video_enc_create_input_sample_d3d11 (GstMFVideoEnc * self, + GstVideoCodecFrame * frame, IMFSample ** sample) +{ + HRESULT hr; + ComPtr new_sample; + ComPtr mf_buffer; + ComPtr dxgi_buffer; + ComPtr mf_texture; + ComPtr dxgi_resource; + ComPtr shared_texture; + ComPtr query; + D3D11_QUERY_DESC query_desc; + BOOL sync_done = FALSE; + HANDLE shared_handle; + GstMemory *mem; + GstD3D11Memory *dmem; + ID3D11Texture2D *texture; + ID3D11Device *device_handle; + ID3D11DeviceContext *context_handle; + GstMapInfo info; + D3D11_BOX src_box = { 0, }; + D3D11_TEXTURE2D_DESC dst_desc, src_desc; + guint subidx; + + if (!self->mf_allocator) { + GST_WARNING_OBJECT (self, "IMFVideoSampleAllocatorEx was configured"); + return FALSE; + } + + mem = gst_buffer_peek_memory (frame->input_buffer, 0); + if (!gst_is_d3d11_memory (mem)) { + GST_WARNING_OBJECT (self, "Non-d3d11 memory"); + return FALSE; + } + + dmem = (GstD3D11Memory * ) mem; + device_handle = gst_d3d11_device_get_device_handle (dmem->device); + context_handle = gst_d3d11_device_get_device_context_handle (dmem->device); + + /* 1) Allocate new encoding surface */ + hr = self->mf_allocator->AllocateSample (&new_sample); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, + "Couldn't allocate new sample via IMFVideoSampleAllocatorEx"); + return FALSE; + } + + hr = new_sample->GetBufferByIndex (0, &mf_buffer); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, "Couldn't get IMFMediaBuffer from sample"); + return FALSE; + } + + hr = mf_buffer.As (&dxgi_buffer); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, "Couldn't get IMFDXGIBuffer from IMFMediaBuffer"); + return FALSE; + } + + hr = dxgi_buffer->GetResource (IID_PPV_ARGS (&mf_texture)); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, + "Couldn't get ID3D11Texture2D from IMFDXGIBuffer"); + return FALSE; + } + + hr = mf_texture.As (&dxgi_resource); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, + "Couldn't get IDXGIResource from ID3D11Texture2D"); + return FALSE; + } + + hr = dxgi_resource->GetSharedHandle (&shared_handle); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, + "Couldn't get shared handle from IDXGIResource"); + return FALSE; + } + + /* Allocation succeeded. Now open shared texture to access it from + * other device */ + hr = device_handle->OpenSharedResource (shared_handle, + IID_PPV_ARGS (&shared_texture)); + if (!gst_mf_result (hr)) { + GST_WARNING_OBJECT (self, "Couldn't open shared resource"); + return FALSE; + } + + /* 2) Copy upstream texture to mf's texture */ + /* Map memory so that ensure pending upload from staging texture */ + if (!gst_memory_map (mem, &info, (GstMapFlags) (GST_MAP_READ | GST_MAP_D3D11))) { + GST_ERROR_OBJECT (self, "Couldn't map d3d11 memory"); + return FALSE; + } + + texture = (ID3D11Texture2D *) info.data; + texture->GetDesc (&src_desc); + shared_texture->GetDesc (&dst_desc); + subidx = gst_d3d11_memory_get_subresource_index (dmem); + + /* src/dst texture size might be different if padding was used. + * select smaller size */ + src_box.left = 0; + src_box.top = 0; + src_box.front = 0; + src_box.back = 1; + src_box.right = MIN (src_desc.Width, dst_desc.Width); + src_box.bottom = MIN (src_desc.Height, dst_desc.Height); + + /* CopySubresourceRegion() might not be able to guarantee + * copying. To ensure it, we will make use of d3d11 query */ + query_desc.Query = D3D11_QUERY_EVENT; + query_desc.MiscFlags = 0; + + hr = device_handle->CreateQuery (&query_desc, &query); + if (!gst_d3d11_result (hr, dmem->device)) { + GST_ERROR_OBJECT (self, "Couldn't Create event query"); + return FALSE; + } + + gst_d3d11_device_lock (dmem->device); + context_handle->CopySubresourceRegion (shared_texture.Get (), 0, 0, 0, 0, + texture, subidx, &src_box); + context_handle->End (query.Get()); + + /* Wait until all issued GPU commands are finished */ + do { + context_handle->GetData (query.Get(), &sync_done, sizeof (BOOL), 0); + } while (!sync_done && (hr == S_OK || hr == S_FALSE)); + + if (!gst_d3d11_result (hr, dmem->device)) { + GST_ERROR_OBJECT (self, "Couldn't sync GPU operation"); + gst_d3d11_device_unlock (dmem->device); + gst_memory_unmap (mem, &info); + + return FALSE; + } + + gst_d3d11_device_unlock (dmem->device); + gst_memory_unmap (mem, &info); + + *sample = new_sample.Detach (); + + return TRUE; +} +#endif + static GstFlowReturn gst_mf_video_enc_handle_frame (GstVideoEncoder * enc, GstVideoCodecFrame * frame) { GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc); GstFlowReturn ret = GST_FLOW_OK; + ComPtr sample; - if (!gst_mf_video_enc_process_input (self, frame)) { +#if GST_MF_HAVE_D3D11 + if (self->mf_allocator && + !gst_mf_video_enc_create_input_sample_d3d11 (self, frame, &sample)) { + GST_WARNING_OBJECT (self, "Failed to create IMFSample for D3D11"); + sample = nullptr; + } +#endif + + if (!sample && !gst_mf_video_enc_create_input_sample (self, frame, &sample)) { + GST_ERROR_OBJECT (self, "Failed to create IMFSample"); + return GST_FLOW_ERROR; + } + + if (!gst_mf_video_enc_process_input (self, frame, sample.Get ())) { GST_ERROR_OBJECT (self, "Failed to process input"); ret = GST_FLOW_ERROR; goto done; @@ -778,6 +1159,153 @@ gst_mf_video_enc_flush (GstVideoEncoder * enc) return TRUE; } + +static gboolean +gst_mf_video_enc_propose_allocation (GstVideoEncoder * enc, + GstQuery * query) +{ +#if GST_MF_HAVE_D3D11 + GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc); + GstVideoInfo info; + GstBufferPool *pool = NULL; + GstCaps *caps; + guint size; + GstD3D11Device *device = self->other_d3d11_device; + + gst_query_parse_allocation (query, &caps, NULL); + + if (caps == NULL) + return FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; + + if (gst_query_get_n_allocation_pools (query) == 0) { + GstCapsFeatures *features; + GstStructure *config; + gboolean is_d3d11 = FALSE; + + features = gst_caps_get_features (caps, 0); + + if (features && gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + GST_DEBUG_OBJECT (self, "Allocation caps supports d3d11 memory"); + pool = gst_d3d11_buffer_pool_new (device); + is_d3d11 = TRUE; + } else { + pool = gst_video_buffer_pool_new (); + } + + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + + /* d3d11 pool does not support video alignment */ + if (!is_d3d11) { + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + } else { + GstD3D11AllocationParams *d3d11_params; + guint misc_flags = 0; + gboolean is_hardware = FALSE; + gint i; + + g_object_get (device, "hardware", &is_hardware, NULL); + + /* In case of hardware, set D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX flag + * so that it can be shared with other d3d11 devices */ + if (is_hardware) + misc_flags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX; + + d3d11_params = + gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!d3d11_params) { + d3d11_params = gst_d3d11_allocation_params_new (device, &info, + (GstD3D11AllocationFlags) 0, 0); + } else { + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) + d3d11_params->desc[i].MiscFlags |= misc_flags; + } + + gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params); + gst_d3d11_allocation_params_free (d3d11_params); + } + + size = GST_VIDEO_INFO_SIZE (&info); + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + + /* d3d11 buffer pool might update buffer size by self */ + if (is_d3d11) + size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; + + gst_query_add_allocation_pool (query, pool, size, 0, 0); + gst_object_unref (pool); + } + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return TRUE; + + /* ERRORS */ +config_failed: + { + GST_ERROR_OBJECT (self, "failed to set config"); + gst_object_unref (pool); + return FALSE; + } + +#else + return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (enc, + query); +#endif +} + +static gboolean +gst_mf_video_enc_sink_query (GstVideoEncoder * enc, GstQuery *query) +{ +#if GST_MF_HAVE_D3D11 + GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (self), + query, self->other_d3d11_device)) { + return TRUE; + } + break; + default: + break; + } +#endif + + return GST_VIDEO_ENCODER_CLASS (parent_class)->sink_query (enc, query); +} + +static gboolean +gst_mf_video_enc_src_query (GstVideoEncoder * enc, GstQuery * query) +{ +#if GST_MF_HAVE_D3D11 + GstMFVideoEnc *self = GST_MF_VIDEO_ENC (enc); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (self), + query, self->other_d3d11_device)) { + return TRUE; + } + break; + default: + break; + } +#endif + + return GST_VIDEO_ENCODER_CLASS (parent_class)->src_query (enc, query); +} + static HRESULT gst_mf_video_on_new_sample (GstMFTransform * object, IMFSample * sample, GstMFVideoEnc * self) @@ -801,8 +1329,8 @@ typedef struct static void gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, - GstMFVideoEncDeviceCaps * device_caps, GstCaps ** sink_template, - GstCaps ** src_template) + GstObject * d3d11_device, GstMFVideoEncDeviceCaps * device_caps, + GstCaps ** sink_template, GstCaps ** src_template) { HRESULT hr; MFT_REGISTER_TYPE_INFO *infos; @@ -810,9 +1338,12 @@ gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, gint i; GstCaps *src_caps = NULL; GstCaps *sink_caps = NULL; + GstCaps *d3d11_caps = NULL; GValue *supported_formats = NULL; GValue *profiles = NULL; gboolean have_I420 = FALSE; + gboolean have_NV12 = FALSE; + gboolean d3d11_aware = FALSE; gchar *device_name = NULL; IMFActivate *activate; IMFTransform *encoder; @@ -885,12 +1416,19 @@ gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, g_value_init (supported_formats, GST_TYPE_LIST); } - /* media foundation has duplicated formats IYUV and I420 */ - if (format == GST_VIDEO_FORMAT_I420) { - if (have_I420) - continue; + switch (format) { + /* media foundation has duplicated formats IYUV and I420 */ + case GST_VIDEO_FORMAT_I420: + if (have_I420) + continue; - have_I420 = TRUE; + have_I420 = TRUE; + break; + case GST_VIDEO_FORMAT_NV12: + have_NV12 = TRUE; + break; + default: + break; } g_value_init (&val, G_TYPE_STRING); @@ -990,10 +1528,6 @@ gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, } sink_caps = gst_caps_new_empty_simple ("video/x-raw"); - gst_caps_set_value (sink_caps, "format", supported_formats); - g_value_unset (supported_formats); - g_free (supported_formats); - /* FIXME: don't hardcode max resolution, but MF doesn't provide * API for querying supported max resolution... */ gst_caps_set_simple (sink_caps, @@ -1003,6 +1537,38 @@ gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, "width", GST_TYPE_INT_RANGE, 64, 8192, "height", GST_TYPE_INT_RANGE, 64, 8192, NULL); +#if GST_MF_HAVE_D3D11 + /* Check whether this MFT can support D3D11 */ + if (d3d11_device && have_NV12) { + g_object_get (transform, "d3d11-aware", &d3d11_aware, NULL); + GST_DEBUG_OBJECT (transform, "d3d11 aware %d", d3d11_aware); + } + + /* TODO: can we use non NV12 format for d3d11 with D3D11_BIND_VIDEO_ENCODER + * flag? */ + if (d3d11_device && have_NV12 && d3d11_aware) { + guint adapter = 0; + + g_object_get (d3d11_device, "adapter", &adapter, NULL); + + d3d11_caps = gst_caps_copy (sink_caps); + + gst_caps_set_simple (d3d11_caps, + "format", G_TYPE_STRING, "NV12", NULL); + gst_caps_set_features_simple (d3d11_caps, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)); + device_caps->d3d11_aware = TRUE; + device_caps->adapter = adapter; + } +#endif + + gst_caps_set_value (sink_caps, "format", supported_formats); + g_value_unset (supported_formats); + g_free (supported_formats); + + if (d3d11_caps) + gst_caps_append (sink_caps, d3d11_caps); + *sink_template = sink_caps; *src_template = src_caps; @@ -1058,12 +1624,13 @@ gst_mf_video_enc_enum_internal (GstMFTransform * transform, GUID &subtype, static GstMFTransform * gst_mf_video_enc_enum (guint enum_flags, GUID * subtype, guint device_index, - GstMFVideoEncDeviceCaps * device_caps, GstCaps ** sink_template, - GstCaps ** src_template) + GstMFVideoEncDeviceCaps * device_caps, GstObject * d3d11_device, + GstCaps ** sink_template, GstCaps ** src_template) { GstMFTransformEnumParams enum_params = { 0, }; MFT_REGISTER_TYPE_INFO output_type; GstMFTransform *transform; + gint64 adapter_luid = 0; *sink_template = NULL; *src_template = NULL; @@ -1077,6 +1644,14 @@ gst_mf_video_enc_enum (guint enum_flags, GUID * subtype, guint device_index, return NULL; } + if (d3d11_device) { + g_object_get (d3d11_device, "adapter-luid", &adapter_luid, NULL); + if (!adapter_luid) { + GST_ERROR ("Couldn't get adapter LUID"); + return NULL; + } + } + output_type.guidMajorType = MFMediaType_Video; output_type.guidSubtype = *subtype; @@ -1084,13 +1659,14 @@ gst_mf_video_enc_enum (guint enum_flags, GUID * subtype, guint device_index, enum_params.output_typeinfo = &output_type; enum_params.device_index = device_index; enum_params.enum_flags = enum_flags; + enum_params.adapter_luid = adapter_luid; transform = gst_mf_transform_new (&enum_params); if (!transform) return NULL; gst_mf_video_enc_enum_internal (transform, output_type.guidSubtype, - device_caps, sink_template, src_template); + d3d11_device, device_caps, sink_template, src_template); return transform; } @@ -1179,7 +1755,7 @@ gst_mf_video_enc_register_internal (GstPlugin * plugin, guint rank, void gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype, - GTypeInfo * type_info) + GTypeInfo * type_info, GList * d3d11_device) { GstMFTransform *transform = NULL; GstCaps *sink_template = NULL; @@ -1192,27 +1768,52 @@ gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype, enum_flags = (MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_ASYNCMFT | MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY); - /* AMD seems to be able to support up to 12 GPUs */ - for (i = 0; i < 12 ; i++) { - transform = gst_mf_video_enc_enum (enum_flags, subtype, i, &device_caps, - &sink_template, &src_template); + if (d3d11_device) { + GList *iter; + for (iter = d3d11_device; iter; iter = g_list_next (iter)) { + GstObject *device = (GstObject *) iter->data; - /* No more MFT to enumerate */ - if (!transform) - break; + transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps, + device, &sink_template, &src_template); - /* Failed to open MFT */ - if (!sink_template) { + /* No more MFT to enumerate */ + if (!transform) + break; + + /* Failed to open MFT */ + if (!sink_template) { + gst_clear_object (&transform); + continue; + } + + gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info, + &device_caps, enum_flags, 0, transform, sink_template, src_template); gst_clear_object (&transform); - continue; + gst_clear_caps (&sink_template); + gst_clear_caps (&src_template); } + } else { + /* AMD seems to be able to support up to 12 GPUs */ + for (i = 0; i < 12; i++) { + transform = gst_mf_video_enc_enum (enum_flags, subtype, i, &device_caps, + NULL, &sink_template, &src_template); - gst_mf_video_enc_register_internal (plugin, rank, subtype, - type_info, &device_caps, enum_flags, i, transform, - sink_template, src_template); - gst_clear_object (&transform); - gst_clear_caps (&sink_template); - gst_clear_caps (&src_template); + /* No more MFT to enumerate */ + if (!transform) + break; + + /* Failed to open MFT */ + if (!sink_template) { + gst_clear_object (&transform); + continue; + } + + gst_mf_video_enc_register_internal (plugin, rank, subtype, type_info, + &device_caps, enum_flags, i, transform, sink_template, src_template); + gst_clear_object (&transform); + gst_clear_caps (&sink_template); + gst_clear_caps (&src_template); + } } /* register software encoders */ @@ -1220,7 +1821,7 @@ gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype, MFT_ENUM_FLAG_SORTANDFILTER | MFT_ENUM_FLAG_SORTANDFILTER_APPROVED_ONLY); transform = gst_mf_video_enc_enum (enum_flags, subtype, 0, &device_caps, - &sink_template, &src_template); + NULL, &sink_template, &src_template); if (!transform) goto done; diff --git a/sys/mediafoundation/gstmfvideoenc.h b/sys/mediafoundation/gstmfvideoenc.h index afc2e66c91..4f30ae22b3 100644 --- a/sys/mediafoundation/gstmfvideoenc.h +++ b/sys/mediafoundation/gstmfvideoenc.h @@ -21,11 +21,17 @@ #ifndef __GST_MF_VIDEO_ENC_H__ #define __GST_MF_VIDEO_ENC_H__ +#include "gstmfconfig.h" + #include #include #include "gstmfutils.h" #include "gstmftransform.h" +#if GST_MF_HAVE_D3D11 +#include +#endif + G_BEGIN_DECLS #define GST_TYPE_MF_VIDEO_ENC (gst_mf_video_enc_get_type()) @@ -66,6 +72,11 @@ struct _GstMFVideoEncDeviceCaps gboolean max_num_ref; /* AVEncVideoMaxNumRefFrame */ guint max_num_ref_high; guint max_num_ref_low; + + /* TRUE if MFT support d3d11 and also we can use d3d11 interop */ + gboolean d3d11_aware; + /* DXGI adapter index to use, ignored if d3d11-unaware */ + guint adapter; }; struct _GstMFVideoEncClassData @@ -88,6 +99,15 @@ struct _GstMFVideoEnc GstFlowReturn last_ret; GstVideoCodecState *input_state; + +#if GST_MF_HAVE_D3D11 + /* For D3D11 interop. */ + GstD3D11Device *other_d3d11_device; + GstD3D11Device *d3d11_device; + IMFDXGIDeviceManager *device_manager; + UINT reset_token; + IMFVideoSampleAllocatorEx *mf_allocator; +#endif }; struct _GstMFVideoEncClass @@ -114,7 +134,8 @@ GType gst_mf_video_enc_get_type (void); void gst_mf_video_enc_register (GstPlugin * plugin, guint rank, GUID * subtype, - GTypeInfo * type_info); + GTypeInfo * type_info, + GList * d3d11_device); G_END_DECLS diff --git a/sys/mediafoundation/gstmfvp9enc.cpp b/sys/mediafoundation/gstmfvp9enc.cpp index 673326cb13..7ff9d65c32 100644 --- a/sys/mediafoundation/gstmfvp9enc.cpp +++ b/sys/mediafoundation/gstmfvp9enc.cpp @@ -105,6 +105,8 @@ enum PROP_THREADS, PROP_CONTENT_TYPE, PROP_LOW_LATENCY, + PROP_D3D11_AWARE, + PROP_ADAPTER, }; #define DEFAULT_BITRATE (2 * 1024) @@ -250,6 +252,21 @@ gst_mf_vp9_enc_class_init (GstMFVP9EncClass * klass, gpointer data) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } + g_object_class_install_property (gobject_class, PROP_D3D11_AWARE, + g_param_spec_boolean ("d3d11-aware", "D3D11 Aware", + "Whether device can support Direct3D11 interop", + device_caps->d3d11_aware, + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + if (device_caps->d3d11_aware) { + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_uint ("adapter", "Adapter", + "DXGI Adapter index for creating device", + 0, G_MAXUINT32, device_caps->adapter, + (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE | + G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + } + long_name = g_strdup_printf ("Media Foundation %s", cdata->device_name); classification = g_strdup_printf ("Codec/Encoder/Video%s", (cdata->enum_flags & MFT_ENUM_FLAG_HARDWARE) == MFT_ENUM_FLAG_HARDWARE ? @@ -300,6 +317,7 @@ gst_mf_vp9_enc_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstMFVP9Enc *self = (GstMFVP9Enc *) (object); + GstMFVideoEncClass *klass = GST_MF_VIDEO_ENC_GET_CLASS (object); switch (prop_id) { case PROP_BITRATE: @@ -326,6 +344,12 @@ gst_mf_vp9_enc_get_property (GObject * object, guint prop_id, case PROP_LOW_LATENCY: g_value_set_boolean (value, self->low_latency); break; + case PROP_D3D11_AWARE: + g_value_set_boolean (value, klass->device_caps.d3d11_aware); + break; + case PROP_ADAPTER: + g_value_set_uint (value, klass->device_caps.adapter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -529,7 +553,8 @@ gst_mf_vp9_enc_set_src_caps (GstMFVideoEnc * mfenc, } void -gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank) +gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank, + GList * d3d11_device) { GTypeInfo type_info = { sizeof (GstMFVP9EncClass), @@ -546,5 +571,5 @@ gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, guint rank) GST_DEBUG_CATEGORY_INIT (gst_mf_vp9_enc_debug, "mfvp9enc", 0, "mfvp9enc"); - gst_mf_video_enc_register (plugin, rank, &subtype, &type_info); + gst_mf_video_enc_register (plugin, rank, &subtype, &type_info, d3d11_device); } \ No newline at end of file diff --git a/sys/mediafoundation/gstmfvp9enc.h b/sys/mediafoundation/gstmfvp9enc.h index 2fc7ccead7..412efee919 100644 --- a/sys/mediafoundation/gstmfvp9enc.h +++ b/sys/mediafoundation/gstmfvp9enc.h @@ -25,7 +25,8 @@ G_BEGIN_DECLS void gst_mf_vp9_enc_plugin_init (GstPlugin * plugin, - guint rank); + guint rank, + GList * d3d11_device); G_END_DECLS diff --git a/sys/mediafoundation/meson.build b/sys/mediafoundation/meson.build index 3077791f42..0747d637f9 100644 --- a/sys/mediafoundation/meson.build +++ b/sys/mediafoundation/meson.build @@ -37,8 +37,11 @@ mf_header_deps = [ winapi_desktop = false winapi_app = false have_capture_engine = false +have_mf_d3d11 = false mf_lib_deps = [] mf_config = configuration_data() +extra_c_args = ['-DCOBJMACROS'] +extra_cpp_args = [] mf_option = get_option('mediafoundation') if host_system != 'windows' or mf_option.disabled() @@ -120,10 +123,22 @@ endif if winapi_desktop mf_sources += mf_desktop_sources + # We need d3d11_4.h header for querying "ExtendedNV12SharedTextureSupported" + # Since MFTEnum2 is desktop only we don't support d3d11 interop for UWP build + # And because MFTEnum2 is Windows 10 API, we will load MFTEnum2 symbol + # by using g_module_open() so that keep supporting old OS versions + if gstd3d11_dep.found() and cc.has_header('d3d11_4.h') + have_mf_d3d11 = true + mf_lib_deps += [gstd3d11_dep, gmodule_dep] + extra_c_args += ['-DGST_USE_UNSTABLE_API'] + extra_cpp_args += ['-DGST_USE_UNSTABLE_API'] + message ('Enable D3D11 interop for MediaFoundation plugin') + endif endif mf_config.set10('GST_MF_WINAPI_APP', winapi_app) mf_config.set10('GST_MF_WINAPI_DESKTOP', winapi_desktop) +mf_config.set10('GST_MF_HAVE_D3D11', have_mf_d3d11) configure_file( output: 'gstmfconfig.h', @@ -132,8 +147,8 @@ configure_file( gstmediafoundation = library('gstmediafoundation', mf_sources, - c_args : gst_plugins_bad_args + ['-DCOBJMACROS'], - cpp_args : gst_plugins_bad_args, + c_args : gst_plugins_bad_args + extra_c_args, + cpp_args : gst_plugins_bad_args + extra_cpp_args, include_directories : [configinc], dependencies : [gstbase_dep, gstvideo_dep, gstaudio_dep, gstpbutils_dep] + mf_lib_deps, install : true, diff --git a/sys/mediafoundation/plugin.c b/sys/mediafoundation/plugin.c index e3d7a51186..7e6ad46a87 100644 --- a/sys/mediafoundation/plugin.c +++ b/sys/mediafoundation/plugin.c @@ -27,6 +27,7 @@ #include #include +#include #include "gstmfvideosrc.h" #include "gstmfdevice.h" #include "gstmfutils.h" @@ -35,6 +36,11 @@ #include "gstmfvp9enc.h" #include "gstmfaacenc.h" #include "gstmfmp3enc.h" +#include "gstmftransform.h" + +#if GST_MF_HAVE_D3D11 +#include +#endif GST_DEBUG_CATEGORY (gst_mf_debug); GST_DEBUG_CATEGORY (gst_mf_utils_debug); @@ -51,11 +57,103 @@ plugin_deinit (gpointer data) MFShutdown (); } +#if GST_MF_HAVE_D3D11 +static GList * +get_d3d11_devices (void) +{ + GList *ret = NULL; + guint i; + HRESULT hr; + IMFVideoSampleAllocatorEx *allocator = NULL; + + /* Check whether we can use IMFVideoSampleAllocatorEx interface */ + hr = MFCreateVideoSampleAllocatorEx (&IID_IMFVideoSampleAllocatorEx, + &allocator); + if (!gst_mf_result (hr)) { + GST_DEBUG ("IMFVideoSampleAllocatorEx interface is unavailable"); + return NULL; + } else { + IMFVideoSampleAllocatorEx_Release (allocator); + } + + /* AMD seems supporting up to 12 cards, and 8 for NVIDIA */ + for (i = 0; i < 12; i++) { + GstD3D11Device *device; + gboolean is_hardware = FALSE; + const GstD3D11Format *d3d11_format; + ID3D11Device *device_handle; + D3D11_FEATURE_DATA_D3D11_OPTIONS4 options = { 0, }; + UINT supported = 0; + + device = gst_d3d11_device_new (i, D3D11_CREATE_DEVICE_VIDEO_SUPPORT); + + if (!device) + break; + + g_object_get (device, "hardware", &is_hardware, NULL); + + if (!is_hardware) { + GST_DEBUG_OBJECT (device, "Given d3d11 device is not for hardware"); + gst_object_unref (device); + continue; + } + + /* device can support NV12 format? */ + d3d11_format = + gst_d3d11_device_format_from_gst (device, GST_VIDEO_FORMAT_NV12); + if (!d3d11_format || d3d11_format->dxgi_format != DXGI_FORMAT_NV12) { + GST_DEBUG_OBJECT (device, + "Given d3d11 device cannot support NV12 format"); + gst_object_unref (device); + continue; + } + + /* device can support ExtendedNV12SharedTextureSupported? + * + * NOTE: we will make use of per encoder object d3d11 device without + * sharing it in a pipeline because MF needs D3D11_CREATE_DEVICE_VIDEO_SUPPORT + * but the flag doesn't used for the other our use cases. + * So we need texture sharing feature so that we can copy d3d11 texture into + * MF specific texture pool without download texture */ + + device_handle = gst_d3d11_device_get_device_handle (device); + hr = ID3D11Device_CheckFeatureSupport (device_handle, + D3D11_FEATURE_D3D11_OPTIONS4, &options, sizeof (options)); + if (!gst_d3d11_result (hr, device) || + !options.ExtendedNV12SharedTextureSupported) { + GST_DEBUG_OBJECT (device, + "Given d3d11 device cannot support NV12 format for shared texture"); + gst_object_unref (device); + continue; + } + + /* can we bind NV12 texture for encoder? */ + hr = ID3D11Device_CheckFormatSupport (device_handle, + DXGI_FORMAT_NV12, &supported); + + if (!gst_d3d11_result (hr, device)) { + GST_DEBUG_OBJECT (device, "Couldn't query format support"); + gst_object_unref (device); + continue; + } else if ((supported & D3D11_FORMAT_SUPPORT_VIDEO_ENCODER) == 0) { + GST_DEBUG_OBJECT (device, "We cannot bind NV12 format for encoding"); + gst_object_unref (device); + continue; + } + + ret = g_list_append (ret, device); + } + + return ret; +} +#endif + static gboolean plugin_init (GstPlugin * plugin) { HRESULT hr; GstRank rank = GST_RANK_SECONDARY; + GList *device_list = NULL; /** * plugin-mediafoundation: @@ -86,13 +184,25 @@ plugin_init (GstPlugin * plugin) rank = GST_RANK_PRIMARY + 1; #endif + /* FIXME: In order to create MFT for a specific GPU, MFTEnum2() API is + * required API but it's desktop only. + * So, resulting MFT and D3D11 might not be compatible in case of multi-GPU + * environment on UWP. */ +#if GST_MF_HAVE_D3D11 + if (gst_mf_transform_load_library ()) + device_list = get_d3d11_devices (); +#endif + gst_element_register (plugin, "mfvideosrc", rank, GST_TYPE_MF_VIDEO_SRC); gst_device_provider_register (plugin, "mfdeviceprovider", rank, GST_TYPE_MF_DEVICE_PROVIDER); - gst_mf_h264_enc_plugin_init (plugin, GST_RANK_SECONDARY); - gst_mf_h265_enc_plugin_init (plugin, GST_RANK_SECONDARY); - gst_mf_vp9_enc_plugin_init (plugin, GST_RANK_SECONDARY); + gst_mf_h264_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list); + gst_mf_h265_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list); + gst_mf_vp9_enc_plugin_init (plugin, GST_RANK_SECONDARY, device_list); + + if (device_list) + g_list_free_full (device_list, gst_object_unref); gst_mf_aac_enc_plugin_init (plugin, GST_RANK_SECONDARY); gst_mf_mp3_enc_plugin_init (plugin, GST_RANK_SECONDARY);