From d61a248ac8fa1996d553366efd2ab0ebfe46976c Mon Sep 17 00:00:00 2001 From: Tong Wu Date: Sat, 7 May 2022 17:10:34 +0800 Subject: [PATCH] MSDK: Introduce d3d11 device to MSDK context Currently MSDK context does not support d3d11va. Now introduce d3d11va device to MSDK context, making it able to create msdk session with d3d11 device and to easily share with upstream and donwstream. Add environment variable to enable user to choose GPU device in multi-GPU environment. This variable is only valid when there's no context returned by upstream or downstream. Otherwise it will use the device that created by upstream or downstream. Part-of: --- .../gst-plugins-bad/sys/msdk/gstmsdkcontext.c | 204 +++++++++++++++++- .../gst-plugins-bad/sys/msdk/gstmsdkcontext.h | 13 +- .../sys/msdk/gstmsdkcontextutil.c | 91 +++++++- .../sys/msdk/gstmsdkcontextutil.h | 8 +- .../gst-plugins-bad/sys/msdk/gstmsdkdec.c | 13 +- .../gst-plugins-bad/sys/msdk/gstmsdkenc.c | 15 +- .../gst-plugins-bad/sys/msdk/gstmsdkvpp.c | 15 +- 7 files changed, 334 insertions(+), 25 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c index 6e962e1f0a..60cc61d339 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.c @@ -38,6 +38,8 @@ #include #include #include +#else +#include #endif GST_DEBUG_CATEGORY_STATIC (gst_debug_msdkcontext); @@ -56,6 +58,8 @@ struct _GstMsdkContextPrivate GstMsdkContext *parent_context; #ifndef _WIN32 GstVaDisplay *display; +#else + GstD3D11Device *device; #endif }; @@ -201,6 +205,101 @@ failed: return FALSE; } +#else +static GstD3D11Device * +get_device_by_index (IDXGIFactory1 * factory, guint idx) +{ + HRESULT hr; + IDXGIAdapter1 *adapter; + ID3D11Device *device_handle; + ID3D10Multithread *multi_thread; + DXGI_ADAPTER_DESC desc; + GstD3D11Device *device = NULL; + gint64 luid; + + hr = IDXGIFactory1_EnumAdapters1 (factory, idx, &adapter); + if (FAILED (hr)) { + return NULL; + } + + hr = IDXGIAdapter1_GetDesc (adapter, &desc); + if (FAILED (hr)) { + IDXGIAdapter1_Release (adapter); + return NULL; + } + + if (desc.VendorId != 0x8086) { + IDXGIAdapter1_Release (adapter); + return NULL; + } + + luid = gst_d3d11_luid_to_int64 (&desc.AdapterLuid); + device = gst_d3d11_device_new_for_adapter_luid (luid, + D3D11_CREATE_DEVICE_BGRA_SUPPORT); + IDXGIAdapter1_Release (adapter); + + device_handle = gst_d3d11_device_get_device_handle (device); + hr = ID3D11Device_QueryInterface (device_handle, + &IID_ID3D10Multithread, (void **) &multi_thread); + if (FAILED (hr)) { + gst_object_unref (device); + return NULL; + } + + hr = ID3D10Multithread_SetMultithreadProtected (multi_thread, TRUE); + ID3D10Multithread_Release (multi_thread); + + return device; +} + +static gboolean +gst_msdk_context_use_d3d11 (GstMsdkContext * context) +{ + HRESULT hr; + IDXGIFactory1 *factory = NULL; + GstD3D11Device *device = NULL; + ID3D11Device *device_handle; + GstMsdkContextPrivate *priv = context->priv; + mfxStatus status; + guint idx = 0; + gint user_idx = -1; + const gchar *user_choice = g_getenv ("GST_MSDK_DEVICE"); + + hr = CreateDXGIFactory1 (&IID_IDXGIFactory1, (void **) &factory); + if (FAILED (hr)) { + GST_ERROR ("Couldn't create DXGI factory"); + return FALSE; + } + + if (user_choice) { + user_idx = atoi (user_choice); + if (!(device = get_device_by_index (factory, user_idx))) + GST_WARNING + ("Failed to get device by user index, try to pick the first available device"); + } + + /* Pick the first available device */ + while (!device) { + device = get_device_by_index (factory, idx++); + } + + IDXGIFactory1_Release (factory); + device_handle = gst_d3d11_device_get_device_handle (device); + + status = + MFXVideoCORE_SetHandle (priv->session.session, MFX_HANDLE_D3D11_DEVICE, + gst_d3d11_device_get_device_handle (device)); + if (status != MFX_ERR_NONE) { + GST_ERROR ("Setting D3D11VA handle failed (%s)", + msdk_status_to_string (status)); + gst_object_unref (device); + return FALSE; + } + + priv->device = device; + + return TRUE; +} #endif static gboolean @@ -210,12 +309,18 @@ gst_msdk_context_open (GstMsdkContext * context, gboolean hardware, mfxU16 codename; GstMsdkContextPrivate *priv = context->priv; MsdkSession msdk_session; + mfxIMPL impl; priv->job_type = job_type; priv->hardware = hardware; - msdk_session = - msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE); + impl = hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE; + +#ifdef _WIN32 + impl |= MFX_IMPL_VIA_D3D11; +#endif + + msdk_session = msdk_open_session (impl); priv->session = msdk_session; if (!priv->session.session) goto failed; @@ -225,6 +330,11 @@ gst_msdk_context_open (GstMsdkContext * context, gboolean hardware, if (!gst_msdk_context_use_vaapi (context)) goto failed; } +#else + if (hardware) { + if (!gst_msdk_context_use_d3d11 (context)) + goto failed; + } #endif codename = msdk_get_platform_codename (priv->session.session); @@ -281,6 +391,9 @@ gst_msdk_context_finalize (GObject * obj) #ifndef _WIN32 if (priv->display) gst_object_unref (priv->display); +#else + if (priv->device) + gst_object_unref (priv->device); #endif done: @@ -336,6 +449,8 @@ gst_msdk_context_new_with_parent (GstMsdkContext * parent) if (MFX_IMPL_VIA_VAAPI == (0x0f00 & (impl))) handle_type = MFX_HANDLE_VA_DISPLAY; + else if (MFX_IMPL_VIA_D3D11 == (0x0f00 & (impl))) + handle_type = MFX_HANDLE_D3D11_DEVICE; if (handle_type) { status = @@ -396,19 +511,21 @@ gst_msdk_context_new_with_parent (GstMsdkContext * parent) g_list_prepend (parent_priv->child_session_list, priv->session.session); #ifndef _WIN32 priv->display = parent_priv->display; +#else + priv->device = parent_priv->device; #endif priv->parent_context = gst_object_ref (parent); return obj; } +#ifndef _WIN32 GstMsdkContext * gst_msdk_context_new_with_va_display (GstObject * display_obj, gboolean hardware, GstMsdkContextJobType job_type) { GstMsdkContext *obj = NULL; -#ifndef _WIN32 GstMsdkContextPrivate *priv; mfxU16 codename; mfxStatus status; @@ -451,10 +568,71 @@ gst_msdk_context_new_with_va_display (GstObject * display_obj, else GST_WARNING ("Unknown MFX platform"); -#endif - return obj; } +#else +GstMsdkContext * +gst_msdk_context_new_with_d3d11_device (GstD3D11Device * device, + gboolean hardware, GstMsdkContextJobType job_type) +{ + GstMsdkContext *obj = NULL; + GstMsdkContextPrivate *priv; + mfxU16 codename; + mfxStatus status; + ID3D10Multithread *multi_thread; + ID3D11Device *device_handle; + HRESULT hr; + + obj = g_object_new (GST_TYPE_MSDK_CONTEXT, NULL); + + priv = obj->priv; + priv->device = gst_object_ref (device); + + priv->job_type = job_type; + priv->hardware = hardware; + priv->session = + msdk_open_session (hardware ? MFX_IMPL_HARDWARE_ANY : MFX_IMPL_SOFTWARE); + if (!priv->session.session) { + goto failed; + } + + device_handle = gst_d3d11_device_get_device_handle (device); + hr = ID3D11Device_QueryInterface (device_handle, + &IID_ID3D10Multithread, (void **) &multi_thread); + if (FAILED (hr)) { + GST_ERROR ("ID3D10Multithread interface is unavailable"); + goto failed; + } + + hr = ID3D10Multithread_SetMultithreadProtected (multi_thread, TRUE); + ID3D10Multithread_Release (multi_thread); + + if (hardware) { + status = + MFXVideoCORE_SetHandle (priv->session.session, MFX_HANDLE_D3D11_DEVICE, + device_handle); + if (status != MFX_ERR_NONE) { + GST_ERROR ("Setting D3D11VA handle failed (%s)", + msdk_status_to_string (status)); + goto failed; + } + } + + codename = msdk_get_platform_codename (priv->session.session); + + if (codename != MFX_PLATFORM_UNKNOWN) + GST_INFO ("Detected MFX platform with device code %d", codename); + else + GST_WARNING ("Unknown MFX platform"); + + return obj; + +failed: + gst_object_unref (obj); + gst_object_unref (device); + return NULL; +} +#endif mfxSession gst_msdk_context_get_session (GstMsdkContext * context) @@ -472,15 +650,23 @@ gst_msdk_context_get_handle (GstMsdkContext * context) #endif } -GstObject * -gst_msdk_context_get_display (GstMsdkContext * context) -{ #ifndef _WIN32 +GstObject * +gst_msdk_context_get_va_display (GstMsdkContext * context) +{ if (context->priv->display) return gst_object_ref (GST_OBJECT_CAST (context->priv->display)); -#endif return NULL; } +#else +GstD3D11Device * +gst_msdk_context_get_d3d11_device (GstMsdkContext * context) +{ + if (context->priv->device) + return gst_object_ref (context->priv->device); + return NULL; +} +#endif static gint _find_response (gconstpointer resp, gconstpointer comp_resp) diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.h index 141f9062e6..88746bd101 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontext.h @@ -37,6 +37,8 @@ #ifndef _WIN32 #include #include +#else +#include #endif G_BEGIN_DECLS @@ -87,12 +89,21 @@ GType gst_msdk_context_get_type (void); GstMsdkContext * gst_msdk_context_new (gboolean hardware, GstMsdkContextJobType job_type); GstMsdkContext * gst_msdk_context_new_with_parent (GstMsdkContext * parent); +#ifndef _WIN32 GstMsdkContext * gst_msdk_context_new_with_va_display (GstObject * display_obj, gboolean hardware, GstMsdkContextJobType job_type); +#else +GstMsdkContext * gst_msdk_context_new_with_d3d11_device (GstD3D11Device * device, + gboolean hardware, GstMsdkContextJobType job_type); +#endif mfxSession gst_msdk_context_get_session (GstMsdkContext * context); gpointer gst_msdk_context_get_handle (GstMsdkContext * context); -GstObject * gst_msdk_context_get_display (GstMsdkContext * context); +#ifndef _WIN32 +GstObject * gst_msdk_context_get_va_display (GstMsdkContext * context); +#else +GstD3D11Device * gst_msdk_context_get_d3d11_device (GstMsdkContext * context); +#endif /* GstMsdkContext contains mfxFrameAllocResponses, * if app calls MFXVideoCORE_SetFrameAllocator. diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.c index 6bbdebb935..c739cae962 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.c @@ -34,6 +34,8 @@ #ifndef _WIN32 #include #include +#else +#include #endif GST_DEBUG_CATEGORY_STATIC (GST_CAT_CONTEXT); @@ -236,7 +238,7 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware, #ifndef _WIN32 /* 2) Query the neighbour the VA display. If already a valid VA display, - using it by gst_msdk_context_from_external_display() in set_context(). */ + using it by gst_msdk_context_from_external_va_display() in set_context(). */ gst_va_context_query (element, GST_VA_DISPLAY_HANDLE_CONTEXT_TYPE_STR); msdk_context = g_atomic_pointer_get (context_ptr); if (msdk_context) { @@ -245,9 +247,23 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware, ret = TRUE; goto done; } +#else + /* 2) Query the neighbour the D3D11 device. If already a valid D3D11 device, + using it by gst_msdk_context_from_external_d3d11_device() in set_context(). */ + _context_query (element, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE); + msdk_context = g_atomic_pointer_get (context_ptr); + if (msdk_context) { + gst_object_ref (msdk_context); + propagate_display = FALSE; + ret = TRUE; + goto done; + } #endif - /* 3) Create a MSDK context from scratch. */ + /* 3) Create a MSDK context from scratch. Currently we use environment variable + to enable user to choose GPU device in multi-GPU environment. This variable + is only valid when there's no context returned by upstream or downstream. + Otherwise it will use the device that created by upstream or downstream. */ msdk_context = gst_msdk_context_new (hardware, job); if (!msdk_context) { GST_ERROR_OBJECT (element, "Context creation failed"); @@ -264,7 +280,7 @@ done: if (propagate_display) { #ifndef _WIN32 GstVaDisplay *display = - (GstVaDisplay *) gst_msdk_context_get_display (msdk_context); + (GstVaDisplay *) gst_msdk_context_get_va_display (msdk_context); gst_va_element_propagate_display_context (element, display); gst_clear_object (&display); #endif @@ -276,11 +292,12 @@ done: return ret; } -gboolean -gst_msdk_context_from_external_display (GstContext * context, gboolean hardware, - GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context) -{ #ifndef _WIN32 +gboolean +gst_msdk_context_from_external_va_display (GstContext * context, + gboolean hardware, GstMsdkContextJobType job_type, + GstMsdkContext ** msdk_context) +{ GstObject *va_display = NULL; const gchar *type; const GstStructure *s; @@ -309,10 +326,50 @@ gst_msdk_context_from_external_display (GstContext * context, gboolean hardware, if (ctx) return TRUE; -#endif + return FALSE; +} +#else +gboolean +gst_msdk_context_from_external_d3d11_device (GstContext * context, + gboolean hardware, GstMsdkContextJobType job_type, + GstMsdkContext ** msdk_context) +{ + GstD3D11Device *d3d11_device = NULL; + const gchar *type; + const GstStructure *s; + GstMsdkContext *ctx = NULL; + guint vendor_id = 0; + + _init_context_debug (); + + type = gst_context_get_context_type (context); + if (g_strcmp0 (type, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE)) + return FALSE; + + s = gst_context_get_structure (context); + if (gst_structure_get (s, "device", GST_TYPE_D3D11_DEVICE, &d3d11_device, + NULL)) { + g_object_get (d3d11_device, "vendor-id", &vendor_id, NULL); + if (vendor_id != 0x8086) { + GST_ERROR ("Not an Intel device"); + gst_clear_object (&d3d11_device); + return FALSE; + } + ctx = + gst_msdk_context_new_with_d3d11_device (d3d11_device, hardware, + job_type); + if (ctx) + *msdk_context = ctx; + + gst_clear_object (&d3d11_device); + } + + if (ctx) + return TRUE; return FALSE; } +#endif gboolean gst_msdk_handle_context_query (GstElement * element, GstQuery * query, @@ -346,7 +403,7 @@ gst_msdk_handle_context_query (GstElement * element, GstQuery * query, #ifndef _WIN32 if (g_strcmp0 (context_type, GST_VA_DISPLAY_HANDLE_CONTEXT_TYPE_STR) == 0) { GstStructure *s; - GstObject *display = gst_msdk_context_get_display (msdk_context); + GstObject *display = gst_msdk_context_get_va_display (msdk_context); if (display) { GST_CAT_LOG (GST_CAT_CONTEXT, @@ -360,6 +417,22 @@ gst_msdk_handle_context_query (GstElement * element, GstQuery * query, ret = TRUE; } } else +#else + if (g_strcmp0 (context_type, GST_D3D11_DEVICE_HANDLE_CONTEXT_TYPE) == 0) { + GstStructure *s; + GstD3D11Device *device = gst_msdk_context_get_d3d11_device (msdk_context); + + if (device) { + GST_CAT_LOG (GST_CAT_CONTEXT, + "setting GstD3D11Device (%" GST_PTR_FORMAT ") on context (%" + GST_PTR_FORMAT ")", device, ctxt); + + s = gst_context_writable_structure (ctxt); + gst_structure_set (s, "device", GST_TYPE_D3D11_DEVICE, device, NULL); + gst_object_unref (device); + ret = TRUE; + } + } else #endif if (g_strcmp0 (context_type, GST_MSDK_CONTEXT_TYPE_NAME) == 0) { GstStructure *s; diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.h b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.h index 0994dc0410..b90ca5e70e 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.h +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkcontextutil.h @@ -52,9 +52,15 @@ gst_msdk_ensure_new_context (GstElement * element, gboolean hardware, GstMsdkCon gboolean gst_msdk_context_get_context (GstContext * context, GstMsdkContext ** msdk_context); +#ifndef _WIN32 gboolean -gst_msdk_context_from_external_display (GstContext * context, gboolean hardware, +gst_msdk_context_from_external_va_display (GstContext * context, gboolean hardware, GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context); +#else +gboolean +gst_msdk_context_from_external_d3d11_device (GstContext * context, gboolean hardware, + GstMsdkContextJobType job_type, GstMsdkContext ** msdk_context); +#endif gboolean gst_msdk_handle_context_query (GstElement * element, GstQuery * query, GstMsdkContext * msdk_context); diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c index 4729d62a3e..ba9c444c21 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkdec.c @@ -323,13 +323,24 @@ gst_msdkdec_set_context (GstElement * element, GstContext * context) gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); - } else if (gst_msdk_context_from_external_display (context, + } else +#ifndef _WIN32 + if (gst_msdk_context_from_external_va_display (context, thiz->hardware, 0 /* GST_MSDK_JOB_DECODER will be set later */ , &msdk_context)) { gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); } +#else + if (gst_msdk_context_from_external_d3d11_device (context, + thiz->hardware, 0 /* GST_MSDK_JOB_DECODER will be set later */ , + &msdk_context)) { + gst_object_replace ((GstObject **) & thiz->context, + (GstObject *) msdk_context); + gst_object_unref (msdk_context); + } +#endif GST_ELEMENT_CLASS (parent_class)->set_context (element, context); } diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c index 1e265b3659..0c9c74f0aa 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkenc.c @@ -146,13 +146,24 @@ gst_msdkenc_set_context (GstElement * element, GstContext * context) gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); - } else if (gst_msdk_context_from_external_display (context, + } else +#ifndef _WIN32 + if (gst_msdk_context_from_external_va_display (context, thiz->hardware, 0 /* GST_MSDK_JOB_ENCODER will be set later */ , &msdk_context)) { gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); } +#else + if (gst_msdk_context_from_external_d3d11_device (context, + thiz->hardware, 0 /* GST_MSDK_JOB_ENCODER will be set later */ , + &msdk_context)) { + gst_object_replace ((GstObject **) & thiz->context, + (GstObject *) msdk_context); + gst_object_unref (msdk_context); + } +#endif GST_ELEMENT_CLASS (parent_class)->set_context (element, context); } @@ -1299,7 +1310,7 @@ gst_msdk_create_va_pool (GstMsdkEnc * thiz, GstCaps * caps, guint num_buffers) GstVaDisplay *display = NULL; GstVideoInfo info = thiz->input_state->info; - display = (GstVaDisplay *) gst_msdk_context_get_display (thiz->context); + display = (GstVaDisplay *) gst_msdk_context_get_va_display (thiz->context); if (thiz->use_dmabuf) { allocator = gst_va_dmabuf_allocator_new (display); diff --git a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c index 9fc5f38308..89ab7acd48 100644 --- a/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c +++ b/subprojects/gst-plugins-bad/sys/msdk/gstmsdkvpp.c @@ -420,7 +420,7 @@ gst_msdk_create_va_pool (GstVideoInfo * info, GstMsdkContext * msdk_context, GstVaDisplay *display = NULL; GstCaps *aligned_caps = NULL; - display = (GstVaDisplay *) gst_msdk_context_get_display (msdk_context); + display = (GstVaDisplay *) gst_msdk_context_get_va_display (msdk_context); if (use_dmabuf) allocator = gst_va_dmabuf_allocator_new (display); @@ -1661,13 +1661,24 @@ gst_msdkvpp_set_context (GstElement * element, GstContext * context) gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); - } else if (gst_msdk_context_from_external_display (context, + } else +#ifndef _WIN32 + if (gst_msdk_context_from_external_va_display (context, thiz->hardware, 0 /* GST_MSDK_JOB_VPP will be set later */ , &msdk_context)) { gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); } +#else + if (gst_msdk_context_from_external_d3d11_device (context, + thiz->hardware, 0 /* GST_MSDK_JOB_VPP will be set later */ , + &msdk_context)) { + gst_object_replace ((GstObject **) & thiz->context, + (GstObject *) msdk_context); + gst_object_unref (msdk_context); + } +#endif GST_ELEMENT_CLASS (parent_class)->set_context (element, context); }