From 222963bd5a9ed13a7333824a5073a4d98a63d34b Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Thu, 5 May 2022 02:16:54 +0900 Subject: [PATCH] d3d11screencapturesrc: Fix crash when d3d11 device is different from owned one GstD3D11ScreenCapture object is pipeline-independent global object and the object can be shared by multiple src elements, in order to overcome a limitation of DXGI Desktop Duplication API. Note that the API allows only single capture session in a process for a monitor. Therefore GstD3D11ScreenCapture object must be able to handle a case where a src element holds different GstD3D11Device object. Which can happen when GstD3D11Device context is not shared by pipelines. What's changed: * Allocates capture texture with D3D11_RESOURCE_MISC_SHARED for the texture to be able to copied into other device's texture * Holds additional shader objects per src element and use it when drawing mouse Fixes: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/1197 Part-of: --- .../sys/d3d11/gstd3d11screencapture.cpp | 149 ++++++++++++------ .../sys/d3d11/gstd3d11screencapture.h | 8 +- .../sys/d3d11/gstd3d11screencapturesrc.cpp | 134 +++++++++++++++- .../examples/d3d11/d3d11screencapturesrc.cpp | 43 ++++- 4 files changed, 282 insertions(+), 52 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.cpp index b049ccb8c6..226ecd7054 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.cpp @@ -280,7 +280,8 @@ public: texture_desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; texture_desc.CPUAccessFlags = 0; - texture_desc.MiscFlags = 0; + /* source element may hold different d3d11 device object */ + texture_desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED; hr = device_handle->CreateTexture2D (&texture_desc, nullptr, &shared_texture_); @@ -344,7 +345,11 @@ public: return GST_FLOW_OK; } - bool DrawMouse (ID3D11RenderTargetView * rtv) + bool + DrawMouse (GstD3D11Device * device, ID3D11RenderTargetView * rtv, + ID3D11VertexShader * vs, ID3D11PixelShader * ps, + ID3D11InputLayout * layout, ID3D11SamplerState * sampler, + ID3D11BlendState * blend) { GST_TRACE ("Drawing mouse"); @@ -359,9 +364,9 @@ public: D3D11_SUBRESOURCE_DATA InitData; D3D11_TEXTURE2D_DESC Desc; D3D11_SHADER_RESOURCE_VIEW_DESC SDesc; - ID3D11Device *device_handle = gst_d3d11_device_get_device_handle (device_); + ID3D11Device *device_handle = gst_d3d11_device_get_device_handle (device); ID3D11DeviceContext *context_handle = - gst_d3d11_device_get_device_context_handle (device_); + gst_d3d11_device_get_device_context_handle (device); VERTEX Vertices[NUMVERTICES] = { @@ -462,7 +467,7 @@ public: // Create mouseshape as texture HRESULT hr = device_handle->CreateTexture2D(&Desc, &InitData, &MouseTex); - if (!gst_d3d11_result (hr, device_)) { + if (!gst_d3d11_result (hr, device)) { GST_ERROR ("Failed to create texture for rendering mouse"); return false; } @@ -470,7 +475,7 @@ public: // Create shader resource from texture hr = device_handle->CreateShaderResourceView(MouseTex.Get(), &SDesc, &ShaderRes); - if (!gst_d3d11_result (hr, device_)) { + if (!gst_d3d11_result (hr, device)) { GST_ERROR ("Failed to create shader resource view for rendering mouse"); return false; } @@ -487,7 +492,7 @@ public: // Create vertex buffer hr = device_handle->CreateBuffer(&BDesc, &InitData, &VertexBufferMouse); - if (!gst_d3d11_result (hr, device_)) { + if (!gst_d3d11_result (hr, device)) { GST_ERROR ("Failed to create vertex buffer for rendering mouse"); return false; } @@ -495,19 +500,18 @@ public: FLOAT BlendFactor[4] = {0.f, 0.f, 0.f, 0.f}; UINT Stride = sizeof(VERTEX); UINT Offset = 0; - ID3D11SamplerState *samplers = sampler_.Get(); ID3D11ShaderResourceView *srv = ShaderRes.Get(); ID3D11Buffer *vert_buf = VertexBufferMouse.Get(); context_handle->IASetVertexBuffers(0, 1, &vert_buf, &Stride, &Offset); - context_handle->OMSetBlendState(blend_.Get(), BlendFactor, 0xFFFFFFFF); + context_handle->OMSetBlendState(blend, BlendFactor, 0xFFFFFFFF); context_handle->OMSetRenderTargets(1, &rtv, nullptr); - context_handle->VSSetShader(vs_.Get(), nullptr, 0); - context_handle->PSSetShader(ps_.Get(), nullptr, 0); + context_handle->VSSetShader(vs, nullptr, 0); + context_handle->PSSetShader(ps, nullptr, 0); context_handle->PSSetShaderResources(0, 1, &srv); - context_handle->PSSetSamplers(0, 1, &samplers); + context_handle->PSSetSamplers(0, 1, &sampler); context_handle->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST); - context_handle->IASetInputLayout(layout_.Get()); + context_handle->IASetInputLayout(layout); D3D11_VIEWPORT VP; VP.Width = static_cast(FullDesc.Width); @@ -531,14 +535,60 @@ public: return true; } - void - CopyToTexture (ID3D11Texture2D * texture) + GstFlowReturn + CopyToTexture (GstD3D11Device * device, ID3D11Texture2D * texture) { - ID3D11DeviceContext *context_handle = - gst_d3d11_device_get_device_context_handle (device_); + ID3D11DeviceContext *context_handle = nullptr; + ComPtr tex; + ComPtr < ID3D11Query > query; + HRESULT hr; + + context_handle = gst_d3d11_device_get_device_context_handle (device); + + if (device == device_) { + tex = shared_texture_; + } else { + ID3D11Device *device_handle = nullptr; + ComPtr < IDXGIResource > dxgi_resource; + D3D11_QUERY_DESC query_desc; + HANDLE shared_handle; + + device_handle = gst_d3d11_device_get_device_handle (device); + + hr = shared_texture_.As (&dxgi_resource); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + hr = dxgi_resource->GetSharedHandle (&shared_handle); + if (!gst_d3d11_result (hr, device_)) + return GST_FLOW_ERROR; + + hr = device_handle->OpenSharedResource (shared_handle, + IID_PPV_ARGS (&tex)); + if (!gst_d3d11_result (hr, device)) + return GST_FLOW_ERROR; + + query_desc.Query = D3D11_QUERY_EVENT; + query_desc.MiscFlags = 0; + + hr = device_handle->CreateQuery (&query_desc, &query); + if (!gst_d3d11_result (hr, device)) + return GST_FLOW_ERROR; + } context_handle->CopySubresourceRegion (texture, 0, 0, 0, 0, - shared_texture_.Get(), 0, nullptr); + tex.Get(), 0, nullptr); + + if (query) { + BOOL sync_done = FALSE; + + do { + hr = context_handle->GetData (query.Get (), + &sync_done, sizeof (BOOL), 0); + } while (!sync_done && (hr == S_OK || hr == S_FALSE)); + } + + return GST_FLOW_OK; } void @@ -628,33 +678,11 @@ private: return false; } - /* For blending mouse pointer texture */ - D3D11_BLEND_DESC blend_desc; - blend_desc.AlphaToCoverageEnable = FALSE; - blend_desc.IndependentBlendEnable = FALSE; - blend_desc.RenderTarget[0].BlendEnable = TRUE; - blend_desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; - blend_desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; - blend_desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; - blend_desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; - blend_desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO; - blend_desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD; - blend_desc.RenderTarget[0].RenderTargetWriteMask = - D3D11_COLOR_WRITE_ENABLE_ALL; - - ComPtr blend; - hr = device_handle->CreateBlendState (&blend_desc, &blend); - if (!gst_d3d11_result (hr, device)) { - GST_ERROR ("Failed to create blend state, hr 0x%x", (guint) hr); - return false; - } - /* Everything is prepared now */ vs_ = vs; ps_ = ps; layout_ = layout; sampler_ = sampler; - blend_ = blend; return true; } @@ -1445,7 +1473,6 @@ private: ComPtr layout_; ComPtr sampler_; ComPtr dupl_; - ComPtr blend_; /* frame metadata */ BYTE *metadata_buffer_; @@ -1479,6 +1506,7 @@ struct _GstD3D11ScreenCapture HMONITOR monitor_handle; RECT desktop_coordinates; gboolean prepared; + gint64 adapter_luid; GRecMutex lock; }; @@ -1623,6 +1651,8 @@ gst_d3d11_screen_capture_constructed (GObject * object) self->desktop_coordinates.right, self->desktop_coordinates.bottom, self->cached_width, self->cached_height); + g_object_get (self->device, "adapter-luid", &self->adapter_luid, nullptr); + ret = TRUE; out: @@ -1798,16 +1828,33 @@ gst_d3d11_screen_capture_get_size (GstD3D11ScreenCapture * capture, GstFlowReturn gst_d3d11_screen_capture_do_capture (GstD3D11ScreenCapture * capture, - ID3D11Texture2D * texture, ID3D11RenderTargetView * rtv, - gboolean draw_mouse) + GstD3D11Device * device, ID3D11Texture2D * texture, + ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs, + ID3D11PixelShader * ps, ID3D11InputLayout * layout, + ID3D11SamplerState * sampler, ID3D11BlendState * blend, gboolean draw_mouse) { GstFlowReturn ret = GST_FLOW_OK; D3D11_TEXTURE2D_DESC desc; + gboolean shared_device = FALSE; guint width, height; g_return_val_if_fail (GST_IS_D3D11_SCREEN_CAPTURE (capture), GST_FLOW_ERROR); g_return_val_if_fail (texture != nullptr, GST_FLOW_ERROR); + if (device != capture->device) { + gint64 luid; + + g_object_get (device, "adapter-luid", &luid, nullptr); + /* source element must hold d3d11 device for the same GPU already + * by DXGI duplication API design */ + if (luid != capture->adapter_luid) { + GST_ERROR_OBJECT (capture, "Trying to capture from different device"); + return GST_FLOW_ERROR; + } + + shared_device = TRUE; + } + g_rec_mutex_lock (&capture->lock); if (!capture->prepared) ret = gst_d3d11_screen_capture_prepare (capture); @@ -1851,14 +1898,24 @@ gst_d3d11_screen_capture_do_capture (GstD3D11ScreenCapture * capture, } GST_LOG_OBJECT (capture, "Capture done"); + if (shared_device) + gst_d3d11_device_lock (device); + + ret = capture->dupl_obj->CopyToTexture (device, texture); + if (ret != GST_FLOW_OK) + goto out; - capture->dupl_obj->CopyToTexture (texture); if (draw_mouse) - capture->dupl_obj->DrawMouse (rtv); + capture->dupl_obj->DrawMouse (device, rtv, vs, ps, layout, sampler, blend); + +out: + if (shared_device) + gst_d3d11_device_unlock (device); + gst_d3d11_device_unlock (capture->device); g_rec_mutex_unlock (&capture->lock); - return GST_FLOW_OK; + return ret; } HRESULT diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.h index 3c9ce3e589..6c6ee8feb4 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.h +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.h @@ -44,8 +44,14 @@ gboolean gst_d3d11_screen_capture_get_size (GstD3D11ScreenCapture * captu guint * height); GstFlowReturn gst_d3d11_screen_capture_do_capture (GstD3D11ScreenCapture * capture, + GstD3D11Device * device, ID3D11Texture2D * texture, - ID3D11RenderTargetView *rtv, + ID3D11RenderTargetView * rtv, + ID3D11VertexShader * vs, + ID3D11PixelShader * ps, + ID3D11InputLayout * layout, + ID3D11SamplerState * sampler, + ID3D11BlendState * blend, gboolean draw_mouse); HRESULT gst_d3d11_screen_capture_find_output_for_monitor (HMONITOR monitor, diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp index 4af31c66e0..e250e3fa95 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp @@ -39,6 +39,7 @@ #include "gstd3d11screencapturesrc.h" #include "gstd3d11screencapture.h" #include "gstd3d11pluginutils.h" +#include "gstd3d11shader.h" #include #include @@ -92,6 +93,12 @@ struct _GstD3D11ScreenCaptureSrc GstClockTime max_latency; gboolean downstream_supports_d3d11; + + ID3D11VertexShader *vs; + ID3D11PixelShader *ps; + ID3D11InputLayout *layout; + ID3D11SamplerState *sampler; + ID3D11BlendState *blend; }; static void gst_d3d11_screen_capture_src_dispose (GObject * object); @@ -503,6 +510,118 @@ error: return FALSE; } +static gboolean +gst_d3d11_screen_capture_prepare_shader (GstD3D11ScreenCaptureSrc * self) +{ + /* *INDENT-OFF* */ + static const gchar vs_str[] = + "struct VS_INPUT {\n" + " float4 Position: POSITION;\n" + " float2 Texture: TEXCOORD;\n" + "};\n" + "\n" + "struct VS_OUTPUT {\n" + " float4 Position: SV_POSITION;\n" + " float2 Texture: TEXCOORD;\n" + "};\n" + "\n" + "VS_OUTPUT main (VS_INPUT input)\n" + "{\n" + " return input;\n" + "}"; + static const gchar ps_str[] = + "Texture2D shaderTexture;\n" + "SamplerState samplerState;\n" + "\n" + "struct PS_INPUT {\n" + " float4 Position: SV_POSITION;\n" + " float2 Texture: TEXCOORD;\n" + "};\n" + "\n" + "struct PS_OUTPUT {\n" + " float4 Plane: SV_Target;\n" + "};\n" + "\n" + "PS_OUTPUT main(PS_INPUT input)\n" + "{\n" + " PS_OUTPUT output;\n" + " output.Plane = shaderTexture.Sample(samplerState, input.Texture);\n" + " return output;\n" + "}"; + /* *INDENT-ON* */ + D3D11_INPUT_ELEMENT_DESC input_desc[] = { + {"POSITION", + 0, DXGI_FORMAT_R32G32B32_FLOAT, 0, 0, D3D11_INPUT_PER_VERTEX_DATA, 0}, + {"TEXCOORD", + 0, DXGI_FORMAT_R32G32_FLOAT, 0, 12, D3D11_INPUT_PER_VERTEX_DATA, 0} + }; + ComPtr < ID3D11VertexShader > vs; + ComPtr < ID3D11InputLayout > layout; + ComPtr < ID3D11PixelShader > ps; + ComPtr < ID3D11SamplerState > sampler; + ComPtr < ID3D11BlendState > blend; + D3D11_SAMPLER_DESC sampler_desc; + D3D11_BLEND_DESC blend_desc; + ID3D11Device *device_handle; + HRESULT hr; + + device_handle = gst_d3d11_device_get_device_handle (self->device); + + if (!gst_d3d11_create_vertex_shader (self->device, + vs_str, input_desc, G_N_ELEMENTS (input_desc), &vs, &layout)) { + GST_ERROR_OBJECT (self, "Failed to create vertex shader"); + return FALSE; + } + + if (!gst_d3d11_create_pixel_shader (self->device, ps_str, &ps)) { + GST_ERROR_OBJECT (self, "Failed to create pixel shader"); + return FALSE; + } + + memset (&sampler_desc, 0, sizeof (D3D11_SAMPLER_DESC)); + sampler_desc.Filter = D3D11_FILTER_MIN_MAG_MIP_LINEAR; + sampler_desc.AddressU = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressV = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.AddressW = D3D11_TEXTURE_ADDRESS_CLAMP; + sampler_desc.ComparisonFunc = D3D11_COMPARISON_NEVER; + sampler_desc.MinLOD = 0; + sampler_desc.MaxLOD = D3D11_FLOAT32_MAX; + + hr = device_handle->CreateSamplerState (&sampler_desc, &sampler); + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, + "Failed to create sampler state, hr 0x%x", (guint) hr); + return FALSE; + } + + blend_desc.AlphaToCoverageEnable = FALSE; + blend_desc.IndependentBlendEnable = FALSE; + blend_desc.RenderTarget[0].BlendEnable = TRUE; + blend_desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA; + blend_desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA; + blend_desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD; + blend_desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE; + blend_desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_ZERO; + blend_desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD; + blend_desc.RenderTarget[0].RenderTargetWriteMask = + D3D11_COLOR_WRITE_ENABLE_ALL; + + hr = device_handle->CreateBlendState (&blend_desc, &blend); + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, + "Failed to create blend state, hr 0x%x", (guint) hr); + return FALSE; + } + + self->vs = vs.Detach (); + self->ps = ps.Detach (); + self->layout = layout.Detach (); + self->sampler = sampler.Detach (); + self->blend = blend.Detach (); + + return TRUE; +} + static gboolean gst_d3d11_screen_capture_src_start (GstBaseSrc * bsrc) { @@ -559,6 +678,9 @@ gst_d3d11_screen_capture_src_start (GstBaseSrc * bsrc) goto error; } + if (!gst_d3d11_screen_capture_prepare_shader (self)) + goto error; + self->last_frame_no = -1; self->min_latency = self->max_latency = GST_CLOCK_TIME_NONE; @@ -594,6 +716,12 @@ gst_d3d11_screen_capture_src_stop (GstBaseSrc * bsrc) gst_clear_object (&self->pool); } + GST_D3D11_CLEAR_COM (self->vs); + GST_D3D11_CLEAR_COM (self->ps); + GST_D3D11_CLEAR_COM (self->layout); + GST_D3D11_CLEAR_COM (self->sampler); + GST_D3D11_CLEAR_COM (self->blend); + gst_clear_object (&self->capture); gst_clear_object (&self->device); @@ -807,9 +935,9 @@ again: texture = (ID3D11Texture2D *) info.data; before_capture = gst_clock_get_time (clock); - ret = - gst_d3d11_screen_capture_do_capture (self->capture, texture, rtv, - draw_mouse); + ret = gst_d3d11_screen_capture_do_capture (self->capture, self->device, + texture, rtv, self->vs, self->ps, self->layout, self->sampler, + self->blend, draw_mouse); gst_memory_unmap (mem, &info); switch (ret) { diff --git a/subprojects/gst-plugins-bad/tests/examples/d3d11/d3d11screencapturesrc.cpp b/subprojects/gst-plugins-bad/tests/examples/d3d11/d3d11screencapturesrc.cpp index 6d963d3d42..2bb74030d5 100644 --- a/subprojects/gst-plugins-bad/tests/examples/d3d11/d3d11screencapturesrc.cpp +++ b/subprojects/gst-plugins-bad/tests/examples/d3d11/d3d11screencapturesrc.cpp @@ -143,9 +143,12 @@ gint main (gint argc, gchar ** argv) { GstElement *pipeline, *src, *queue, *sink; + GstElement *pipeline_1 = nullptr, *src_1, *queue_1, *sink_1; GMainLoop *loop; gboolean ret; gboolean show_devices = FALSE; + gboolean multi_pipelines = FALSE; + gboolean show_cursor = FALSE; gint64 hmonitor = 0; gint monitor_index = -1; GError *err = nullptr; @@ -158,6 +161,10 @@ main (gint argc, gchar ** argv) "Address of HMONITOR handle", nullptr}, {"index", 0, 0, G_OPTION_ARG_INT, &monitor_index, "Monitor index to capture (-1 for primary monitor)", nullptr}, + {"multi-pipelines", 0, 0, G_OPTION_ARG_NONE, &multi_pipelines, + "Run two separate pipelines for capturing a single monitor", nullptr}, + {"show-cursor", 0, 0, G_OPTION_ARG_NONE, &show_cursor, + "Draw mouse cursor", nullptr}, {nullptr} }; @@ -185,12 +192,25 @@ main (gint argc, gchar ** argv) } src = gst_device_create_element (device, nullptr); - gst_object_unref (device); if (!src) { g_warning ("Failed to create d3d11screencapture element"); return 1; } + g_object_set (src, "show-cursor", show_cursor, nullptr); + + if (multi_pipelines) { + src_1 = gst_device_create_element (device, nullptr); + if (!src_1) { + g_warning ("Failed to create second d3d11screencapture element"); + return 1; + } + + g_object_set (src_1, "show-cursor", show_cursor, nullptr); + } + + gst_object_unref (device); + loop = g_main_loop_new (nullptr, FALSE); pipeline = gst_pipeline_new (nullptr); @@ -203,12 +223,31 @@ main (gint argc, gchar ** argv) gst_bus_add_watch (GST_ELEMENT_BUS (pipeline), (GstBusFunc) bus_msg, loop); gst_element_set_state (pipeline, GST_STATE_PLAYING); + if (multi_pipelines) { + pipeline_1 = gst_pipeline_new (nullptr); + + queue_1 = gst_element_factory_make ("queue", nullptr); + sink_1 = gst_element_factory_make ("d3d11videosink", nullptr); + + gst_bin_add_many (GST_BIN (pipeline_1), src_1, queue_1, sink_1, nullptr); + gst_element_link_many (src_1, queue_1, sink_1, nullptr); + + gst_bus_add_watch (GST_ELEMENT_BUS (pipeline_1), (GstBusFunc) bus_msg, loop); + gst_element_set_state (pipeline_1, GST_STATE_PLAYING); + } + g_main_loop_run (loop); gst_element_set_state (pipeline, GST_STATE_NULL); gst_bus_remove_watch (GST_ELEMENT_BUS (pipeline)); - gst_object_unref (pipeline); + + if (multi_pipelines) { + gst_element_set_state (pipeline_1, GST_STATE_NULL); + gst_bus_remove_watch (GST_ELEMENT_BUS (pipeline)); + gst_object_unref (pipeline_1); + } + g_main_loop_unref (loop); return 0;