d3d11screencapturesrc: Set rasterizer state

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5572>

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11dxgicapture.cpp

@@ -331,9 +331,7 @@ public:
 
   bool
   DrawMouse (GstD3D11Device * device, ID3D11RenderTargetView * rtv,
-      ID3D11VertexShader * vs, ID3D11PixelShader * ps,
+      ShaderResource * resource, D3D11_BOX * cropBox)
-      ID3D11InputLayout * layout, ID3D11SamplerState * sampler,
-      ID3D11BlendState * blend, D3D11_BOX * cropBox)
   {
     GST_TRACE ("Drawing mouse");
 
@@ -495,14 +493,14 @@ public:
     ID3D11Buffer *vert_buf = VertexBufferMouse.Get();
 
     context_handle->IASetVertexBuffers(0, 1, &vert_buf, &Stride, &Offset);
-    context_handle->OMSetBlendState(blend, BlendFactor, 0xFFFFFFFF);
+    context_handle->OMSetBlendState(resource->blend, BlendFactor, 0xFFFFFFFF);
     context_handle->OMSetRenderTargets(1, &rtv, nullptr);
-    context_handle->VSSetShader(vs, nullptr, 0);
+    context_handle->VSSetShader(resource->vs, nullptr, 0);
-    context_handle->PSSetShader(ps, nullptr, 0);
+    context_handle->PSSetShader(resource->ps, nullptr, 0);
     context_handle->PSSetShaderResources(0, 1, &srv);
-    context_handle->PSSetSamplers(0, 1, &sampler);
+    context_handle->PSSetSamplers(0, 1, &resource->sampler);
     context_handle->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
-    context_handle->IASetInputLayout(layout);
+    context_handle->IASetInputLayout(resource->layout);
 
     D3D11_VIEWPORT VP;
     VP.Width = static_cast<FLOAT>(FullDesc.Width);
@@ -512,6 +510,7 @@ public:
     VP.TopLeftX = 0.0f;
     VP.TopLeftY = 0.0f;
     context_handle->RSSetViewports(1, &VP);
+    context_handle->RSSetState (resource->rs);
 
     context_handle->Draw(NUMVERTICES, 0);
 
@@ -594,6 +593,7 @@ private:
     ComPtr<ID3D11VertexShader> vs;
     ComPtr<ID3D11PixelShader> ps;
     ComPtr<ID3D11InputLayout> layout;
+    ComPtr<ID3D11RasterizerState> rs;
     HRESULT hr;
 
     hr = gst_d3d11_get_vertex_shader_coord (device, &vs, &layout);
@@ -616,11 +616,18 @@ private:
       return false;
     }
 
+    hr = gst_d3d11_device_get_rasterizer (device, &rs);
+    if (!gst_d3d11_result (hr, device)) {
+      GST_ERROR ("Couldn't get rasterizer state");
+      return false;
+    }
+
     /* Everything is prepared now */
     vs_ = vs;
     ps_ = ps;
     layout_ = layout;
     sampler_ = sampler;
+    rs_ = rs;
 
     return true;
   }
@@ -1200,6 +1207,7 @@ private:
     VP.TopLeftX = 0.0f;
     VP.TopLeftY = 0.0f;
     device_context->RSSetViewports(1, &VP);
+    device_context->RSSetState (rs_.Get ());
 
     device_context->Draw(NUMVERTICES * DirtyCount, 0);
 
@@ -1409,6 +1417,7 @@ private:
   ComPtr<ID3D11PixelShader> ps_;
   ComPtr<ID3D11InputLayout> layout_;
   ComPtr<ID3D11SamplerState> sampler_;
+  ComPtr<ID3D11RasterizerState> rs_;
   ComPtr<IDXGIOutputDuplication> dupl_;
 
   /* frame metadata */
@@ -1463,9 +1472,7 @@ gst_d3d11_dxgi_capture_get_colorimetry (GstD3D11ScreenCapture * capture,
 static GstFlowReturn
 gst_d3d11_dxgi_capture_do_capture (GstD3D11ScreenCapture * capture,
     GstD3D11Device * device, ID3D11Texture2D * texture,
-    ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs,
+    ID3D11RenderTargetView * rtv, ShaderResource * resource,
-    ID3D11PixelShader * ps, ID3D11InputLayout * layout,
-    ID3D11SamplerState * sampler, ID3D11BlendState * blend,
     D3D11_BOX * crop_box, gboolean draw_mouse);
 
 #define gst_d3d11_dxgi_capture_parent_class parent_class
@@ -1816,9 +1823,7 @@ gst_d3d11_dxgi_capture_get_colorimetry (GstD3D11ScreenCapture * capture,
 static GstFlowReturn
 gst_d3d11_dxgi_capture_do_capture (GstD3D11ScreenCapture * capture,
     GstD3D11Device * device, ID3D11Texture2D * texture,
-    ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs,
+    ID3D11RenderTargetView * rtv, ShaderResource * resource,
-    ID3D11PixelShader * ps, ID3D11InputLayout * layout,
-    ID3D11SamplerState * sampler, ID3D11BlendState * blend,
     D3D11_BOX * crop_box, gboolean draw_mouse)
 {
   GstD3D11DxgiCapture *self = GST_D3D11_DXGI_CAPTURE (capture);
@@ -1887,10 +1892,8 @@ gst_d3d11_dxgi_capture_do_capture (GstD3D11ScreenCapture * capture,
   if (ret != GST_FLOW_OK)
     goto out;
 
-  if (draw_mouse) {
+  if (draw_mouse)
-    self->dupl_obj->DrawMouse (device,
+    self->dupl_obj->DrawMouse (device, rtv, resource, crop_box);
-        rtv, vs, ps, layout, sampler, blend, crop_box);
-  }
 
 out:
   if (shared_device)

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.cpp

@@ -137,9 +137,7 @@ gst_d3d11_screen_capture_show_border (GstD3D11ScreenCapture * capture,
 GstFlowReturn
 gst_d3d11_screen_capture_do_capture (GstD3D11ScreenCapture * capture,
     GstD3D11Device * device, ID3D11Texture2D * texture,
-    ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs,
+    ID3D11RenderTargetView * rtv, ShaderResource * resource,
-    ID3D11PixelShader * ps, ID3D11InputLayout * layout,
-    ID3D11SamplerState * sampler, ID3D11BlendState * blend,
     D3D11_BOX * crop_box, gboolean draw_mouse)
 {
   GstD3D11ScreenCaptureClass *klass;
@@ -151,7 +149,7 @@ gst_d3d11_screen_capture_do_capture (GstD3D11ScreenCapture * capture,
   g_assert (klass->do_capture);
 
   return klass->do_capture (capture, device, texture, rtv,
-      vs, ps, layout, sampler, blend, crop_box, draw_mouse);
+      resource, crop_box, draw_mouse);
 }
 
 HRESULT

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapture.h

@@ -41,6 +41,16 @@ typedef struct _GstD3D11ScreenCaptureClass GstD3D11ScreenCaptureClass;
 #define GST_D3D11_SCREEN_CAPTURE_FLOW_SIZE_CHANGED GST_FLOW_CUSTOM_SUCCESS_1
 #define GST_D3D11_SCREEN_CAPTURE_FLOW_UNSUPPORTED GST_FLOW_CUSTOM_ERROR
 
+struct ShaderResource
+{
+  ID3D11VertexShader * vs;
+  ID3D11PixelShader * ps;
+  ID3D11InputLayout * layout;
+  ID3D11SamplerState * sampler;
+  ID3D11BlendState * blend;
+  ID3D11RasterizerState * rs;
+};
+
 struct _GstD3D11ScreenCapture
 {
   GstObject parent;
@@ -70,11 +80,7 @@ struct _GstD3D11ScreenCaptureClass
                                     GstD3D11Device * device,
                                     ID3D11Texture2D * texture,
                                     ID3D11RenderTargetView * rtv,
-                                    ID3D11VertexShader * vs,
+                                    ShaderResource * resource,
-                                    ID3D11PixelShader * ps,
-                                    ID3D11InputLayout * layout,
-                                    ID3D11SamplerState * sampler,
-                                    ID3D11BlendState * blend,
                                     D3D11_BOX * crop_box,
                                     gboolean draw_mouse);
 };
@@ -101,11 +107,7 @@ GstFlowReturn   gst_d3d11_screen_capture_do_capture  (GstD3D11ScreenCapture * ca
                                                       GstD3D11Device * device,
                                                       ID3D11Texture2D * texture,
                                                       ID3D11RenderTargetView * rtv,
-                                                      ID3D11VertexShader * vs,
+                                                      ShaderResource * resource,
-                                                      ID3D11PixelShader * ps,
-                                                      ID3D11InputLayout * layout,
-                                                      ID3D11SamplerState * sampler,
-                                                      ID3D11BlendState * blend,
                                                       D3D11_BOX * crop_box,
                                                       gboolean draw_mouse);
 

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11screencapturesrc.cpp

@@ -201,11 +201,7 @@ struct _GstD3D11ScreenCaptureSrc
 
   gboolean downstream_supports_d3d11;
 
-  ID3D11VertexShader *vs;
+  ShaderResource resource;
-  ID3D11PixelShader *ps;
-  ID3D11InputLayout *layout;
-  ID3D11SamplerState *sampler;
-  ID3D11BlendState *blend;
 
   CRITICAL_SECTION lock;
 };
@@ -875,9 +871,11 @@ gst_d3d11_screen_capture_prepare_shader (GstD3D11ScreenCaptureSrc * self)
   ComPtr < ID3D11PixelShader > ps;
   ComPtr < ID3D11SamplerState > sampler;
   ComPtr < ID3D11BlendState > blend;
+  ComPtr < ID3D11RasterizerState > rs;
   D3D11_BLEND_DESC blend_desc;
   ID3D11Device *device_handle;
   HRESULT hr;
+  ShaderResource *resource = &self->resource;
 
   device_handle = gst_d3d11_device_get_device_handle (self->device);
 
@@ -920,11 +918,18 @@ gst_d3d11_screen_capture_prepare_shader (GstD3D11ScreenCaptureSrc * self)
     return FALSE;
   }
 
-  self->vs = vs.Detach ();
+  hr = gst_d3d11_device_get_rasterizer (self->device, &rs);
-  self->ps = ps.Detach ();
+  if (!gst_d3d11_result (hr, self->device)) {
-  self->layout = layout.Detach ();
+    GST_ERROR_OBJECT (self, "Couldn't get rasterizer state");
-  self->sampler = sampler.Detach ();
+    return FALSE;
-  self->blend = blend.Detach ();
+  }
+
+  resource->vs = vs.Detach ();
+  resource->ps = ps.Detach ();
+  resource->layout = layout.Detach ();
+  resource->sampler = sampler.Detach ();
+  resource->blend = blend.Detach ();
+  resource->rs = rs.Detach ();
 
   return TRUE;
 }
@@ -1079,6 +1084,7 @@ static gboolean
 gst_d3d11_screen_capture_src_stop (GstBaseSrc * bsrc)
 {
   GstD3D11ScreenCaptureSrc *self = GST_D3D11_SCREEN_CAPTURE_SRC (bsrc);
+  ShaderResource *resource = &self->resource;
   GstD3D11CSLockGuard lk (&self->lock);
 
   if (self->pool) {
@@ -1086,11 +1092,12 @@ gst_d3d11_screen_capture_src_stop (GstBaseSrc * bsrc)
     gst_clear_object (&self->pool);
   }
 
-  GST_D3D11_CLEAR_COM (self->vs);
+  GST_D3D11_CLEAR_COM (resource->vs);
-  GST_D3D11_CLEAR_COM (self->ps);
+  GST_D3D11_CLEAR_COM (resource->ps);
-  GST_D3D11_CLEAR_COM (self->layout);
+  GST_D3D11_CLEAR_COM (resource->layout);
-  GST_D3D11_CLEAR_COM (self->sampler);
+  GST_D3D11_CLEAR_COM (resource->sampler);
-  GST_D3D11_CLEAR_COM (self->blend);
+  GST_D3D11_CLEAR_COM (resource->blend);
+  GST_D3D11_CLEAR_COM (resource->rs);
 
   gst_clear_object (&self->capture);
   gst_clear_object (&self->device);
@@ -1325,8 +1332,7 @@ again:
   texture = (ID3D11Texture2D *) info.data;
   before_capture = gst_clock_get_time (clock);
   ret = gst_d3d11_screen_capture_do_capture (self->capture, self->device,
-      texture, rtv, self->vs, self->ps, self->layout, self->sampler,
+      texture, rtv, &self->resource, &self->crop_box, draw_mouse);
-      self->blend, &self->crop_box, draw_mouse);
   gst_memory_unmap (mem, &info);
 
   switch (ret) {

subprojects/gst-plugins-bad/sys/d3d11/gstd3d11winrtcapture.cpp

@@ -271,9 +271,7 @@ gst_d3d11_winrt_capture_show_border (GstD3D11ScreenCapture * capture,
 static GstFlowReturn
 gst_d3d11_winrt_capture_do_capture (GstD3D11ScreenCapture * capture,
     GstD3D11Device * device, ID3D11Texture2D * texture,
-    ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs,
+    ID3D11RenderTargetView * rtv, ShaderResource * resource,
-    ID3D11PixelShader * ps, ID3D11InputLayout * layout,
-    ID3D11SamplerState * sampler, ID3D11BlendState * blend,
     D3D11_BOX * crop_box, gboolean draw_mouse);
 static gpointer
 gst_d3d11_winrt_capture_thread_func (GstD3D11WinRTCapture * self);
@@ -848,9 +846,7 @@ gst_d3d11_winrt_capture_show_border (GstD3D11ScreenCapture * capture,
 static GstFlowReturn
 gst_d3d11_winrt_capture_do_capture (GstD3D11ScreenCapture * capture,
     GstD3D11Device * device, ID3D11Texture2D * texture,
-    ID3D11RenderTargetView * rtv, ID3D11VertexShader * vs,
+    ID3D11RenderTargetView * rtv, ShaderResource * resource,
-    ID3D11PixelShader * ps, ID3D11InputLayout * layout,
-    ID3D11SamplerState * sampler, ID3D11BlendState * blend,
     D3D11_BOX * crop_box, gboolean draw_mouse)
 {
   GstD3D11WinRTCapture *self = GST_D3D11_WINRT_CAPTURE (capture);