From 3a99517f7c6a7c7e99bcc40e448958f6956ad9ee Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Mon, 15 Mar 2021 19:48:56 +0900 Subject: [PATCH] d3d11videosink: Avoid switching conversion tool during playback Decoder might be able to copy decoded texture to the other buffer pool during playback depending on context. In that case, copied one has no D3D11_BIND_DECODER bind flag. If we used ID3D11VideoProcessor previously for decoder texture, and incoming texture supports ID3D11VideoProcessor as well even if it has no D3D11_BIND_DECODER flag (having D3D11_BIND_RENDER_TARGET for example), allow zero-copying instead of using our fallback texture. Frequent conversion tool change (between ID3D11VideoProcessor and generic shader) might result in inconsistent image quality. Part-of: --- sys/d3d11/gstd3d11videosink.cpp | 8 +++ sys/d3d11/gstd3d11window.cpp | 93 ++++++++++++++++++++++++--------- sys/d3d11/gstd3d11window.h | 2 + 3 files changed, 78 insertions(+), 25 deletions(-) diff --git a/sys/d3d11/gstd3d11videosink.cpp b/sys/d3d11/gstd3d11videosink.cpp index 9e1fffe850..605f6248a2 100644 --- a/sys/d3d11/gstd3d11videosink.cpp +++ b/sys/d3d11/gstd3d11videosink.cpp @@ -125,6 +125,7 @@ struct _GstD3D11VideoSink GstBufferPool *fallback_pool; gboolean can_convert; gboolean have_video_processor; + gboolean processor_in_use; /* For drawing on user texture */ GstD3D11VideoSinkCallbacks callbacks; @@ -701,6 +702,8 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps) return FALSE; } + self->processor_in_use = FALSE; + return TRUE; /* ERRORS */ @@ -1118,6 +1121,11 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf) GST_TRACE_OBJECT (self, "Got VideoProcessor compatible texture, do direct rendering"); direct_rendering = TRUE; + self->processor_in_use = TRUE; + } else if (self->processor_in_use && + (desc.BindFlags & D3D11_BIND_RENDER_TARGET) == + D3D11_BIND_RENDER_TARGET) { + direct_rendering = TRUE; } } diff --git a/sys/d3d11/gstd3d11window.cpp b/sys/d3d11/gstd3d11window.cpp index 8077f5ba07..cf7479f70e 100644 --- a/sys/d3d11/gstd3d11window.cpp +++ b/sys/d3d11/gstd3d11window.cpp @@ -650,6 +650,8 @@ gst_d3d11_window_prepare_default (GstD3D11Window * window, guint display_width, g_clear_pointer (&window->converter, gst_d3d11_converter_free); g_clear_pointer (&window->compositor, gst_d3d11_overlay_compositor_free); + window->processor_in_use = FALSE; + /* Step 2: Decide display color format * If upstream format is 10bits, try DXGI_FORMAT_R10G10B10A2_UNORM first * Otherwise, use DXGI_FORMAT_B8G8R8A8_UNORM or DXGI_FORMAT_B8G8R8A8_UNORM @@ -1059,6 +1061,40 @@ gst_d3d11_window_present_d2d (GstD3D11Window * self, GstStructure * stats) } #endif +static gboolean +gst_d3d11_window_do_processor (GstD3D11Window * self, + ID3D11VideoProcessorInputView * piv, ID3D11VideoProcessorOutputView * pov) +{ + gboolean ret; + + ret = gst_d3d11_video_processor_render_unlocked (self->processor, + &self->input_rect, piv, &self->render_rect, pov); + if (!ret) { + GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using processor"); + } else { + GST_TRACE_OBJECT (self, "Rendered using processor"); + self->processor_in_use = TRUE; + } + + return ret; +} + +static gboolean +gst_d3d11_window_do_convert (GstD3D11Window * self, + ID3D11ShaderResourceView * srv[GST_VIDEO_MAX_PLANES], + ID3D11RenderTargetView * rtv) +{ + if (!gst_d3d11_converter_convert_unlocked (self->converter, + srv, &rtv, NULL, NULL)) { + GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using converter"); + return FALSE; + } else { + GST_TRACE_OBJECT (self, "Rendered using converter"); + } + + return TRUE; +} + static GstFlowReturn gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer, GstStructure * stats, ID3D11VideoProcessorOutputView * pov, @@ -1077,19 +1113,25 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer, ID3D11VideoProcessorInputView *piv = NULL; ID3D11Device *device_handle = gst_d3d11_device_get_device_handle (self->device); + gboolean can_convert = FALSE; + gboolean can_process = FALSE; + gboolean convert_ret = FALSE; /* Map memory in any case so that we can upload pending stage texture */ if (!gst_d3d11_buffer_map (buffer, device_handle, infos, GST_MAP_READ)) { GST_ERROR_OBJECT (self, "Couldn't map buffer"); - return GST_FLOW_ERROR; } - if (!gst_d3d11_buffer_get_shader_resource_view (buffer, srv)) { - if (!gst_d3d11_window_buffer_ensure_processor_input (self, buffer, &piv)) { - GST_ERROR_OBJECT (self, "Input texture cannot be used for converter"); - return GST_FLOW_ERROR; - } + can_convert = gst_d3d11_buffer_get_shader_resource_view (buffer, srv); + if (pov) { + can_process = gst_d3d11_window_buffer_ensure_processor_input (self, + buffer, &piv); + } + + if (!can_convert && !can_process) { + GST_ERROR_OBJECT (self, "Input texture cannot be used for converter"); + return GST_FLOW_ERROR; } if (self->first_present) { @@ -1106,26 +1148,27 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer, &viewport); } - if (self->processor && piv && pov) { - if (!gst_d3d11_video_processor_render_unlocked (self->processor, - &self->input_rect, piv, &self->render_rect, pov)) { - GST_ERROR_OBJECT (self, - "Couldn't render to backbuffer using processor"); - ret = GST_FLOW_ERROR; - goto unmap_and_out; - } else { - GST_TRACE_OBJECT (self, "Rendered using processor"); - } + /* Converter preference order + * 1) If this texture can be converted via processor, and we used processor + * previously, use processor + * 2) If SRV is available, use converter + * 3) otherwise, use processor + */ + if (can_process && self->processor_in_use) { + convert_ret = gst_d3d11_window_do_processor (self, piv, pov); + } else if (can_convert) { + convert_ret = gst_d3d11_window_do_convert (self, srv, rtv); + } else if (can_process) { + convert_ret = gst_d3d11_window_do_processor (self, piv, pov); } else { - if (!gst_d3d11_converter_convert_unlocked (self->converter, - srv, &rtv, NULL, NULL)) { - GST_ERROR_OBJECT (self, - "Couldn't render to backbuffer using converter"); - ret = GST_FLOW_ERROR; - goto unmap_and_out; - } else { - GST_TRACE_OBJECT (self, "Rendered using converter"); - } + g_assert_not_reached (); + ret = GST_FLOW_ERROR; + goto unmap_and_out; + } + + if (!convert_ret) { + ret = GST_FLOW_ERROR; + goto unmap_and_out; } gst_d3d11_overlay_compositor_upload (self->compositor, buffer); diff --git a/sys/d3d11/gstd3d11window.h b/sys/d3d11/gstd3d11window.h index e50a72a85b..43dfd83d0f 100644 --- a/sys/d3d11/gstd3d11window.h +++ b/sys/d3d11/gstd3d11window.h @@ -102,6 +102,8 @@ struct _GstD3D11Window GstD3D11Converter *converter; GstD3D11OverlayCompositor *compositor; + gboolean processor_in_use; + /* calculated rect with aspect ratio and window area */ RECT render_rect;