From 7aad9187e4201f3099113809847cfd1917e4a927 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 29 Jan 2020 21:10:00 +0900 Subject: [PATCH] d3d11videosink: Use ID3D11VideoProcessor interface ...for color space conversion if available ID3D11VideoProcessor is equivalent to DXVA-HD video processor which might use specialized blocks for video processing instead of general GPU resource. In addition to that feature, we need to use this API for color space conversion of DXVA2 decoder output memory, because any d3d11 texture arrays that were created with D3D11_BIND_DECODER cannot be used for shader resource. This is prework for d3d11decoder zero-copy rendering and also for conditional HDR tone-map support. Note that some Intel platform is known to support tone-mapping at the driver level using this API on Windows 10. --- sys/d3d11/gstd3d11videosink.c | 43 ++++++-- sys/d3d11/gstd3d11videosink.h | 1 + sys/d3d11/gstd3d11window.cpp | 195 ++++++++++++++++++++++++++++++++-- sys/d3d11/gstd3d11window.h | 7 ++ 4 files changed, 227 insertions(+), 19 deletions(-) diff --git a/sys/d3d11/gstd3d11videosink.c b/sys/d3d11/gstd3d11videosink.c index 59ef022543..49a87c9ba9 100644 --- a/sys/d3d11/gstd3d11videosink.c +++ b/sys/d3d11/gstd3d11videosink.c @@ -27,6 +27,7 @@ #include "gstd3d11device.h" #include "gstd3d11bufferpool.h" #include "gstd3d11format.h" +#include "gstd3d11videoprocessor.h" #if GST_D3D11_WINAPI_ONLY_APP #include "gstd3d11window_corewindow.h" @@ -386,9 +387,10 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps) self->pending_render_rect = FALSE; GST_OBJECT_UNLOCK (self); + self->have_video_processor = FALSE; if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self), GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d, - caps, &error)) { + caps, &self->have_video_processor, &error)) { GstMessage *error_msg; GST_ERROR_OBJECT (self, "cannot create swapchain"); @@ -412,15 +414,30 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps) { GstD3D11AllocationParams *d3d11_params; + gint bind_flags = D3D11_BIND_SHADER_RESOURCE; + + if (self->have_video_processor) { + /* To create video processor input view, one of following bind flags + * is required + * NOTE: Any texture arrays which were created with D3D11_BIND_DECODER flag + * cannot be used for shader input. + * + * D3D11_BIND_DECODER + * D3D11_BIND_VIDEO_ENCODER + * D3D11_BIND_RENDER_TARGET + * D3D11_BIND_UNORDERED_ACCESS_VIEW + */ + bind_flags |= D3D11_BIND_RENDER_TARGET; + } d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); if (!d3d11_params) { d3d11_params = gst_d3d11_allocation_params_new (self->device, - &self->info, 0, D3D11_BIND_SHADER_RESOURCE); + &self->info, 0, bind_flags); } else { /* Set bind flag */ for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) { - d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE; + d3d11_params->desc[i].BindFlags |= bind_flags; } } @@ -836,13 +853,6 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf) break; } - if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) { - GST_LOG_OBJECT (sink, - "shader resource view is unavailable, need fallback"); - render_buf = NULL; - break; - } - if (dmem->desc.Usage == D3D11_USAGE_DEFAULT) { if (!gst_memory_map (mem, &map, (GST_MAP_READ | GST_MAP_D3D11))) { GST_ERROR_OBJECT (self, "cannot map d3d11 memory"); @@ -851,6 +861,19 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf) gst_memory_unmap (mem, &map); } + + if (gst_buffer_n_memory (buf) == 1 && self->have_video_processor && + gst_d3d11_video_processor_check_bind_flags_for_input_view + (dmem->desc.BindFlags)) { + break; + } + + if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) { + GST_LOG_OBJECT (sink, + "shader resource view is unavailable, need fallback"); + render_buf = NULL; + break; + } } if (!render_buf) { diff --git a/sys/d3d11/gstd3d11videosink.h b/sys/d3d11/gstd3d11videosink.h index c443906f5c..c599f930fe 100644 --- a/sys/d3d11/gstd3d11videosink.h +++ b/sys/d3d11/gstd3d11videosink.h @@ -66,6 +66,7 @@ struct _GstD3D11VideoSink GstBufferPool *fallback_pool; gboolean can_convert; + gboolean have_video_processor; }; struct _GstD3D11VideoSinkClass diff --git a/sys/d3d11/gstd3d11window.cpp b/sys/d3d11/gstd3d11window.cpp index 922c787df0..96f26ef5ce 100644 --- a/sys/d3d11/gstd3d11window.cpp +++ b/sys/d3d11/gstd3d11window.cpp @@ -306,6 +306,11 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height) window->rtv = NULL; } + if (window->pov) { + window->pov->Release (); + window->pov = NULL; + } + swap_chain->GetDesc (&swap_desc); hr = swap_chain->ResizeBuffers (0, width, height, DXGI_FORMAT_UNKNOWN, swap_desc.Flags); @@ -367,6 +372,17 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height) goto done; } + if (window->processor) { + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc; + + pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + pov_desc.Texture2D.MipSlice = 0; + + if (!gst_d3d11_video_processor_create_output_view (window->processor, + &pov_desc, (ID3D11Resource *) backbuffer, &window->pov)) + goto done; + } + window->first_present = TRUE; /* redraw the last scene if cached buffer exits */ @@ -406,11 +422,14 @@ gst_d3d11_window_on_mouse_event (GstD3D11Window * window, const gchar * event, gboolean gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, - guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, GError ** error) + guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, + gboolean * video_processor_available, GError ** error) { GstD3D11WindowClass *klass; GstCaps *render_caps; guint swapchain_flags = 0; + gboolean need_processor_output_configure = FALSE; + gboolean need_processor_input_configure = FALSE; g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE); g_return_val_if_fail (aspect_ratio_n > 0, FALSE); @@ -453,6 +472,10 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, gst_video_info_from_caps (&window->info, caps); + if (window->processor) + gst_d3d11_video_processor_free (window->processor); + window->processor = NULL; + if (window->converter) gst_d3d11_color_converter_free (window->converter); window->converter = NULL; @@ -469,6 +492,50 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, window->info.colorimetry.primaries; window->render_info.colorimetry.transfer = window->info.colorimetry.transfer; + window->processor = + gst_d3d11_video_processor_new (window->device, width, height, width, + height); + if (window->processor) { + const GstD3D11Format *in_format; + const GstD3D11Format *out_format; + gboolean input_support = FALSE; + gboolean out_support = FALSE; + + in_format = gst_d3d11_device_format_from_gst (window->device, + GST_VIDEO_INFO_FORMAT (&window->info)); + out_format = gst_d3d11_device_format_from_gst (window->device, + GST_VIDEO_INFO_FORMAT (&window->render_info)); + + if (gst_d3d11_video_processor_supports_input_format (window->processor, + in_format->dxgi_format)) { + input_support = TRUE; + } else { + GST_DEBUG_OBJECT (window, + "IVideoProcessor cannot support input dxgi format %d", + in_format->dxgi_format); + } + + if (gst_d3d11_video_processor_supports_output_format (window->processor, + out_format->dxgi_format)) { + out_support = TRUE; + } else { + GST_DEBUG_OBJECT (window, + "IVideoProcessor cannot support output dxgi format %d", + out_format->dxgi_format); + } + + if (!input_support || !out_support) { + gst_d3d11_video_processor_free (window->processor); + window->processor = NULL; + } else { + GST_DEBUG_OBJECT (window, "IVideoProcessor interface available"); + *video_processor_available = TRUE; + need_processor_input_configure = TRUE; + need_processor_output_configure = TRUE; + } + } + + /* configure shader even if video processor is available for fallback */ window->converter = gst_d3d11_color_converter_new (window->device, &window->info, &window->render_info); @@ -512,6 +579,8 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, window->render_rect.right = width; window->render_rect.bottom = height; + window->input_rect = window->render_rect; + window->width = width; window->height = height; @@ -544,14 +613,46 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, ctype, (guint) hr); } else { GST_DEBUG_OBJECT (window, "Set colorspace %d", ctype); + + if (window->processor) + need_processor_output_configure = + !gst_d3d11_video_processor_set_output_dxgi_color_space + (window->processor, ctype); } } swapchain3->Release (); } } + + if (window->processor) { + if (need_processor_output_configure) { + /* Set most common color space */ + need_processor_output_configure = + !gst_d3d11_video_processor_set_output_dxgi_color_space + (window->processor, DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709); + } + + if (need_processor_input_configure) { + DXGI_COLOR_SPACE_TYPE ctype; + gst_d3d11_video_info_to_dxgi_color_space (&window->info, &ctype); + need_processor_input_configure = + !gst_d3d11_video_processor_set_input_dxgi_color_space + (window->processor, ctype); + } + } #endif + if (window->processor && need_processor_output_configure) { + gst_d3d11_video_processor_set_output_color_space (window->processor, + &window->render_info.colorimetry); + } + + if (window->processor && need_processor_input_configure) { + gst_d3d11_video_processor_set_input_color_space (window->processor, + &window->info.colorimetry); + } + #if (DXGI_HEADER_VERSION >= 5) { GstVideoMasteringDisplayInfo minfo; @@ -576,6 +677,11 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, if (!gst_d3d11_result (hr, window->device)) { GST_WARNING_OBJECT (window, "Couldn't set HDR metadata, hr 0x%x", (guint) hr); + } else if (window->processor) { + gst_d3d11_video_processor_set_input_hdr10_metadata (window->processor, + &metadata); + gst_d3d11_video_processor_set_output_hdr10_metadata (window->processor, + &metadata); } swapchain4->Release (); @@ -615,6 +721,58 @@ gst_d3d11_window_set_render_rectangle (GstD3D11Window * window, gint x, gint y, /* TODO: resize window and view */ } +static gboolean +gst_d3d11_window_buffer_ensure_processor_input (GstD3D11Window * self, + GstBuffer * buffer, ID3D11VideoProcessorInputView ** in_view) +{ + GstD3D11Memory *mem; + ID3D11VideoProcessorInputView *view; + GQuark quark; + + if (!self->processor) + return FALSE; + + if (gst_buffer_n_memory (buffer) != 1) + return FALSE; + + mem = (GstD3D11Memory *) gst_buffer_peek_memory (buffer, 0); + + if (!gst_d3d11_video_processor_check_bind_flags_for_input_view + (mem->desc.BindFlags)) { + return FALSE; + } + + quark = gst_d3d11_video_processor_input_view_quark (); + view = (ID3D11VideoProcessorInputView *) + gst_mini_object_get_qdata (GST_MINI_OBJECT (mem), quark); + + if (!view) { + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC in_desc; + + in_desc.FourCC = 0; + in_desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + in_desc.Texture2D.MipSlice = 0; + in_desc.Texture2D.ArraySlice = mem->subresource_index; + + GST_TRACE_OBJECT (self, "Create new processor input view"); + + if (!gst_d3d11_video_processor_create_input_view (self->processor, + &in_desc, mem->texture, &view)) { + GST_LOG_OBJECT (self, "Failed to create processor input view"); + return FALSE; + } + + gst_mini_object_set_qdata (GST_MINI_OBJECT (mem), quark, view, + (GDestroyNotify) gst_d3d11_video_processor_input_view_release); + } else { + GST_TRACE_OBJECT (self, "Reuse existing processor input view %p", view); + } + + *in_view = view; + + return TRUE; +} + static GstFlowReturn gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) { @@ -628,14 +786,18 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) if (self->cached_buffer) { ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES]; + ID3D11VideoProcessorInputView *piv = NULL; guint i, j, k; - for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) { - GstD3D11Memory *mem = - (GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i); - for (k = 0; k < mem->num_shader_resource_views; k++) { - srv[j] = mem->shader_resource_view[k]; - j++; + if (!gst_d3d11_window_buffer_ensure_processor_input (self, + self->cached_buffer, &piv)) { + for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) { + GstD3D11Memory *mem = + (GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i); + for (k = 0; k < mem->num_shader_resource_views; k++) { + srv[j] = mem->shader_resource_view[k]; + j++; + } } } @@ -646,8 +808,23 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer) &self->render_rect); } - gst_d3d11_color_converter_convert_unlocked (self->converter, - srv, &self->rtv); + if (self->processor && piv && self->pov) { + if (!gst_d3d11_video_processor_render_unlocked (self->processor, + &self->input_rect, piv, &self->render_rect, self->pov)) { + GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using processor"); + return GST_FLOW_ERROR; + } else { + GST_TRACE_OBJECT (self, "Rendered using processor"); + } + } else { + if (!gst_d3d11_color_converter_convert_unlocked (self->converter, + srv, &self->rtv)) { + GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using converter"); + return GST_FLOW_ERROR; + } else { + GST_TRACE_OBJECT (self, "Rendered using converter"); + } + } gst_d3d11_overlay_compositor_upload (self->compositor, self->cached_buffer); gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &self->rtv); diff --git a/sys/d3d11/gstd3d11window.h b/sys/d3d11/gstd3d11window.h index 2d743d3c34..60ba7b8bba 100644 --- a/sys/d3d11/gstd3d11window.h +++ b/sys/d3d11/gstd3d11window.h @@ -27,6 +27,7 @@ #include "gstd3d11_fwd.h" #include "gstd3d11colorconverter.h" #include "gstd3d11overlaycompositor.h" +#include "gstd3d11videoprocessor.h" G_BEGIN_DECLS @@ -79,12 +80,16 @@ struct _GstD3D11Window GstVideoInfo info; GstVideoInfo render_info; const GstD3D11Format *render_format; + GstD3D11VideoProcessor *processor; GstD3D11ColorConverter *converter; GstD3D11OverlayCompositor *compositor; /* calculated rect with aspect ratio and window area */ RECT render_rect; + /* input resolution */ + RECT input_rect; + /* requested rect via gst_d3d11_window_render */ GstVideoRectangle rect; @@ -99,6 +104,7 @@ struct _GstD3D11Window IDXGISwapChain *swap_chain; ID3D11RenderTargetView *rtv; + ID3D11VideoProcessorOutputView *pov; GstBuffer *cached_buffer; gboolean first_present; @@ -144,6 +150,7 @@ gboolean gst_d3d11_window_prepare (GstD3D11Window * window, guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, + gboolean * video_processor_available, GError ** error); GstFlowReturn gst_d3d11_window_render (GstD3D11Window * window,