mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-10-01 08:12:30 +00:00
d3d11videosink: Use ID3D11VideoProcessor interface
...for color space conversion if available ID3D11VideoProcessor is equivalent to DXVA-HD video processor which might use specialized blocks for video processing instead of general GPU resource. In addition to that feature, we need to use this API for color space conversion of DXVA2 decoder output memory, because any d3d11 texture arrays that were created with D3D11_BIND_DECODER cannot be used for shader resource. This is prework for d3d11decoder zero-copy rendering and also for conditional HDR tone-map support. Note that some Intel platform is known to support tone-mapping at the driver level using this API on Windows 10.
This commit is contained in:
Seungha Yang
GStreamer Merge Bot
parent
122a9b93eb
commit
7aad9187e4
4 changed files with 227 additions and 19 deletions
|
|
@ -27,6 +27,7 @@
|
|||
#include "gstd3d11device.h"
|
||||
#include "gstd3d11bufferpool.h"
|
||||
#include "gstd3d11format.h"
|
||||
#include "gstd3d11videoprocessor.h"
|
||||
|
||||
#if GST_D3D11_WINAPI_ONLY_APP
|
||||
#include "gstd3d11window_corewindow.h"
|
||||
|
|
@ -386,9 +387,10 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
|
|||
self->pending_render_rect = FALSE;
|
||||
GST_OBJECT_UNLOCK (self);
|
||||
|
||||
self->have_video_processor = FALSE;
|
||||
if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
|
||||
GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d,
|
||||
caps, &error)) {
|
||||
caps, &self->have_video_processor, &error)) {
|
||||
GstMessage *error_msg;
|
||||
|
||||
GST_ERROR_OBJECT (self, "cannot create swapchain");
|
||||
|
|
@ -412,15 +414,30 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
|
|||
|
||||
{
|
||||
GstD3D11AllocationParams *d3d11_params;
|
||||
gint bind_flags = D3D11_BIND_SHADER_RESOURCE;
|
||||
|
||||
if (self->have_video_processor) {
|
||||
/* To create video processor input view, one of following bind flags
|
||||
* is required
|
||||
* NOTE: Any texture arrays which were created with D3D11_BIND_DECODER flag
|
||||
* cannot be used for shader input.
|
||||
*
|
||||
* D3D11_BIND_DECODER
|
||||
* D3D11_BIND_VIDEO_ENCODER
|
||||
* D3D11_BIND_RENDER_TARGET
|
||||
* D3D11_BIND_UNORDERED_ACCESS_VIEW
|
||||
*/
|
||||
bind_flags |= D3D11_BIND_RENDER_TARGET;
|
||||
}
|
||||
|
||||
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
|
||||
if (!d3d11_params) {
|
||||
d3d11_params = gst_d3d11_allocation_params_new (self->device,
|
||||
&self->info, 0, D3D11_BIND_SHADER_RESOURCE);
|
||||
&self->info, 0, bind_flags);
|
||||
} else {
|
||||
/* Set bind flag */
|
||||
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) {
|
||||
d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
|
||||
d3d11_params->desc[i].BindFlags |= bind_flags;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -836,13 +853,6 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
|
|||
break;
|
||||
}
|
||||
|
||||
if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
|
||||
GST_LOG_OBJECT (sink,
|
||||
"shader resource view is unavailable, need fallback");
|
||||
render_buf = NULL;
|
||||
break;
|
||||
}
|
||||
|
||||
if (dmem->desc.Usage == D3D11_USAGE_DEFAULT) {
|
||||
if (!gst_memory_map (mem, &map, (GST_MAP_READ | GST_MAP_D3D11))) {
|
||||
GST_ERROR_OBJECT (self, "cannot map d3d11 memory");
|
||||
|
|
@ -851,6 +861,19 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
|
|||
|
||||
gst_memory_unmap (mem, &map);
|
||||
}
|
||||
|
||||
if (gst_buffer_n_memory (buf) == 1 && self->have_video_processor &&
|
||||
gst_d3d11_video_processor_check_bind_flags_for_input_view
|
||||
(dmem->desc.BindFlags)) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
|
||||
GST_LOG_OBJECT (sink,
|
||||
"shader resource view is unavailable, need fallback");
|
||||
render_buf = NULL;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!render_buf) {
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ struct _GstD3D11VideoSink
|
|||
|
||||
GstBufferPool *fallback_pool;
|
||||
gboolean can_convert;
|
||||
gboolean have_video_processor;
|
||||
};
|
||||
|
||||
struct _GstD3D11VideoSinkClass
|
||||
|
|
|
|||
|
|
@ -306,6 +306,11 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height)
|
|||
window->rtv = NULL;
|
||||
}
|
||||
|
||||
if (window->pov) {
|
||||
window->pov->Release ();
|
||||
window->pov = NULL;
|
||||
}
|
||||
|
||||
swap_chain->GetDesc (&swap_desc);
|
||||
hr = swap_chain->ResizeBuffers (0, width, height, DXGI_FORMAT_UNKNOWN,
|
||||
swap_desc.Flags);
|
||||
|
|
@ -367,6 +372,17 @@ gst_d3d11_window_on_resize (GstD3D11Window * window, guint width, guint height)
|
|||
goto done;
|
||||
}
|
||||
|
||||
if (window->processor) {
|
||||
D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC pov_desc;
|
||||
|
||||
pov_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
|
||||
pov_desc.Texture2D.MipSlice = 0;
|
||||
|
||||
if (!gst_d3d11_video_processor_create_output_view (window->processor,
|
||||
&pov_desc, (ID3D11Resource *) backbuffer, &window->pov))
|
||||
goto done;
|
||||
}
|
||||
|
||||
window->first_present = TRUE;
|
||||
|
||||
/* redraw the last scene if cached buffer exits */
|
||||
|
|
@ -406,11 +422,14 @@ gst_d3d11_window_on_mouse_event (GstD3D11Window * window, const gchar * event,
|
|||
|
||||
gboolean
|
||||
gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
||||
guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, GError ** error)
|
||||
guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps,
|
||||
gboolean * video_processor_available, GError ** error)
|
||||
{
|
||||
GstD3D11WindowClass *klass;
|
||||
GstCaps *render_caps;
|
||||
guint swapchain_flags = 0;
|
||||
gboolean need_processor_output_configure = FALSE;
|
||||
gboolean need_processor_input_configure = FALSE;
|
||||
|
||||
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
|
||||
g_return_val_if_fail (aspect_ratio_n > 0, FALSE);
|
||||
|
|
@ -453,6 +472,10 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
|||
|
||||
gst_video_info_from_caps (&window->info, caps);
|
||||
|
||||
if (window->processor)
|
||||
gst_d3d11_video_processor_free (window->processor);
|
||||
window->processor = NULL;
|
||||
|
||||
if (window->converter)
|
||||
gst_d3d11_color_converter_free (window->converter);
|
||||
window->converter = NULL;
|
||||
|
|
@ -469,6 +492,50 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
|||
window->info.colorimetry.primaries;
|
||||
window->render_info.colorimetry.transfer = window->info.colorimetry.transfer;
|
||||
|
||||
window->processor =
|
||||
gst_d3d11_video_processor_new (window->device, width, height, width,
|
||||
height);
|
||||
if (window->processor) {
|
||||
const GstD3D11Format *in_format;
|
||||
const GstD3D11Format *out_format;
|
||||
gboolean input_support = FALSE;
|
||||
gboolean out_support = FALSE;
|
||||
|
||||
in_format = gst_d3d11_device_format_from_gst (window->device,
|
||||
GST_VIDEO_INFO_FORMAT (&window->info));
|
||||
out_format = gst_d3d11_device_format_from_gst (window->device,
|
||||
GST_VIDEO_INFO_FORMAT (&window->render_info));
|
||||
|
||||
if (gst_d3d11_video_processor_supports_input_format (window->processor,
|
||||
in_format->dxgi_format)) {
|
||||
input_support = TRUE;
|
||||
} else {
|
||||
GST_DEBUG_OBJECT (window,
|
||||
"IVideoProcessor cannot support input dxgi format %d",
|
||||
in_format->dxgi_format);
|
||||
}
|
||||
|
||||
if (gst_d3d11_video_processor_supports_output_format (window->processor,
|
||||
out_format->dxgi_format)) {
|
||||
out_support = TRUE;
|
||||
} else {
|
||||
GST_DEBUG_OBJECT (window,
|
||||
"IVideoProcessor cannot support output dxgi format %d",
|
||||
out_format->dxgi_format);
|
||||
}
|
||||
|
||||
if (!input_support || !out_support) {
|
||||
gst_d3d11_video_processor_free (window->processor);
|
||||
window->processor = NULL;
|
||||
} else {
|
||||
GST_DEBUG_OBJECT (window, "IVideoProcessor interface available");
|
||||
*video_processor_available = TRUE;
|
||||
need_processor_input_configure = TRUE;
|
||||
need_processor_output_configure = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
/* configure shader even if video processor is available for fallback */
|
||||
window->converter =
|
||||
gst_d3d11_color_converter_new (window->device, &window->info,
|
||||
&window->render_info);
|
||||
|
|
@ -512,6 +579,8 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
|||
window->render_rect.right = width;
|
||||
window->render_rect.bottom = height;
|
||||
|
||||
window->input_rect = window->render_rect;
|
||||
|
||||
window->width = width;
|
||||
window->height = height;
|
||||
|
||||
|
|
@ -544,14 +613,46 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
|||
ctype, (guint) hr);
|
||||
} else {
|
||||
GST_DEBUG_OBJECT (window, "Set colorspace %d", ctype);
|
||||
|
||||
if (window->processor)
|
||||
need_processor_output_configure =
|
||||
!gst_d3d11_video_processor_set_output_dxgi_color_space
|
||||
(window->processor, ctype);
|
||||
}
|
||||
}
|
||||
|
||||
swapchain3->Release ();
|
||||
}
|
||||
}
|
||||
|
||||
if (window->processor) {
|
||||
if (need_processor_output_configure) {
|
||||
/* Set most common color space */
|
||||
need_processor_output_configure =
|
||||
!gst_d3d11_video_processor_set_output_dxgi_color_space
|
||||
(window->processor, DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709);
|
||||
}
|
||||
|
||||
if (need_processor_input_configure) {
|
||||
DXGI_COLOR_SPACE_TYPE ctype;
|
||||
gst_d3d11_video_info_to_dxgi_color_space (&window->info, &ctype);
|
||||
need_processor_input_configure =
|
||||
!gst_d3d11_video_processor_set_input_dxgi_color_space
|
||||
(window->processor, ctype);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (window->processor && need_processor_output_configure) {
|
||||
gst_d3d11_video_processor_set_output_color_space (window->processor,
|
||||
&window->render_info.colorimetry);
|
||||
}
|
||||
|
||||
if (window->processor && need_processor_input_configure) {
|
||||
gst_d3d11_video_processor_set_input_color_space (window->processor,
|
||||
&window->info.colorimetry);
|
||||
}
|
||||
|
||||
#if (DXGI_HEADER_VERSION >= 5)
|
||||
{
|
||||
GstVideoMasteringDisplayInfo minfo;
|
||||
|
|
@ -576,6 +677,11 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
|
|||
if (!gst_d3d11_result (hr, window->device)) {
|
||||
GST_WARNING_OBJECT (window, "Couldn't set HDR metadata, hr 0x%x",
|
||||
(guint) hr);
|
||||
} else if (window->processor) {
|
||||
gst_d3d11_video_processor_set_input_hdr10_metadata (window->processor,
|
||||
&metadata);
|
||||
gst_d3d11_video_processor_set_output_hdr10_metadata (window->processor,
|
||||
&metadata);
|
||||
}
|
||||
|
||||
swapchain4->Release ();
|
||||
|
|
@ -615,6 +721,58 @@ gst_d3d11_window_set_render_rectangle (GstD3D11Window * window, gint x, gint y,
|
|||
/* TODO: resize window and view */
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_d3d11_window_buffer_ensure_processor_input (GstD3D11Window * self,
|
||||
GstBuffer * buffer, ID3D11VideoProcessorInputView ** in_view)
|
||||
{
|
||||
GstD3D11Memory *mem;
|
||||
ID3D11VideoProcessorInputView *view;
|
||||
GQuark quark;
|
||||
|
||||
if (!self->processor)
|
||||
return FALSE;
|
||||
|
||||
if (gst_buffer_n_memory (buffer) != 1)
|
||||
return FALSE;
|
||||
|
||||
mem = (GstD3D11Memory *) gst_buffer_peek_memory (buffer, 0);
|
||||
|
||||
if (!gst_d3d11_video_processor_check_bind_flags_for_input_view
|
||||
(mem->desc.BindFlags)) {
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
quark = gst_d3d11_video_processor_input_view_quark ();
|
||||
view = (ID3D11VideoProcessorInputView *)
|
||||
gst_mini_object_get_qdata (GST_MINI_OBJECT (mem), quark);
|
||||
|
||||
if (!view) {
|
||||
D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC in_desc;
|
||||
|
||||
in_desc.FourCC = 0;
|
||||
in_desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D;
|
||||
in_desc.Texture2D.MipSlice = 0;
|
||||
in_desc.Texture2D.ArraySlice = mem->subresource_index;
|
||||
|
||||
GST_TRACE_OBJECT (self, "Create new processor input view");
|
||||
|
||||
if (!gst_d3d11_video_processor_create_input_view (self->processor,
|
||||
&in_desc, mem->texture, &view)) {
|
||||
GST_LOG_OBJECT (self, "Failed to create processor input view");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
gst_mini_object_set_qdata (GST_MINI_OBJECT (mem), quark, view,
|
||||
(GDestroyNotify) gst_d3d11_video_processor_input_view_release);
|
||||
} else {
|
||||
GST_TRACE_OBJECT (self, "Reuse existing processor input view %p", view);
|
||||
}
|
||||
|
||||
*in_view = view;
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static GstFlowReturn
|
||||
gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
|
||||
{
|
||||
|
|
@ -628,8 +786,11 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
|
|||
|
||||
if (self->cached_buffer) {
|
||||
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES];
|
||||
ID3D11VideoProcessorInputView *piv = NULL;
|
||||
guint i, j, k;
|
||||
|
||||
if (!gst_d3d11_window_buffer_ensure_processor_input (self,
|
||||
self->cached_buffer, &piv)) {
|
||||
for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) {
|
||||
GstD3D11Memory *mem =
|
||||
(GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
|
||||
|
|
@ -638,6 +799,7 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
|
|||
j++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (self->first_present) {
|
||||
gst_d3d11_color_converter_update_rect (self->converter,
|
||||
|
|
@ -646,8 +808,23 @@ gst_d3d111_window_present (GstD3D11Window * self, GstBuffer * buffer)
|
|||
&self->render_rect);
|
||||
}
|
||||
|
||||
gst_d3d11_color_converter_convert_unlocked (self->converter,
|
||||
srv, &self->rtv);
|
||||
if (self->processor && piv && self->pov) {
|
||||
if (!gst_d3d11_video_processor_render_unlocked (self->processor,
|
||||
&self->input_rect, piv, &self->render_rect, self->pov)) {
|
||||
GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using processor");
|
||||
return GST_FLOW_ERROR;
|
||||
} else {
|
||||
GST_TRACE_OBJECT (self, "Rendered using processor");
|
||||
}
|
||||
} else {
|
||||
if (!gst_d3d11_color_converter_convert_unlocked (self->converter,
|
||||
srv, &self->rtv)) {
|
||||
GST_ERROR_OBJECT (self, "Couldn't render to backbuffer using converter");
|
||||
return GST_FLOW_ERROR;
|
||||
} else {
|
||||
GST_TRACE_OBJECT (self, "Rendered using converter");
|
||||
}
|
||||
}
|
||||
|
||||
gst_d3d11_overlay_compositor_upload (self->compositor, self->cached_buffer);
|
||||
gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &self->rtv);
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
#include "gstd3d11_fwd.h"
|
||||
#include "gstd3d11colorconverter.h"
|
||||
#include "gstd3d11overlaycompositor.h"
|
||||
#include "gstd3d11videoprocessor.h"
|
||||
|
||||
G_BEGIN_DECLS
|
||||
|
||||
|
|
@ -79,12 +80,16 @@ struct _GstD3D11Window
|
|||
GstVideoInfo info;
|
||||
GstVideoInfo render_info;
|
||||
const GstD3D11Format *render_format;
|
||||
GstD3D11VideoProcessor *processor;
|
||||
GstD3D11ColorConverter *converter;
|
||||
GstD3D11OverlayCompositor *compositor;
|
||||
|
||||
/* calculated rect with aspect ratio and window area */
|
||||
RECT render_rect;
|
||||
|
||||
/* input resolution */
|
||||
RECT input_rect;
|
||||
|
||||
/* requested rect via gst_d3d11_window_render */
|
||||
GstVideoRectangle rect;
|
||||
|
||||
|
|
@ -99,6 +104,7 @@ struct _GstD3D11Window
|
|||
|
||||
IDXGISwapChain *swap_chain;
|
||||
ID3D11RenderTargetView *rtv;
|
||||
ID3D11VideoProcessorOutputView *pov;
|
||||
|
||||
GstBuffer *cached_buffer;
|
||||
gboolean first_present;
|
||||
|
|
@ -144,6 +150,7 @@ gboolean gst_d3d11_window_prepare (GstD3D11Window * window,
|
|||
guint aspect_ratio_n,
|
||||
guint aspect_ratio_d,
|
||||
GstCaps * caps,
|
||||
gboolean * video_processor_available,
|
||||
GError ** error);
|
||||
|
||||
GstFlowReturn gst_d3d11_window_render (GstD3D11Window * window,
|
||||
|
|
|
|||
Loading…
Reference in a new issue