d3d11videosink: Add color conversion support

Draw to back buffer texture directly. It would reduce the number of
copy at least once when color conversion is required.
This commit is contained in:
Seungha Yang 2019-12-03 22:54:26 +09:00 committed by GStreamer Merge Bot
parent 622733ed0d
commit ca3ddf7848
6 changed files with 244 additions and 97 deletions

View file

@ -943,3 +943,17 @@ gst_d3d11_color_converter_convert (GstD3D11ColorConverter * converter,
return data.ret;
}
gboolean
gst_d3d11_color_converter_update_rect (GstD3D11ColorConverter * converter,
RECT * rect)
{
g_return_val_if_fail (converter != NULL, FALSE);
converter->viewport.TopLeftX = rect->left;
converter->viewport.TopLeftY = rect->top;
converter->viewport.Width = rect->right - rect->left;
converter->viewport.Height = rect->bottom - rect->top;
return TRUE;
}

View file

@ -38,6 +38,9 @@ gboolean gst_d3d11_color_converter_convert (GstD3D11ColorConvert
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES],
ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES]);
gboolean gst_d3d11_color_converter_update_rect (GstD3D11ColorConverter * converter,
RECT *rect);
G_END_DECLS
#endif /* __GST_D3D11_COLOR_CONVERTER_H__ */

View file

@ -26,6 +26,7 @@
#include "gstd3d11utils.h"
#include "gstd3d11device.h"
#include "gstd3d11bufferpool.h"
#include "gstd3d11format.h"
enum
{
@ -39,13 +40,11 @@ enum
#define DEFAULT_FORCE_ASPECT_RATIO TRUE
#define DEFAULT_ENABLE_NAVIGATION_EVENTS TRUE
#define CAPS_FORMAT "{ BGRA, RGBA, RGB10A2_LE }"
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, CAPS_FORMAT)
(GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_FORMATS)
));
GST_DEBUG_CATEGORY (d3d11_video_sink_debug);
@ -221,10 +220,9 @@ gst_d3d11_video_sink_get_caps (GstBaseSink * sink, GstCaps * filter)
GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
GstCaps *caps = NULL;
if (self->device)
if (self->device && !self->can_convert)
caps = gst_d3d11_device_get_supported_caps (self->device,
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY |
D3D11_FORMAT_SUPPORT_RENDER_TARGET);
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY);
if (!caps)
caps = gst_pad_get_pad_template_caps (GST_VIDEO_SINK_PAD (sink));
@ -243,36 +241,20 @@ static gboolean
gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
{
GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
GstCaps *sink_caps = NULL;
gint video_width, video_height;
gint video_par_n, video_par_d; /* video's PAR */
gint display_par_n = 1, display_par_d = 1; /* display's PAR */
guint num, den;
GError *error = NULL;
const GstD3D11Format *d3d11_format = NULL;
GstStructure *config;
GstD3D11AllocationParams *d3d11_params;
gint i;
GST_DEBUG_OBJECT (self, "set caps %" GST_PTR_FORMAT, caps);
sink_caps = gst_d3d11_device_get_supported_caps (self->device,
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY |
D3D11_FORMAT_SUPPORT_RENDER_TARGET);
GST_DEBUG_OBJECT (self, "supported caps %" GST_PTR_FORMAT, sink_caps);
if (!gst_caps_can_intersect (sink_caps, caps))
goto incompatible_caps;
gst_clear_caps (&sink_caps);
if (!gst_video_info_from_caps (&self->info, caps))
goto invalid_format;
d3d11_format =
gst_d3d11_format_from_gst (GST_VIDEO_INFO_FORMAT (&self->info));
if (!d3d11_format || d3d11_format->dxgi_format == DXGI_FORMAT_UNKNOWN)
goto invalid_format;
video_width = GST_VIDEO_INFO_WIDTH (&self->info);
video_height = GST_VIDEO_INFO_HEIGHT (&self->info);
video_par_n = GST_VIDEO_INFO_PAR_N (&self->info);
@ -326,8 +308,6 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
if (GST_VIDEO_SINK_WIDTH (self) <= 0 || GST_VIDEO_SINK_HEIGHT (self) <= 0)
goto no_display_size;
self->dxgi_format = d3d11_format->dxgi_format;
if (!self->window_id)
gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (self));
@ -359,7 +339,7 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d,
self->dxgi_format, caps, &error)) {
caps, &error)) {
GstMessage *error_msg;
GST_ERROR_OBJECT (self, "cannot create swapchain");
@ -380,17 +360,26 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
config = gst_buffer_pool_get_config (self->fallback_pool);
gst_buffer_pool_config_set_params (config,
caps, GST_VIDEO_INFO_SIZE (&self->info), 0, 2);
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (&self->info,
GST_D3D11_ALLOCATION_FLAG_USE_RESOURCE_FORMAT, D3D11_USAGE_DEFAULT,
D3D11_BIND_SHADER_RESOURCE);
} else {
/* Set bind flag */
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&self->info); i++) {
d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
}
}
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
gst_buffer_pool_set_config (self->fallback_pool, config);
return TRUE;
/* ERRORS */
incompatible_caps:
{
GST_ERROR_OBJECT (sink, "caps incompatible");
gst_clear_caps (&sink_caps);
return FALSE;
}
invalid_format:
{
GST_DEBUG_OBJECT (sink,
@ -447,6 +436,7 @@ static gboolean
gst_d3d11_video_sink_start (GstBaseSink * sink)
{
GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
gboolean is_hardware = TRUE;
GST_DEBUG_OBJECT (self, "Start");
@ -462,6 +452,15 @@ gst_d3d11_video_sink_start (GstBaseSink * sink)
return FALSE;
}
g_object_get (self->device, "hardware", &is_hardware, NULL);
if (!is_hardware) {
GST_WARNING_OBJECT (self, "D3D11 device is running on software emulation");
self->can_convert = FALSE;
} else {
self->can_convert = TRUE;
}
g_object_set (self->window,
"enable-navigation-events", self->enable_navigation_events, NULL);
@ -520,6 +519,9 @@ gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query)
size = info.size;
if (need_pool) {
GstD3D11AllocationParams *d3d11_params;
gint i;
GST_DEBUG_OBJECT (self, "create new pool");
pool = gst_d3d11_buffer_pool_new (self->device);
@ -527,6 +529,21 @@ gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query)
gst_buffer_pool_config_set_params (config, caps, size, 2,
DXGI_MAX_SWAP_CHAIN_BUFFERS);
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (&info,
GST_D3D11_ALLOCATION_FLAG_USE_RESOURCE_FORMAT, D3D11_USAGE_DEFAULT,
D3D11_BIND_SHADER_RESOURCE);
} else {
/* Set bind flag */
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
}
}
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
if (!gst_buffer_pool_set_config (pool, config)) {
g_object_unref (pool);
goto config_failed;
@ -612,19 +629,34 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
GstD3D11VideoSink *self = GST_D3D11_VIDEO_SINK (sink);
GstMapInfo map;
GstFlowReturn ret;
GstMemory *mem;
GstVideoRectangle rect = { 0, };
GstVideoCropMeta *crop;
GstBuffer *render_buf;
gboolean need_unref = FALSE;
gint i;
if (gst_buffer_n_memory (buf) == 1 && (mem = gst_buffer_peek_memory (buf, 0))
&& gst_memory_is_type (mem, GST_D3D11_MEMORY_NAME)) {
GstD3D11Memory *dmem = (GstD3D11Memory *) mem;
render_buf = buf;
/* If this buffer has been allocated using our buffer management we simply
put the ximage which is in the PRIVATE pointer */
GST_TRACE_OBJECT (self, "buffer %p from our pool, writing directly", buf);
for (i = 0; i < gst_buffer_n_memory (buf); i++) {
GstMemory *mem;
GstD3D11Memory *dmem;
mem = gst_buffer_peek_memory (buf, i);
if (!gst_is_d3d11_memory (mem)) {
render_buf = NULL;
break;
}
dmem = (GstD3D11Memory *) mem;
if (dmem->device != self->device) {
render_buf = NULL;
break;
}
if (!gst_d3d11_memory_ensure_shader_resource_view (mem)) {
render_buf = NULL;
break;
}
if (dmem->desc.Usage == D3D11_USAGE_DEFAULT) {
if (!gst_memory_map (mem, &map, (GST_MAP_READ | GST_MAP_D3D11))) {
@ -634,9 +666,9 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
gst_memory_unmap (mem, &map);
}
}
render_buf = buf;
} else {
if (!render_buf) {
GstVideoFrame frame, fallback_frame;
if (!self->fallback_pool ||

View file

@ -50,7 +50,6 @@ struct _GstD3D11VideoSink
gint video_height;
GstVideoInfo info;
DXGI_FORMAT dxgi_format;
guintptr window_id;
@ -64,6 +63,7 @@ struct _GstD3D11VideoSink
gboolean pending_render_rect;
GstBufferPool *fallback_pool;
gboolean can_convert;
};
struct _GstD3D11VideoSinkClass

View file

@ -27,6 +27,7 @@
#include "gstd3d11window.h"
#include "gstd3d11device.h"
#include "gstd3d11memory.h"
#include "gstd3d11utils.h"
#include <windows.h>
@ -258,6 +259,11 @@ gst_d3d11_window_dispose (GObject * object)
(GstD3D11DeviceThreadFunc) gst_d3d11_window_release_resources, self);
}
if (self->converter) {
gst_d3d11_color_converter_free (self->converter);
self->converter = NULL;
}
gst_clear_buffer (&self->cached_buffer);
gst_clear_object (&self->device);
@ -521,33 +527,54 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
window->rtv = NULL;
}
width = window->width;
height = window->height;
if (width != window->surface_width || height != window->surface_height) {
GstVideoRectangle src_rect, dst_rect;
src_rect.x = 0;
src_rect.y = 0;
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
dst_rect.x = 0;
dst_rect.y = 0;
dst_rect.w = window->surface_width;
dst_rect.h = window->surface_height;
if (window->converter) {
if (window->force_aspect_ratio) {
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect,
TRUE);
} else {
window->render_rect = dst_rect;
}
width = window->surface_width;
height = window->surface_height;
} else {
/* NOTE: there can be various way to resize texture, but
* we just copy incoming texture toward resized swap chain buffer in order to
* avoid shader coding.
* To keep aspect ratio, required vertical or horizontal padding area
* will be calculated in here.
*/
width = window->width;
height = window->height;
if (width != window->surface_width || height != window->surface_height) {
GstVideoRectangle src_rect, dst_rect;
gdouble src_ratio, dst_ratio;
gdouble aspect_ratio =
(gdouble) window->aspect_ratio_n / (gdouble) window->aspect_ratio_d;
src_ratio = (gdouble) width / height;
dst_ratio =
(gdouble) window->surface_width / window->surface_height / aspect_ratio;
(gdouble) window->surface_width / window->surface_height /
aspect_ratio;
src_rect.x = 0;
src_rect.y = 0;
src_rect.w = width;
src_rect.h = height;
dst_rect.x = 0;
dst_rect.y = 0;
if (window->force_aspect_ratio) {
if (src_ratio > dst_ratio) {
/* padding top and bottom */
@ -563,11 +590,13 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
dst_rect.h = height;
}
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect, TRUE);
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect,
TRUE);
width = dst_rect.w;
height = dst_rect.h;
}
}
hr = IDXGISwapChain_ResizeBuffers (window->swap_chain,
0, width, height, DXGI_FORMAT_UNKNOWN, 0);
@ -824,21 +853,63 @@ mastering_display_gst_to_dxgi (GstVideoMasteringDisplayInfo * m,
gboolean
gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
guint aspect_ratio_n, guint aspect_ratio_d, DXGI_FORMAT format,
GstCaps * caps, GError ** error)
guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, GError ** error)
{
DXGI_SWAP_CHAIN_DESC desc = { 0, };
gboolean have_cll = FALSE;
gboolean have_mastering = FALSE;
gboolean hdr_api_available = FALSE;
GstD3D11ThreadFuncData data;
GstCaps *render_caps;
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
g_return_val_if_fail (aspect_ratio_n > 0, FALSE);
g_return_val_if_fail (aspect_ratio_d > 0, FALSE);
GST_DEBUG_OBJECT (window, "Prepare window with %dx%d format %d",
width, height, format);
GST_DEBUG_OBJECT (window, "Prepare window with %dx%d caps %" GST_PTR_FORMAT,
width, height, caps);
render_caps = gst_d3d11_device_get_supported_caps (window->device,
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY);
GST_DEBUG_OBJECT (window, "rendering caps %" GST_PTR_FORMAT, render_caps);
render_caps = gst_d3d11_caps_fixate_format (caps, render_caps);
if (!render_caps || gst_caps_is_empty (render_caps)) {
GST_ERROR_OBJECT (window, "Couldn't define render caps");
gst_clear_caps (&render_caps);
return FALSE;
}
render_caps = gst_caps_fixate (render_caps);
gst_video_info_from_caps (&window->render_info, render_caps);
gst_clear_caps (&render_caps);
window->render_format =
gst_d3d11_format_from_gst (GST_VIDEO_INFO_FORMAT (&window->render_info));
if (!window->render_format ||
window->render_format->dxgi_format == DXGI_FORMAT_UNKNOWN) {
GST_ERROR_OBJECT (window, "Unknown dxgi render format");
return FALSE;
}
gst_video_info_from_caps (&window->info, caps);
if (window->converter)
gst_d3d11_color_converter_free (window->converter);
window->converter = NULL;
if (GST_VIDEO_INFO_FORMAT (&window->info) !=
GST_VIDEO_INFO_FORMAT (&window->render_info)) {
window->converter =
gst_d3d11_color_converter_new (window->device, &window->info,
&window->render_info);
if (!window->converter) {
GST_ERROR_OBJECT (window, "Cannot create converter");
return FALSE;
}
}
data.self = window;
data.error = error;
@ -861,7 +932,6 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
window->thread = NULL;
}
gst_video_info_from_caps (&window->info, caps);
if (!gst_video_content_light_level_from_caps (&window->content_light_level,
caps)) {
gst_video_content_light_level_init (&window->content_light_level);
@ -897,7 +967,7 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
/* don't care refresh rate */
desc.BufferDesc.RefreshRate.Numerator = 0;
desc.BufferDesc.RefreshRate.Denominator = 1;
desc.BufferDesc.Format = format;
desc.BufferDesc.Format = window->render_format->dxgi_format;
desc.BufferDesc.ScanlineOrdering = DXGI_MODE_SCANLINE_ORDER_UNSPECIFIED;
desc.BufferDesc.Scaling = DXGI_MODE_SCALING_UNSPECIFIED;
desc.SampleDesc.Count = 1;
@ -925,7 +995,8 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
return FALSE;
}
#ifdef HAVE_DXGI_1_5_H
if (hdr_api_available && format == DXGI_FORMAT_R10G10B10A2_UNORM &&
if (hdr_api_available &&
window->render_format->dxgi_format == DXGI_FORMAT_R10G10B10A2_UNORM &&
have_cll && have_mastering) {
UINT can_support = 0;
HRESULT hr;
@ -1068,6 +1139,28 @@ _present_on_device_thread (GstD3D11Device * device, FramePresentData * data)
gst_buffer_replace (&self->cached_buffer, data->buffer);
if (self->cached_buffer) {
if (self->converter) {
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES];
gint i, j, k;
RECT rect;
for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) {
GstD3D11Memory *mem =
(GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
for (k = 0; k < mem->num_shader_resource_views; k++) {
srv[j] = mem->shader_resource_view[k];
j++;
}
}
rect.left = self->render_rect.x;
rect.right = self->render_rect.x + self->render_rect.w;
rect.top = self->render_rect.y;
rect.bottom = self->render_rect.y + self->render_rect.h;
gst_d3d11_color_converter_update_rect (self->converter, &rect);
gst_d3d11_color_converter_convert (self->converter, srv, &self->rtv);
} else {
GstD3D11Memory *mem =
(GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, 0);
@ -1087,6 +1180,7 @@ _present_on_device_thread (GstD3D11Device * device, FramePresentData * data)
(ID3D11Resource *) self->backbuffer, 0, self->render_rect.x,
self->render_rect.y, 0, (ID3D11Resource *) mem->texture, 0, &src_box);
}
}
hr = IDXGISwapChain_Present (self->swap_chain, 0, DXGI_PRESENT_DO_NOT_WAIT);

View file

@ -25,6 +25,7 @@
#include <gst/gst.h>
#include <gst/video/video.h>
#include "gstd3d11_fwd.h"
#include "gstd3d11colorconverter.h"
G_BEGIN_DECLS
@ -45,6 +46,10 @@ struct _GstD3D11Window
GstObject parent;
GstVideoInfo info;
GstVideoInfo render_info;
const GstD3D11Format *render_format;
GstD3D11ColorConverter *converter;
GstVideoMasteringDisplayInfo mastering_display_info;
GstVideoContentLightLevel content_light_level;
@ -124,7 +129,6 @@ gboolean gst_d3d11_window_prepare (GstD3D11Window * window,
guint height,
guint aspect_ratio_n,
guint aspect_ratio_d,
DXGI_FORMAT format,
GstCaps * caps,
GError ** error);