mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-04-26 06:54:49 +00:00
Code Issues 1 Projects Releases Wiki Activity

d3d11videosink: Remove resizing window hack and unify resizing flow

Browse source 
In earlier implementation of d3d11videosink where no shader was implemented,
the aspect ratio and render size were adjusted by manipulating the backbuffer size
with unintuitive formula. Since now we do color conversion and resize using
shader, we can remove the hack.
This commit is contained in:
Seungha Yang 2019-12-15 16:23:00 +09:00
parent 61cb6b2bbe
commit 9dada90108
4 changed files with 94 additions and 184 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
sys/d3d11/gstd3d11videosink.c
View file

@ -338,7 +338,7 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self), if (!gst_d3d11_window_prepare (self->window, GST_VIDEO_SINK_WIDTH (self),
GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d, GST_VIDEO_SINK_HEIGHT (self), video_par_n, video_par_d,
caps, &self->need_srv, &error)) { caps, &error)) {
GstMessage *error_msg; GstMessage *error_msg;
GST_ERROR_OBJECT (self, "cannot create swapchain"); GST_ERROR_OBJECT (self, "cannot create swapchain");
@ -360,7 +360,7 @@ gst_d3d11_video_sink_set_caps (GstBaseSink * sink, GstCaps * caps)
gst_buffer_pool_config_set_params (config, gst_buffer_pool_config_set_params (config,
caps, GST_VIDEO_INFO_SIZE (&self->info), 0, 2); caps, GST_VIDEO_INFO_SIZE (&self->info), 0, 2);
if (self->need_srv) { {
GstD3D11AllocationParams *d3d11_params; GstD3D11AllocationParams *d3d11_params;
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
@ -525,7 +525,7 @@ gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query)
if (need_pool) { if (need_pool) {
gint i; gint i;
GstCaps *render_caps; GstD3D11AllocationParams *d3d11_params;
GST_DEBUG_OBJECT (self, "create new pool"); GST_DEBUG_OBJECT (self, "create new pool");
@ -534,34 +534,22 @@ gst_d3d11_video_sink_propose_allocation (GstBaseSink * sink, GstQuery * query)
gst_buffer_pool_config_set_params (config, caps, size, 2, gst_buffer_pool_config_set_params (config, caps, size, 2,
DXGI_MAX_SWAP_CHAIN_BUFFERS); DXGI_MAX_SWAP_CHAIN_BUFFERS);
render_caps = gst_d3d11_device_get_supported_caps (self->device, d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
D3D11_FORMAT_SUPPORT_TEXTURE2D | D3D11_FORMAT_SUPPORT_DISPLAY); if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (&info,
/* if we need conversion, request shader resource view */ GST_D3D11_ALLOCATION_FLAG_USE_RESOURCE_FORMAT, D3D11_USAGE_DEFAULT,
if (render_caps && !gst_caps_can_intersect (caps, render_caps)) { D3D11_BIND_SHADER_RESOURCE);
GstD3D11AllocationParams *d3d11_params; } else {
/* Set bind flag */
GST_DEBUG_OBJECT (self, for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
"upstream format %s is not display foramt, need shader resource", d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&info)));
d3d11_params =
gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (&info,
GST_D3D11_ALLOCATION_FLAG_USE_RESOURCE_FORMAT, D3D11_USAGE_DEFAULT,
D3D11_BIND_SHADER_RESOURCE);
} else {
/* Set bind flag */
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
}
} }
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
} }
gst_clear_caps (&render_caps);
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
if (!gst_buffer_pool_set_config (pool, config)) { if (!gst_buffer_pool_set_config (pool, config)) {
g_object_unref (pool); g_object_unref (pool);
@ -672,7 +660,7 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
break; break;
} }
if (self->need_srv && !gst_d3d11_memory_ensure_shader_resource_view (dmem)) { if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
GST_LOG_OBJECT (sink, GST_LOG_OBJECT (sink,
"shader resource view is unavailable, need fallback"); "shader resource view is unavailable, need fallback");
render_buf = NULL; render_buf = NULL;
@ -742,8 +730,7 @@ gst_d3d11_video_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
GST_MAP_READ | GST_MAP_D3D11); GST_MAP_READ | GST_MAP_D3D11);
gst_memory_unmap (GST_MEMORY_CAST (dmem), &info); gst_memory_unmap (GST_MEMORY_CAST (dmem), &info);
if (self->need_srv && if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
!gst_d3d11_memory_ensure_shader_resource_view (dmem)) {
GST_ERROR_OBJECT (self, "shader resource view is not available"); GST_ERROR_OBJECT (self, "shader resource view is not available");
gst_buffer_unref (render_buf); gst_buffer_unref (render_buf);

3
sys/d3d11/gstd3d11videosink.h
View file

@ -64,9 +64,6 @@ struct _GstD3D11VideoSink
GstBufferPool *fallback_pool; GstBufferPool *fallback_pool;
gboolean can_convert; gboolean can_convert;
/* whether shader resource view is required for direct rendering or not */
gboolean need_srv;
}; };
struct _GstD3D11VideoSinkClass struct _GstD3D11VideoSinkClass

223
sys/d3d11/gstd3d11window.c
View file

@ -530,14 +530,14 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
{ {
HRESULT hr; HRESULT hr;
ID3D11Device *d3d11_dev; ID3D11Device *d3d11_dev;
ID3D11DeviceContext *d3d11_context;
guint width, height; guint width, height;
D3D11_TEXTURE2D_DESC desc;
DXGI_SWAP_CHAIN_DESC swap_desc;
if (!window->swap_chain) if (!window->swap_chain)
return; return;
d3d11_dev = gst_d3d11_device_get_device_handle (device); d3d11_dev = gst_d3d11_device_get_device_handle (device);
d3d11_context = gst_d3d11_device_get_device_context_handle (device);
if (window->backbuffer) { if (window->backbuffer) {
ID3D11Texture2D_Release (window->backbuffer); ID3D11Texture2D_Release (window->backbuffer);
@ -549,79 +549,10 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
window->rtv = NULL; window->rtv = NULL;
} }
width = window->width; /* Set zero width and height here. dxgi will decide client area by itself */
height = window->height; IDXGISwapChain_GetDesc (window->swap_chain, &swap_desc);
if (width != window->surface_width || height != window->surface_height) {
GstVideoRectangle src_rect, dst_rect;
src_rect.x = 0;
src_rect.y = 0;
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
dst_rect.x = 0;
dst_rect.y = 0;
dst_rect.w = window->surface_width;
dst_rect.h = window->surface_height;
if (window->converter) {
if (window->force_aspect_ratio) {
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect,
TRUE);
} else {
window->render_rect = dst_rect;
}
width = window->surface_width;
height = window->surface_height;
} else {
/* NOTE: there can be various way to resize texture, but
* we just copy incoming texture toward resized swap chain buffer in order to
* avoid shader coding.
* To keep aspect ratio, required vertical or horizontal padding area
* will be calculated in here.
*/
gdouble src_ratio, dst_ratio;
gdouble aspect_ratio =
(gdouble) window->aspect_ratio_n / (gdouble) window->aspect_ratio_d;
src_ratio = (gdouble) width / height;
dst_ratio =
(gdouble) window->surface_width / window->surface_height /
aspect_ratio;
src_rect.w = width;
src_rect.h = height;
if (window->force_aspect_ratio) {
if (src_ratio > dst_ratio) {
/* padding top and bottom */
dst_rect.w = width;
dst_rect.h = width / dst_ratio;
} else {
/* padding left and right */
dst_rect.w = height * dst_ratio;
dst_rect.h = height;
}
} else {
dst_rect.w = width;
dst_rect.h = height;
}
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect,
TRUE);
width = dst_rect.w;
height = dst_rect.h;
}
}
hr = IDXGISwapChain_ResizeBuffers (window->swap_chain, hr = IDXGISwapChain_ResizeBuffers (window->swap_chain,
0, width, height, DXGI_FORMAT_UNKNOWN, 0); 0, 0, 0, DXGI_FORMAT_UNKNOWN, swap_desc.Flags);
if (!gst_d3d11_result (hr)) { if (!gst_d3d11_result (hr)) {
GST_ERROR_OBJECT (window, "Couldn't resize buffers, hr: 0x%x", (guint) hr); GST_ERROR_OBJECT (window, "Couldn't resize buffers, hr: 0x%x", (guint) hr);
return; return;
@ -635,6 +566,42 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
return; return;
} }
ID3D11Texture2D_GetDesc (window->backbuffer, &desc);
window->surface_width = desc.Width;
window->surface_height = desc.Height;
width = window->width;
height = window->height;
{
GstVideoRectangle src_rect, dst_rect;
src_rect.x = 0;
src_rect.y = 0;
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
dst_rect.x = 0;
dst_rect.y = 0;
dst_rect.w = window->surface_width;
dst_rect.h = window->surface_height;
if (window->force_aspect_ratio) {
src_rect.w = width * window->aspect_ratio_n;
src_rect.h = height * window->aspect_ratio_d;
gst_video_sink_center_rect (src_rect, dst_rect, &window->render_rect,
TRUE);
} else {
window->render_rect = dst_rect;
}
}
GST_LOG_OBJECT (window,
"New client area %dx%d, render rect x: %d, y: %d, %dx%d",
desc.Width, desc.Height, window->render_rect.x, window->render_rect.y,
window->render_rect.w, window->render_rect.h);
hr = ID3D11Device_CreateRenderTargetView (d3d11_dev, hr = ID3D11Device_CreateRenderTargetView (d3d11_dev,
(ID3D11Resource *) window->backbuffer, NULL, &window->rtv); (ID3D11Resource *) window->backbuffer, NULL, &window->rtv);
if (!gst_d3d11_result (hr)) { if (!gst_d3d11_result (hr)) {
@ -643,7 +610,6 @@ gst_d3d11_window_on_resize (GstD3D11Device * device, GstD3D11Window * window)
return; return;
} }
ID3D11DeviceContext_OMSetRenderTargets (d3d11_context, 1, &window->rtv, NULL);
if (window->cached_buffer) { if (window->cached_buffer) {
FramePresentData present_data = { 0, }; FramePresentData present_data = { 0, };
@ -660,16 +626,6 @@ static void
gst_d3d11_window_on_size (GstD3D11Window * self, gst_d3d11_window_on_size (GstD3D11Window * self,
HWND hWnd, WPARAM wParam, LPARAM lParam) HWND hWnd, WPARAM wParam, LPARAM lParam)
{ {
RECT clientRect = { 0, };
GetClientRect (hWnd, &clientRect);
self->surface_width = clientRect.right - clientRect.left;
self->surface_height = clientRect.bottom - clientRect.top;
GST_LOG_OBJECT (self, "WM_PAINT, surface %ux%u",
self->surface_width, self->surface_height);
gst_d3d11_device_thread_add_full (self->device, G_PRIORITY_HIGH, gst_d3d11_device_thread_add_full (self->device, G_PRIORITY_HIGH,
(GstD3D11DeviceThreadFunc) gst_d3d11_window_on_resize, self, NULL); (GstD3D11DeviceThreadFunc) gst_d3d11_window_on_resize, self, NULL);
} }
@ -1013,8 +969,7 @@ gst_d3d11_window_color_space_from_video_info (GstD3D11Window * self,
gboolean gboolean
gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height, gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, guint aspect_ratio_n, guint aspect_ratio_d, GstCaps * caps, GError ** error)
gboolean * do_convert, GError ** error)
{ {
DXGI_SWAP_CHAIN_DESC desc = { 0, }; DXGI_SWAP_CHAIN_DESC desc = { 0, };
GstCaps *render_caps; GstCaps *render_caps;
@ -1027,7 +982,6 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE); g_return_val_if_fail (GST_IS_D3D11_WINDOW (window), FALSE);
g_return_val_if_fail (aspect_ratio_n > 0, FALSE); g_return_val_if_fail (aspect_ratio_n > 0, FALSE);
g_return_val_if_fail (aspect_ratio_d > 0, FALSE); g_return_val_if_fail (aspect_ratio_d > 0, FALSE);
g_return_val_if_fail (do_convert != NULL, FALSE);
GST_DEBUG_OBJECT (window, "Prepare window with %dx%d caps %" GST_PTR_FORMAT, GST_DEBUG_OBJECT (window, "Prepare window with %dx%d caps %" GST_PTR_FORMAT,
width, height, caps); width, height, caps);
@ -1073,19 +1027,16 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
window->info.colorimetry.primaries; window->info.colorimetry.primaries;
window->render_info.colorimetry.transfer = window->info.colorimetry.transfer; window->render_info.colorimetry.transfer = window->info.colorimetry.transfer;
if (GST_VIDEO_INFO_FORMAT (&window->info) != window->converter =
GST_VIDEO_INFO_FORMAT (&window->render_info)) { gst_d3d11_color_converter_new (window->device, &window->info,
window->converter = &window->render_info);
gst_d3d11_color_converter_new (window->device, &window->info,
&window->render_info);
if (!window->converter) { if (!window->converter) {
GST_ERROR_OBJECT (window, "Cannot create converter"); GST_ERROR_OBJECT (window, "Cannot create converter");
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED, g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_FAILED,
"Cannot create converter"); "Cannot create converter");
return FALSE; return FALSE;
}
} }
#if defined(HAVE_DXGI_1_5_H) #if defined(HAVE_DXGI_1_5_H)
if (!gst_video_content_light_level_from_caps (&window->content_light_level, if (!gst_video_content_light_level_from_caps (&window->content_light_level,
@ -1133,8 +1084,12 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
window->surface_height = height; window->surface_height = height;
} }
desc.BufferDesc.Width = window->width = window->surface_width; window->width = width;
desc.BufferDesc.Height = window->height = window->surface_height; window->height = height;
/* we will get client area at on_resize */
desc.BufferDesc.Width = 0;
desc.BufferDesc.Height = 0;
/* don't care refresh rate */ /* don't care refresh rate */
desc.BufferDesc.RefreshRate.Numerator = 0; desc.BufferDesc.RefreshRate.Numerator = 0;
desc.BufferDesc.RefreshRate.Denominator = 1; desc.BufferDesc.RefreshRate.Denominator = 1;
@ -1222,8 +1177,6 @@ gst_d3d11_window_prepare (GstD3D11Window * window, guint width, guint height,
GST_DEBUG_OBJECT (window, "New swap chain 0x%p created", window->swap_chain); GST_DEBUG_OBJECT (window, "New swap chain 0x%p created", window->swap_chain);
*do_convert = ! !window->converter;
return TRUE; return TRUE;
} }
@ -1312,54 +1265,31 @@ _present_on_device_thread (GstD3D11Device * device, FramePresentData * data)
GstD3D11Window *self = data->window; GstD3D11Window *self = data->window;
ID3D11DeviceContext *device_context; ID3D11DeviceContext *device_context;
HRESULT hr; HRESULT hr;
float black[] = { 0.0f, 0.0f, 0.0f, 0.0f };
D3D11_BOX src_box;
device_context = gst_d3d11_device_get_device_context_handle (device); device_context = gst_d3d11_device_get_device_context_handle (device);
gst_buffer_replace (&self->cached_buffer, data->buffer); gst_buffer_replace (&self->cached_buffer, data->buffer);
if (self->cached_buffer) { if (self->cached_buffer) {
if (self->converter) { ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES];
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES]; gint i, j, k;
gint i, j, k; RECT rect;
RECT rect;
for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) { for (i = 0, j = 0; i < gst_buffer_n_memory (self->cached_buffer); i++) {
GstD3D11Memory *mem =
(GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
for (k = 0; k < mem->num_shader_resource_views; k++) {
srv[j] = mem->shader_resource_view[k];
j++;
}
}
rect.left = self->render_rect.x;
rect.right = self->render_rect.x + self->render_rect.w;
rect.top = self->render_rect.y;
rect.bottom = self->render_rect.y + self->render_rect.h;
gst_d3d11_color_converter_update_rect (self->converter, &rect);
gst_d3d11_color_converter_convert (self->converter, srv, &self->rtv);
} else {
GstD3D11Memory *mem = GstD3D11Memory *mem =
(GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, 0); (GstD3D11Memory *) gst_buffer_peek_memory (self->cached_buffer, i);
for (k = 0; k < mem->num_shader_resource_views; k++) {
self->rect = *data->rect; srv[j] = mem->shader_resource_view[k];
src_box.left = self->rect.x; j++;
src_box.right = self->rect.x + self->rect.w; }
src_box.top = self->rect.y;
src_box.bottom = self->rect.y + self->rect.h;
src_box.front = 0;
src_box.back = 1;
ID3D11DeviceContext_OMSetRenderTargets (device_context,
1, &self->rtv, NULL);
ID3D11DeviceContext_ClearRenderTargetView (device_context, self->rtv,
black);
ID3D11DeviceContext_CopySubresourceRegion (device_context,
(ID3D11Resource *) self->backbuffer, 0, self->render_rect.x,
self->render_rect.y, 0, (ID3D11Resource *) mem->texture, 0, &src_box);
} }
rect.left = self->render_rect.x;
rect.right = self->render_rect.x + self->render_rect.w;
rect.top = self->render_rect.y;
rect.bottom = self->render_rect.y + self->render_rect.h;
gst_d3d11_color_converter_update_rect (self->converter, &rect);
gst_d3d11_color_converter_convert (self->converter, srv, &self->rtv);
} }
hr = IDXGISwapChain_Present (self->swap_chain, 0, DXGI_PRESENT_DO_NOT_WAIT); hr = IDXGISwapChain_Present (self->swap_chain, 0, DXGI_PRESENT_DO_NOT_WAIT);
@ -1401,13 +1331,10 @@ gst_d3d11_window_render (GstD3D11Window * window, GstBuffer * buffer,
g_mutex_unlock (&window->lock); g_mutex_unlock (&window->lock);
GST_OBJECT_LOCK (window); GST_OBJECT_LOCK (window);
if (rect->w != window->width || rect->h != window->height || if (window->pending_resize) {
window->pending_resize) {
window->width = rect->w;
window->height = rect->h;
gst_d3d11_device_thread_add (window->device, gst_d3d11_device_thread_add (window->device,
(GstD3D11DeviceThreadFunc) gst_d3d11_window_on_resize, window); (GstD3D11DeviceThreadFunc) gst_d3d11_window_on_resize, window);
window->pending_resize = FALSE;
} }
GST_OBJECT_UNLOCK (window); GST_OBJECT_UNLOCK (window);

1
sys/d3d11/gstd3d11window.h
View file

@ -138,7 +138,6 @@ gboolean gst_d3d11_window_prepare (GstD3D11Window * window,
guint aspect_ratio_n, guint aspect_ratio_n,
guint aspect_ratio_d, guint aspect_ratio_d,
GstCaps * caps, GstCaps * caps,
gboolean * do_convert,
GError ** error); GError ** error);
GstFlowReturn gst_d3d11_window_render (GstD3D11Window * window, GstFlowReturn gst_d3d11_window_render (GstD3D11Window * window,