gstreamer/subprojects/gst-plugins-bad/tests/examples/d3d11/d3d11videosink-present.cpp

670 lines
20 KiB
C++

/*
* GStreamer
* Copyright (C) 2022 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/d3d11/gstd3d11.h>
#include <d2d1.h>
#include <dwrite.h>
#include <wrl.h>
#include <string>
#include <queue>
/* *INDENT-OFF* */
using namespace Microsoft::WRL;
struct DisplayContext
{
HWND window_handle = nullptr;
GstElement *pipeline = nullptr;
GstElement *sink = nullptr;
GIOChannel *io_ch = nullptr;
bool enable_overlay = false;
ID2D1Factory *d2d_factory = nullptr;
IDWriteFactory *dwrite_factory = nullptr;
IDWriteTextFormat *format = nullptr;
IDWriteTextLayout *layout = nullptr;
/* D3D objects for background redraw with alpha blending */
ID3D11BlendState *blend = nullptr;
ID3D11PixelShader *ps = nullptr;
ID3D11VertexShader *vs = nullptr;
ID3D11InputLayout *input_layout = nullptr;
ID3D11Buffer *index_buf = nullptr;
ID3D11Buffer *vertex_buf = nullptr;
UINT width = 0;
UINT height = 0;
SRWLOCK lock = RTL_SRWLOCK_INIT;
double avg_framerate = 0;
double last_framerate = 0;
LARGE_INTEGER frequency;
std::queue < LARGE_INTEGER > render_timestamp;
GMainLoop *loop = nullptr;
};
static const gchar templ_vs_color[] =
"struct VS_INPUT {\n"
" float4 Position: POSITION;\n"
" float4 Color: COLOR;\n"
"};\n"
"struct VS_OUTPUT {\n"
" float4 Position: SV_POSITION;\n"
" float4 Color: COLOR;\n"
"};\n"
"VS_OUTPUT main (VS_INPUT input)\n"
"{\n"
" return input;\n"
"}";
static const gchar templ_ps_color[] =
"struct PS_INPUT {\n"
" float4 Position: SV_POSITION;\n"
" float4 Color: COLOR;\n"
"};\n"
"float4 main(PS_INPUT input) : SV_TARGET\n"
"{\n"
" return input.Color;\n"
"}";
/* *INDENT-ON* */
#define DISPLAY_CONTEXT_PROP "d3d11videosink.example.context"
#define CLEAR_COM(obj) do { \
if (obj) { \
obj->Release (); \
obj = nullptr; \
} \
} while (0)
static LRESULT CALLBACK
window_proc (HWND hwnd, UINT message, WPARAM wParam, LPARAM lParam)
{
DisplayContext *context = (DisplayContext *) GetPropA (hwnd,
DISPLAY_CONTEXT_PROP);
switch (message) {
case WM_DESTROY:
gst_println ("Window is destroying");
if (context) {
context->window_handle = nullptr;
RemovePropA (hwnd, DISPLAY_CONTEXT_PROP);
g_main_loop_quit (context->loop);
}
break;
case WM_LBUTTONUP:
if (!context) {
gst_printerrln ("Display context is not attached on HWND");
} else {
context->enable_overlay = !context->enable_overlay;
gst_println ("Enable overlay %d", context->enable_overlay);
/* Call expose method so that videosink can immediately
* redraw client area */
if (context->sink)
gst_video_overlay_expose (GST_VIDEO_OVERLAY (context->sink));
}
break;
default:
break;
}
return DefWindowProc (hwnd, message, wParam, lParam);
}
static gboolean
bus_msg (GstBus * bus, GstMessage * msg, DisplayContext * context)
{
switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_ERROR:{
GError *err;
gchar *dbg;
gst_message_parse_error (msg, &err, &dbg);
gst_printerrln ("ERROR %s", err->message);
if (dbg)
gst_printerrln ("ERROR debug information: %s", dbg);
g_clear_error (&err);
g_free (dbg);
g_main_loop_quit (context->loop);
break;
}
case GST_MESSAGE_EOS:
gst_println ("Got EOS");
g_main_loop_quit (context->loop);
break;
default:
break;
}
return TRUE;
}
static gboolean
msg_cb (GIOChannel * source, GIOCondition condition, gpointer data)
{
MSG msg;
if (!PeekMessage (&msg, nullptr, 0, 0, PM_REMOVE))
return G_SOURCE_CONTINUE;
TranslateMessage (&msg);
DispatchMessage (&msg);
return G_SOURCE_CONTINUE;
}
typedef struct
{
struct
{
FLOAT x;
FLOAT y;
FLOAT z;
} position;
struct
{
FLOAT r;
FLOAT g;
FLOAT b;
FLOAT a;
} color;
} VertexData;
static void
ensure_d3d11_resource (DisplayContext * context, GstD3D11Device * device)
{
D3D11_BLEND_DESC blend_desc;
D3D11_INPUT_ELEMENT_DESC input_desc[2];
D3D11_BUFFER_DESC buffer_desc;
D3D11_MAPPED_SUBRESOURCE map;
VertexData *vertex_data;
WORD *indices;
HRESULT hr;
ID3D11Device *device_handle;
ID3D11DeviceContext *context_handle;
if (context->blend)
return;
device_handle = gst_d3d11_device_get_device_handle (device);
context_handle = gst_d3d11_device_get_device_context_handle (device);
ZeroMemory (&blend_desc, sizeof (blend_desc));
ZeroMemory (input_desc, sizeof (input_desc));
ZeroMemory (&buffer_desc, sizeof (buffer_desc));
input_desc[0].SemanticName = "POSITION";
input_desc[0].SemanticIndex = 0;
input_desc[0].Format = DXGI_FORMAT_R32G32B32_FLOAT;
input_desc[0].InputSlot = 0;
input_desc[0].AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT;
input_desc[0].InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
input_desc[0].InstanceDataStepRate = 0;
input_desc[1].SemanticName = "COLOR";
input_desc[1].SemanticIndex = 0;
input_desc[1].Format = DXGI_FORMAT_R32G32B32A32_FLOAT;
input_desc[1].InputSlot = 0;
input_desc[1].AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT;
input_desc[1].InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
input_desc[1].InstanceDataStepRate = 0;
hr = gst_d3d11_create_vertex_shader_simple (device, templ_vs_color,
"main", input_desc, G_N_ELEMENTS (input_desc), &context->vs,
&context->input_layout);
g_assert (SUCCEEDED (hr));
hr = gst_d3d11_create_pixel_shader_simple (device,
templ_ps_color, "main", &context->ps);
buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
buffer_desc.ByteWidth = sizeof (VertexData) * 4;
buffer_desc.BindFlags = D3D11_BIND_VERTEX_BUFFER;
buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
hr = device_handle->CreateBuffer (&buffer_desc, nullptr,
&context->vertex_buf);
g_assert (SUCCEEDED (hr));
buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
buffer_desc.ByteWidth = sizeof (WORD) * 6;
buffer_desc.BindFlags = D3D11_BIND_INDEX_BUFFER;
buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
hr = device_handle->CreateBuffer (&buffer_desc, nullptr, &context->index_buf);
g_assert (SUCCEEDED (hr));
blend_desc.AlphaToCoverageEnable = FALSE;
blend_desc.IndependentBlendEnable = FALSE;
blend_desc.RenderTarget[0].BlendEnable = TRUE;
blend_desc.RenderTarget[0].SrcBlend = D3D11_BLEND_SRC_ALPHA;
blend_desc.RenderTarget[0].DestBlend = D3D11_BLEND_INV_SRC_ALPHA;
blend_desc.RenderTarget[0].BlendOp = D3D11_BLEND_OP_ADD;
blend_desc.RenderTarget[0].SrcBlendAlpha = D3D11_BLEND_ONE;
blend_desc.RenderTarget[0].DestBlendAlpha = D3D11_BLEND_INV_SRC_ALPHA;
blend_desc.RenderTarget[0].BlendOpAlpha = D3D11_BLEND_OP_ADD;
blend_desc.RenderTarget[0].RenderTargetWriteMask =
D3D11_COLOR_WRITE_ENABLE_ALL;
hr = device_handle->CreateBlendState (&blend_desc, &context->blend);
g_assert (SUCCEEDED (hr));
hr = context_handle->Map (context->vertex_buf, 0, D3D11_MAP_WRITE_DISCARD, 0,
&map);
g_assert (SUCCEEDED (hr));
vertex_data = (VertexData *) map.pData;
hr = context_handle->Map (context->index_buf, 0, D3D11_MAP_WRITE_DISCARD, 0,
&map);
g_assert (SUCCEEDED (hr));
indices = (WORD *) map.pData;
for (guint i = 0; i < 4; i++) {
vertex_data[i].color.r = 0.0f;
vertex_data[i].color.g = 0.5f;
vertex_data[i].color.b = 0.5f;
vertex_data[i].color.a = 0.5f;
}
/* bottom left */
vertex_data[0].position.x = -1.0f;
vertex_data[0].position.y = -1.0f;
vertex_data[0].position.z = 0.0f;
/* top left */
vertex_data[1].position.x = -1.0f;
vertex_data[1].position.y = 1.0f;
vertex_data[1].position.z = 0.0f;
/* top right */
vertex_data[2].position.x = 1.0f;
vertex_data[2].position.y = 1.0f;
vertex_data[2].position.z = 0.0f;
/* bottom right */
vertex_data[3].position.x = 1.0f;
vertex_data[3].position.y = -1.0f;
vertex_data[3].position.z = 0.0f;
/* clockwise indexing */
indices[0] = 0; /* bottom left */
indices[1] = 1; /* top left */
indices[2] = 2; /* top right */
indices[3] = 3; /* bottom right */
indices[4] = 0; /* bottom left */
indices[5] = 2; /* top right */
context_handle->Unmap (context->vertex_buf, 0);
context_handle->Unmap (context->index_buf, 0);
}
/* This callback will be called with gst_d3d11_device_lock() taken by
* d3d11videosink. We can perform GPU operation here safely */
static void
on_present (GstElement * sink, GstD3D11Device * device,
ID3D11RenderTargetView * rtv, DisplayContext * context)
{
ComPtr < ID3D11Resource > resource;
ComPtr < ID3D11Texture2D > texture;
ComPtr < IDXGISurface > surface;
ComPtr < ID2D1RenderTarget > d2d_target;
ComPtr < ID2D1SolidColorBrush > text_brush;
ID3D11DeviceContext *device_context;
HRESULT hr;
D3D11_TEXTURE2D_DESC desc;
ID2D1Factory *d2d_factory;
double framerate;
D3D11_VIEWPORT viewport;
UINT vertex_stride = sizeof (VertexData);
UINT offsets = 0;
if (!context->enable_overlay)
return;
rtv->GetResource (&resource);
hr = resource.As (&texture);
g_assert (SUCCEEDED (hr));
hr = texture.As (&surface);
g_assert (SUCCEEDED (hr));
texture->GetDesc (&desc);
ensure_d3d11_resource (context, device);
device_context = gst_d3d11_device_get_device_context_handle (device);
viewport.TopLeftX = 0;
viewport.TopLeftY = 0;
viewport.Width = desc.Width;
viewport.Height = desc.Height / 5.0f;
viewport.MinDepth = 0.0f;
viewport.MaxDepth = 1.0f;
/* Draw background using D3D11 */
device_context->IASetPrimitiveTopology
(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
device_context->IASetInputLayout (context->input_layout);
device_context->IASetVertexBuffers (0, 1, &context->vertex_buf,
&vertex_stride, &offsets);
device_context->IASetIndexBuffer (context->index_buf, DXGI_FORMAT_R16_UINT,
0);
device_context->VSSetShader (context->vs, nullptr, 0);
device_context->PSSetShader (context->ps, nullptr, 0);
device_context->RSSetViewports (1, &viewport);
device_context->OMSetRenderTargets (1, &rtv, nullptr);
device_context->OMSetBlendState (context->blend, nullptr, 0xffffffff);
device_context->DrawIndexed (6, 0, 0);
/* Creates new layout on window size or framerate change */
AcquireSRWLockExclusive (&context->lock);
framerate = context->avg_framerate;
if (context->layout && (context->width != desc.Width ||
context->height != desc.Height
|| context->last_framerate != framerate)) {
CLEAR_COM (context->layout);
}
context->last_framerate = framerate;
ReleaseSRWLockExclusive (&context->lock);
context->width = desc.Width;
context->height = desc.Height;
if (!context->layout) {
IDWriteFactory *factory = context->dwrite_factory;
std::wstring overlay_string;
wchar_t fps_buf[128];
DWRITE_TEXT_METRICS metrics;
FLOAT font_size;
bool was_decreased = false;
DWRITE_TEXT_RANGE range;
overlay_string = L"Text Overlay, FPS: ";
std::swprintf (fps_buf, L"%.1f", framerate);
overlay_string += fps_buf;
hr = factory->CreateTextLayout (overlay_string.c_str (),
overlay_string.length (),
context->format, desc.Width, desc.Height / 5.0f, &context->layout);
g_assert (SUCCEEDED (hr));
context->layout->SetTextAlignment (DWRITE_TEXT_ALIGNMENT_CENTER);
context->layout->SetParagraphAlignment (DWRITE_PARAGRAPH_ALIGNMENT_CENTER);
range.startPosition = 0;
range.length = overlay_string.length ();
/* Calculate best font size */
do {
hr = context->layout->GetMetrics (&metrics);
g_assert (SUCCEEDED (hr));
context->layout->GetFontSize (0, &font_size);
if (metrics.widthIncludingTrailingWhitespace >= (FLOAT) desc.Width) {
if (font_size > 1.0f) {
font_size -= 0.5f;
was_decreased = true;
hr = context->layout->SetFontSize (font_size, range);
g_assert (SUCCEEDED (hr));
continue;
}
break;
}
if (was_decreased)
break;
if (metrics.widthIncludingTrailingWhitespace < (FLOAT) desc.Width) {
if (metrics.widthIncludingTrailingWhitespace >= desc.Width * 0.7)
break;
font_size += 0.5f;
hr = context->layout->SetFontSize (font_size, range);
g_assert (SUCCEEDED (hr));
continue;
}
} while (true);
}
d2d_factory = context->d2d_factory;
D2D1_RENDER_TARGET_PROPERTIES props;
props.type = D2D1_RENDER_TARGET_TYPE_DEFAULT;
props.pixelFormat.format = DXGI_FORMAT_B8G8R8A8_UNORM;
props.pixelFormat.alphaMode = D2D1_ALPHA_MODE_PREMULTIPLIED;
/* default DPI */
props.dpiX = 0;
props.dpiY = 0;
props.usage = D2D1_RENDER_TARGET_USAGE_NONE;
props.minLevel = D2D1_FEATURE_LEVEL_DEFAULT;
/* Creates D2D render target using swapchin's backbuffer */
hr = d2d_factory->CreateDxgiSurfaceRenderTarget (surface.Get (), props,
&d2d_target);
g_assert (SUCCEEDED (hr));
/* text brush */
hr = d2d_target->CreateSolidColorBrush (D2D1::ColorF (D2D1::ColorF::Black),
&text_brush);
g_assert (SUCCEEDED (hr));
d2d_target->BeginDraw ();
/* Draw text */
d2d_target->DrawTextLayout (D2D1::Point2F (0, 0),
context->layout, text_brush.Get (), D2D1_DRAW_TEXT_OPTIONS_NONE);
d2d_target->EndDraw ();
}
static GstPadProbeReturn
framerate_calculate_probe (GstPad * pad, GstPadProbeInfo * info,
DisplayContext * context)
{
LARGE_INTEGER now;
AcquireSRWLockExclusive (&context->lock);
QueryPerformanceCounter (&now);
context->render_timestamp.push (now);
if (context->render_timestamp.size () > 10) {
LARGE_INTEGER last = context->render_timestamp.back ();
LARGE_INTEGER first = context->render_timestamp.front ();
double diff = last.QuadPart - first.QuadPart;
context->avg_framerate =
(double) context->frequency.QuadPart *
(context->render_timestamp.size () - 1) / diff;
std::queue < LARGE_INTEGER > empty_queue;
std::swap (context->render_timestamp, empty_queue);
}
ReleaseSRWLockExclusive (&context->lock);
return GST_PAD_PROBE_OK;
}
gint
main (gint argc, gchar ** argv)
{
WNDCLASSEXA wc = { 0, };
HINSTANCE hinstance = GetModuleHandle (nullptr);
GOptionContext *option_ctx;
GError *error = nullptr;
RECT wr = { 0, 0, 320, 240 };
gboolean ret;
gchar *uri = nullptr;
GOptionEntry options[] = {
{"uri", 0, 0, G_OPTION_ARG_STRING, &uri, "URI to play", nullptr},
{nullptr}
};
HRESULT hr;
DisplayContext context;
GstPad *pad;
option_ctx =
g_option_context_new ("d3d11videosink \"present\" signal example");
g_option_context_add_main_entries (option_ctx, options, nullptr);
g_option_context_add_group (option_ctx, gst_init_get_option_group ());
ret = g_option_context_parse (option_ctx, &argc, &argv, &error);
g_option_context_free (option_ctx);
if (!ret) {
gst_printerrln ("option parsing failed: %s", error->message);
g_clear_error (&error);
return 1;
}
if (!uri) {
gst_printerrln ("File name or URI must be provided");
return 1;
}
if (!gst_uri_is_valid (uri)) {
gchar *file = gst_filename_to_uri (uri, nullptr);
g_free (uri);
uri = file;
}
if (!uri) {
gst_printerrln ("No valid URI");
return 1;
}
/* Prepare device independent D2D objects */
hr = D2D1CreateFactory (D2D1_FACTORY_TYPE_MULTI_THREADED,
IID_PPV_ARGS (&context.d2d_factory));
if (FAILED (hr)) {
gst_printerrln ("Couldn't create D2D factory");
return 1;
}
hr = DWriteCreateFactory (DWRITE_FACTORY_TYPE_SHARED,
__uuidof (context.dwrite_factory),
reinterpret_cast < IUnknown ** >(&context.dwrite_factory));
if (FAILED (hr)) {
gst_printerrln ("Couldn't create DirectWrite factory");
return 1;
}
/* Font size will be re-calculated on present */
hr = context.dwrite_factory->CreateTextFormat (L"Consolas", nullptr,
DWRITE_FONT_WEIGHT_REGULAR, DWRITE_FONT_STYLE_NORMAL,
DWRITE_FONT_STRETCH_NORMAL, 12.0f, L"en-us", &context.format);
/* For rendered framerate calculation */
QueryPerformanceFrequency (&context.frequency);
context.loop = g_main_loop_new (nullptr, FALSE);
wc.cbSize = sizeof (WNDCLASSEXA);
wc.style = CS_HREDRAW | CS_VREDRAW;
wc.lpfnWndProc = (WNDPROC) window_proc;
wc.hInstance = hinstance;
wc.hCursor = LoadCursor (nullptr, IDC_ARROW);
wc.lpszClassName = "GstD3D11VideoSinkExample";
RegisterClassExA (&wc);
AdjustWindowRect (&wr, WS_OVERLAPPEDWINDOW, FALSE);
context.window_handle =
CreateWindowExA (0, wc.lpszClassName, "GstD3D11VideoSinkExample",
WS_CLIPSIBLINGS | WS_CLIPCHILDREN | WS_OVERLAPPEDWINDOW,
CW_USEDEFAULT, CW_USEDEFAULT,
wr.right - wr.left, wr.bottom - wr.top, nullptr, nullptr,
hinstance, nullptr);
context.io_ch = g_io_channel_win32_new_messages (0);
g_io_add_watch (context.io_ch, G_IO_IN, msg_cb, context.window_handle);
context.pipeline = gst_element_factory_make ("playbin", nullptr);
g_assert (context.pipeline);
context.sink = gst_element_factory_make ("d3d11videosink", nullptr);
g_assert (context.sink);
/* Enables present signal */
g_object_set (context.sink, "emit-present", TRUE, nullptr);
/* D2D <-> DXGI interop requires BGRA format */
g_object_set (context.sink, "display-format", DXGI_FORMAT_B8G8R8A8_UNORM,
nullptr);
g_signal_connect (context.sink, "present", G_CALLBACK (on_present), &context);
gst_video_overlay_set_window_handle (GST_VIDEO_OVERLAY (context.sink),
(guintptr) context.window_handle);
/* Attach our display context on HWND */
SetPropA (context.window_handle, DISPLAY_CONTEXT_PROP, &context);
g_object_set (context.pipeline,
"uri", uri, "video-sink", context.sink, nullptr);
gst_bus_add_watch (GST_ELEMENT_BUS (context.pipeline),
(GstBusFunc) bus_msg, &context);
pad = gst_element_get_static_pad (context.sink, "sink");
gst_pad_add_probe (pad, GST_PAD_PROBE_TYPE_BUFFER,
(GstPadProbeCallback) framerate_calculate_probe, &context, nullptr);
if (gst_element_set_state (context.pipeline, GST_STATE_PLAYING) ==
GST_STATE_CHANGE_FAILURE) {
gst_printerrln ("Could not set state to playing for uri %s", uri);
return 1;
}
ShowWindow (context.window_handle, SW_SHOW);
gst_println ("Click window client area to toggle overlay");
g_main_loop_run (context.loop);
gst_element_set_state (context.pipeline, GST_STATE_NULL);
gst_bus_remove_watch (GST_ELEMENT_BUS (context.pipeline));
gst_object_unref (context.pipeline);
g_io_channel_unref (context.io_ch);
if (context.window_handle)
DestroyWindow (context.window_handle);
CLEAR_COM (context.blend);
CLEAR_COM (context.ps);
CLEAR_COM (context.vs);
CLEAR_COM (context.input_layout);
CLEAR_COM (context.index_buf);
CLEAR_COM (context.vertex_buf);
CLEAR_COM (context.layout);
context.format->Release ();
context.d2d_factory->Release ();
context.dwrite_factory->Release ();
g_main_loop_unref (context.loop);
return 0;
}