mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-23 10:11:08 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12swapchainsink.cpp
Seungha Yang 4bb3854772 d3d12: Add d3d12swapchainsink element 
Adding a new videosink element for Windows composition API based
applications. Unlike d3d12videosink, this element will create only
DXGI swapchain by using IDXGIFactory2::CreateSwapChainForComposition()
without actual window handle, so that video scene can be composed
via Windows native composition API, such as DirectComposition.
Note that this videosink does not support GstVideoOverlay interface
because of the design.

The swapchain created by this element can be used with
* DirectComposition's IDCompositionVisual in Win32 app
* WinRT and WinUI3's UI.Composition in Win32/UWP app
* UWP and WinUI3 XAML's SwapChainPanel

See also examples in this commit which show usage of the videosink

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7287>
2024-08-19 11:07:17 +09:00

1082 lines
34 KiB
C++
Raw Blame History

/* GStreamer
* Copyright (C) 2024 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstd3d12swapchainsink.h"
#include "gstd3d12pluginutils.h"
#include "gstd3d12overlaycompositor.h"
#include <directx/d3dx12.h>
#include <mutex>
#include <wrl.h>
#include <vector>
#include <memory>
/* *INDENT-OFF* */
using namespace Microsoft::WRL;
/* *INDENT-ON* */
enum
{
PROP_0,
PROP_ADAPTER,
PROP_FORCE_ASPECT_RATIO,
PROP_WIDTH,
PROP_HEIGHT,
PROP_BORDER_COLOR,
PROP_SWAPCHAIN,
};
#define DEFAULT_ADAPTER -1
#define DEFAULT_FORCE_ASPECT_RATIO TRUE
#define DEFAULT_WIDTH 1280
#define DEFAULT_HEIGHT 720
#define DEFAULT_BORDER_COLOR (G_GUINT64_CONSTANT(0xffff000000000000))
#define BACK_BUFFER_COUNT 2
enum
{
SIGNAL_RESIZE,
SIGNAL_LAST
};
static guint d3d12_swapchain_sink_signals[SIGNAL_LAST] = { 0, };
static GstStaticPadTemplate sink_template =
GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, GST_D3D12_ALL_FORMATS) "; "
GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY ","
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
GST_D3D12_ALL_FORMATS) ";"
GST_VIDEO_CAPS_MAKE (GST_D3D12_ALL_FORMATS) "; "
GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY ","
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
GST_D3D12_ALL_FORMATS)));
GST_DEBUG_CATEGORY_STATIC (gst_d3d12_swapchain_sink_debug);
#define GST_CAT_DEFAULT gst_d3d12_swapchain_sink_debug
/* *INDENT-OFF* */
struct BackBuffer
{
BackBuffer (GstBuffer * buffer, ID3D12Resource * res)
{
backbuf = buffer;
resource = res;
}
~BackBuffer ()
{
gst_clear_buffer (&backbuf);
}
GstBuffer *backbuf = nullptr;
ComPtr<ID3D12Resource> resource;
};
struct GstD3D12SwapChainSinkPrivate
{
GstD3D12SwapChainSinkPrivate ()
{
convert_config = gst_structure_new_empty ("convert-config");
fence_data_pool = gst_d3d12_fence_data_pool_new ();
gst_video_info_init (&info);
gst_video_info_set_format (&display_info, GST_VIDEO_FORMAT_RGBA,
width, height);
update_border_color ();
}
~GstD3D12SwapChainSinkPrivate ()
{
stop ();
gst_structure_free (convert_config);
gst_clear_object (&comp);
gst_clear_object (&ca_pool);
gst_clear_object (&fence_data_pool);
}
void stop ()
{
if (cq && swapchain && fence_val > 0)
gst_d3d12_command_queue_idle_for_swapchain (cq, fence_val);
if (pool) {
gst_buffer_pool_set_active (pool, FALSE);
gst_clear_object (&pool);
}
gst_clear_caps (&caps);
gst_clear_buffer (&cached_buf);
gst_clear_object (&conv);
backbuf.clear ();
convert_format = GST_VIDEO_FORMAT_UNKNOWN;
caps_updated = false;
first_present = true;
}
void update_border_color ()
{
border_color_val[0] = ((border_color &
G_GUINT64_CONSTANT(0x0000ffff00000000)) >> 32) / (FLOAT) G_MAXUINT16;
border_color_val[1] = ((border_color &
G_GUINT64_CONSTANT(0x00000000ffff0000)) >> 16) / (FLOAT) G_MAXUINT16;
border_color_val[2] = (border_color &
G_GUINT64_CONSTANT(0x000000000000ffff)) / (FLOAT) G_MAXUINT16;
border_color_val[3] = ((border_color &
G_GUINT64_CONSTANT(0xffff000000000000)) >> 48) / (FLOAT) G_MAXUINT16;
}
std::recursive_mutex lock;
GstVideoInfo info;
GstVideoInfo display_info;
guint display_width;
guint display_height;
GstVideoFormat convert_format = GST_VIDEO_FORMAT_UNKNOWN;
ComPtr<IDXGISwapChain4> swapchain;
ComPtr<ID3D12GraphicsCommandList> cl;
std::vector<std::shared_ptr<BackBuffer>> backbuf;
GstStructure *convert_config = nullptr;
GstD3D12FenceDataPool *fence_data_pool = nullptr;
GstBufferPool *pool = nullptr;
GstD3D12CommandQueue *cq = nullptr;
GstD3D12CommandAllocatorPool *ca_pool = nullptr;
GstBuffer *cached_buf = nullptr;
GstCaps *caps = nullptr;
GstD3D12Converter *conv = nullptr;
GstD3D12OverlayCompositor *comp = nullptr;
guint64 fence_val = 0;
bool caps_updated = false;
bool first_present = true;
D3D12_BOX crop_rect = { };
D3D12_BOX prev_crop_rect = { };
FLOAT border_color_val[4];
GstVideoRectangle viewport = { };
gint adapter = DEFAULT_ADAPTER;
gint force_aspect_ratio = DEFAULT_FORCE_ASPECT_RATIO;
guint width = DEFAULT_WIDTH;
guint height = DEFAULT_HEIGHT;
guint64 border_color = DEFAULT_BORDER_COLOR;
};
/* *INDENT-ON* */
struct _GstD3D12SwapChainSink
{
GstVideoSink parent;
GstD3D12Device *device;
GstD3D12SwapChainSinkPrivate *priv;
};
static void gst_d3d12_swapchain_sink_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_d3d12_swapchain_sink_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static void gst_d3d12_swapchain_sink_finalize (GObject * object);
static void gst_d3d12_swapchain_sink_set_context (GstElement * element,
GstContext * context);
static gboolean gst_d3d12_swapchain_sink_start (GstBaseSink * sink);
static gboolean gst_d3d12_swapchain_sink_stop (GstBaseSink * sink);
static gboolean gst_d3d12_swapchain_sink_propose_allocation (GstBaseSink * sink,
GstQuery * query);
static gboolean gst_d3d12_swapchain_sink_query (GstBaseSink * sink,
GstQuery * query);
static GstFlowReturn gst_d3d12_swapchain_sink_prepare (GstBaseSink * sink,
GstBuffer * buf);
static gboolean gst_d3d12_swapchain_sink_set_info (GstVideoSink * sink,
GstCaps * caps, const GstVideoInfo * info);
static GstFlowReturn gst_d3d12_swapchain_sink_show_frame (GstVideoSink * sink,
GstBuffer * buf);
static void gst_d3d12_swapchain_sink_resize (GstD3D12SwapChainSink * self,
guint width, guint height);
#define gst_d3d12_swapchain_sink_parent_class parent_class
G_DEFINE_TYPE (GstD3D12SwapChainSink, gst_d3d12_swapchain_sink,
GST_TYPE_VIDEO_SINK);
static void
gst_d3d12_swapchain_sink_class_init (GstD3D12SwapChainSinkClass * klass)
{
auto object_class = G_OBJECT_CLASS (klass);
auto element_class = GST_ELEMENT_CLASS (klass);
auto basesink_class = GST_BASE_SINK_CLASS (klass);
auto videosink_class = GST_VIDEO_SINK_CLASS (klass);
object_class->set_property = gst_d3d12_swapchain_sink_set_property;
object_class->get_property = gst_d3d12_swapchain_sink_get_property;
object_class->finalize = gst_d3d12_swapchain_sink_finalize;
g_object_class_install_property (object_class, PROP_ADAPTER,
g_param_spec_int ("adapter", "Adapter",
"Adapter index for creating device (-1 for default)",
-1, G_MAXINT32, DEFAULT_ADAPTER,
(GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_FORCE_ASPECT_RATIO,
g_param_spec_boolean ("force-aspect-ratio",
"Force aspect ratio",
"When enabled, scaling will respect original aspect ratio",
DEFAULT_FORCE_ASPECT_RATIO,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_WIDTH,
g_param_spec_uint ("swapchain-width", "Swapchain Width",
"Width of swapchain buffers",
1, D3D12_REQ_TEXTURE2D_U_OR_V_DIMENSION, DEFAULT_WIDTH,
(GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_HEIGHT,
g_param_spec_uint ("swapchain-height", "Swapchain Height",
"Height of swapchain buffers",
1, D3D12_REQ_TEXTURE2D_U_OR_V_DIMENSION, DEFAULT_HEIGHT,
(GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_BORDER_COLOR,
g_param_spec_uint64 ("border-color", "Border Color",
"ARGB64 representation of the border color to use",
0, G_MAXUINT64, DEFAULT_BORDER_COLOR,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (object_class, PROP_SWAPCHAIN,
g_param_spec_pointer ("swapchain", "SwapChain",
"DXGI swapchain handle",
(GParamFlags) (GST_PARAM_DOC_SHOW_DEFAULT | G_PARAM_READABLE |
G_PARAM_STATIC_STRINGS)));
d3d12_swapchain_sink_signals[SIGNAL_RESIZE] =
g_signal_new_class_handler ("resize", G_TYPE_FROM_CLASS (klass),
(GSignalFlags) (G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
G_CALLBACK (gst_d3d12_swapchain_sink_resize), nullptr, nullptr, nullptr,
G_TYPE_NONE, 2, G_TYPE_UINT, G_TYPE_UINT);
element_class->set_context =
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_set_context);
gst_element_class_set_static_metadata (element_class,
"Direct3D12 SwapChain Sink", "Sink/Video",
"DXGI composition swapchain sink",
"Seungha Yang <seungha@centricular.com>");
gst_element_class_add_static_pad_template (element_class, &sink_template);
basesink_class->start = GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_start);
basesink_class->stop = GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_stop);
basesink_class->propose_allocation =
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_propose_allocation);
basesink_class->query = GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_query);
basesink_class->prepare =
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_prepare);
videosink_class->set_info =
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_set_info);
videosink_class->show_frame =
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_show_frame);
GST_DEBUG_CATEGORY_INIT (gst_d3d12_swapchain_sink_debug,
"d3d12swapchainsink", 0, "d3d12swapchainsink");
}
static void
gst_d3d12_swapchain_sink_init (GstD3D12SwapChainSink * self)
{
self->priv = new GstD3D12SwapChainSinkPrivate ();
}
static void
gst_d3d12_swapchain_sink_finalize (GObject * object)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (object);
delete self->priv;
gst_clear_object (&self->device);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_d3d12_swapchain_sink_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (object);
auto priv = self->priv;
std::lock_guard < std::recursive_mutex > lk (priv->lock);
switch (prop_id) {
case PROP_ADAPTER:
priv->adapter = g_value_get_int (value);
break;
case PROP_FORCE_ASPECT_RATIO:
{
auto val = g_value_get_boolean (value);
if (val != priv->force_aspect_ratio) {
priv->force_aspect_ratio = val;
gst_d3d12_swapchain_sink_resize (self, priv->width, priv->height);
}
break;
}
case PROP_BORDER_COLOR:
priv->border_color = g_value_get_uint64 (value);
priv->update_border_color ();
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_d3d12_swapchain_sink_resize_unlocked (GstD3D12SwapChainSink * self,
guint width, guint height)
{
auto priv = self->priv;
if (width != priv->width || height != priv->height) {
GST_DEBUG_OBJECT (self, "Resizing swapchain, %ux%u -> %ux%u",
priv->width, priv->height, width, height);
if (priv->cq && priv->swapchain && priv->fence_val > 0)
gst_d3d12_command_queue_idle_for_swapchain (priv->cq, priv->fence_val);
priv->backbuf.clear ();
priv->width = width;
priv->height = height;
priv->first_present = true;
gst_video_info_set_format (&priv->display_info, GST_VIDEO_FORMAT_RGBA,
width, height);
}
if (priv->swapchain && priv->backbuf.empty ()) {
auto hr = priv->swapchain->ResizeBuffers (BACK_BUFFER_COUNT,
priv->width, priv->height, DXGI_FORMAT_R8G8B8A8_UNORM, 0);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Resize failed");
return FALSE;
}
for (guint i = 0; i < BACK_BUFFER_COUNT; i++) {
ComPtr < ID3D12Resource > backbuf;
hr = priv->swapchain->GetBuffer (i, IID_PPV_ARGS (&backbuf));
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't get backbuffer");
priv->backbuf.clear ();
return FALSE;
}
auto mem = gst_d3d12_allocator_alloc_wrapped (nullptr, self->device,
backbuf.Get (), 0, nullptr, nullptr);
auto buf = gst_buffer_new ();
gst_buffer_append_memory (buf, mem);
auto swapbuf = std::make_shared < BackBuffer > (buf, backbuf.Get ());
priv->backbuf.push_back (swapbuf);
}
}
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_ensure_swapchain (GstD3D12SwapChainSink * self)
{
auto priv = self->priv;
if (priv->swapchain)
return TRUE;
if (!gst_d3d12_ensure_element_data (GST_ELEMENT_CAST (self), priv->adapter,
&self->device)) {
GST_ERROR_OBJECT (self, "Cannot create device");
return FALSE;
}
priv->cq = gst_d3d12_device_get_command_queue (self->device,
D3D12_COMMAND_LIST_TYPE_DIRECT);
auto cq = gst_d3d12_command_queue_get_handle (priv->cq);
auto factory = gst_d3d12_device_get_factory_handle (self->device);
DXGI_SWAP_CHAIN_DESC1 desc = { };
desc.Width = priv->width;
desc.Height = priv->height;
desc.Format = DXGI_FORMAT_R8G8B8A8_UNORM;
desc.SampleDesc.Count = 1;
desc.BufferUsage = DXGI_USAGE_RENDER_TARGET_OUTPUT;
desc.BufferCount = BACK_BUFFER_COUNT;
desc.Scaling = DXGI_SCALING_STRETCH;
desc.SwapEffect = DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL;
desc.AlphaMode = DXGI_ALPHA_MODE_PREMULTIPLIED;
ComPtr < IDXGISwapChain1 > swapchain;
auto hr = factory->CreateSwapChainForComposition (cq, &desc, nullptr,
&swapchain);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't create swapchain");
return FALSE;
}
hr = swapchain.As (&priv->swapchain);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't get IDXGISwapChain4 interface");
return FALSE;
}
auto device = gst_d3d12_device_get_device_handle (self->device);
priv->ca_pool = gst_d3d12_command_allocator_pool_new (device,
D3D12_COMMAND_LIST_TYPE_DIRECT);
GstVideoInfo info;
gst_video_info_set_format (&info,
GST_VIDEO_FORMAT_RGBA, priv->width, priv->height);
priv->comp = gst_d3d12_overlay_compositor_new (self->device, &info);
return gst_d3d12_swapchain_sink_resize_unlocked (self,
priv->width, priv->height);
}
static void
gst_d3d12_swapchain_sink_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (object);
auto priv = self->priv;
std::lock_guard < std::recursive_mutex > lk (priv->lock);
switch (prop_id) {
case PROP_ADAPTER:
g_value_set_int (value, priv->adapter);
break;
case PROP_FORCE_ASPECT_RATIO:
g_value_set_boolean (value, priv->force_aspect_ratio);
break;
case PROP_WIDTH:
g_value_set_uint (value, priv->width);
break;
case PROP_HEIGHT:
g_value_set_uint (value, priv->height);
break;
case PROP_BORDER_COLOR:
g_value_set_uint64 (value, priv->border_color);
break;
case PROP_SWAPCHAIN:
gst_d3d12_swapchain_sink_ensure_swapchain (self);
g_value_set_pointer (value, priv->swapchain.Get ());
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_d3d12_swapchain_sink_set_context (GstElement * element,
GstContext * context)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (element);
auto priv = self->priv;
gst_d3d12_handle_set_context (element, context, priv->adapter, &self->device);
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
static gboolean
gst_d3d12_swapchain_sink_set_info (GstVideoSink * sink, GstCaps * caps,
const GstVideoInfo * info)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
auto priv = self->priv;
GST_DEBUG_OBJECT (self, "set caps %" GST_PTR_FORMAT, caps);
std::lock_guard < std::recursive_mutex > lk (priv->lock);
gst_caps_replace (&priv->caps, caps);
priv->info = *info;
priv->caps_updated = true;
auto video_width = GST_VIDEO_INFO_WIDTH (&priv->info);
auto video_height = GST_VIDEO_INFO_HEIGHT (&priv->info);
auto video_par_n = GST_VIDEO_INFO_PAR_N (&priv->info);
auto video_par_d = GST_VIDEO_INFO_PAR_D (&priv->info);
gint display_par_n = 1;
gint display_par_d = 1;
guint num, den;
if (!gst_video_calculate_display_ratio (&num, &den, video_width,
video_height, video_par_n, video_par_d, display_par_n,
display_par_d)) {
GST_ELEMENT_WARNING (self, CORE, NEGOTIATION, (nullptr),
("Error calculating the output display ratio of the video."));
GST_VIDEO_SINK_WIDTH (self) = video_width;
GST_VIDEO_SINK_HEIGHT (self) = video_height;
} else {
GST_DEBUG_OBJECT (self,
"video width/height: %dx%d, calculated display ratio: %d/%d format: %s",
video_width, video_height, num, den,
gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&priv->info)));
if (video_height % den == 0) {
GST_DEBUG_OBJECT (self, "keeping video height");
GST_VIDEO_SINK_WIDTH (self) = (guint)
gst_util_uint64_scale_int (video_height, num, den);
GST_VIDEO_SINK_HEIGHT (self) = video_height;
} else if (video_width % num == 0) {
GST_DEBUG_OBJECT (self, "keeping video width");
GST_VIDEO_SINK_WIDTH (self) = video_width;
GST_VIDEO_SINK_HEIGHT (self) = (guint)
gst_util_uint64_scale_int (video_width, den, num);
} else {
GST_DEBUG_OBJECT (self, "approximating while keeping video height");
GST_VIDEO_SINK_WIDTH (self) = (guint)
gst_util_uint64_scale_int (video_height, num, den);
GST_VIDEO_SINK_HEIGHT (self) = video_height;
}
}
if (GST_VIDEO_SINK_WIDTH (self) <= 0) {
GST_WARNING_OBJECT (self, "Invalid display width %d",
GST_VIDEO_SINK_WIDTH (self));
GST_VIDEO_SINK_WIDTH (self) = 8;
}
if (GST_VIDEO_SINK_HEIGHT (self) <= 0) {
GST_WARNING_OBJECT (self, "Invalid display height %d",
GST_VIDEO_SINK_HEIGHT (self));
GST_VIDEO_SINK_HEIGHT (self) = 8;
}
GST_DEBUG_OBJECT (self, "scaling to %dx%d",
GST_VIDEO_SINK_WIDTH (self), GST_VIDEO_SINK_HEIGHT (self));
if (priv->pool) {
gst_buffer_pool_set_active (priv->pool, FALSE);
gst_clear_object (&priv->pool);
}
priv->pool = gst_d3d12_buffer_pool_new (self->device);
auto config = gst_buffer_pool_get_config (priv->pool);
gst_buffer_pool_config_set_params (config, priv->caps, priv->info.size, 0, 0);
if (!gst_buffer_pool_set_config (priv->pool, config) ||
!gst_buffer_pool_set_active (priv->pool, TRUE)) {
GST_ELEMENT_ERROR (self, RESOURCE, FAILED, (nullptr),
("Couldn't setup buffer pool"));
gst_clear_object (&priv->pool);
return FALSE;
}
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_foreach_meta (GstBuffer * buffer, GstMeta ** meta,
GstBuffer * uploaded)
{
if ((*meta)->info->api != GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE)
return TRUE;
auto cmeta = (GstVideoOverlayCompositionMeta *) (*meta);
if (!cmeta->overlay)
return TRUE;
if (gst_video_overlay_composition_n_rectangles (cmeta->overlay) == 0)
return TRUE;
gst_buffer_add_video_overlay_composition_meta (uploaded, cmeta->overlay);
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_render (GstD3D12SwapChainSink * self)
{
auto priv = self->priv;
if (!priv->cached_buf) {
GST_DEBUG_OBJECT (self, "No cached buffer");
return TRUE;
}
auto crop_rect = priv->crop_rect;
auto crop_meta = gst_buffer_get_video_crop_meta (priv->cached_buf);
if (crop_meta) {
crop_rect = CD3DX12_BOX (crop_meta->x, crop_meta->y,
crop_meta->x + crop_meta->width, crop_meta->y + crop_meta->height);
}
if (crop_rect != priv->prev_crop_rect) {
g_object_set (priv->conv, "src-x", (gint) crop_rect.left,
"src-y", (gint) crop_rect.top,
"src-width", (gint) (crop_rect.right - crop_rect.left),
"src-height", (gint) (crop_rect.bottom - crop_rect.top), nullptr);
priv->prev_crop_rect = crop_rect;
}
if (priv->first_present) {
GstVideoRectangle dst_rect = { };
dst_rect.w = priv->width;
dst_rect.h = priv->height;
if (priv->force_aspect_ratio) {
GstVideoRectangle src_rect = { };
src_rect.w = priv->display_width;
src_rect.h = priv->display_height;
gst_video_sink_center_rect (src_rect, dst_rect, &priv->viewport, TRUE);
} else {
priv->viewport = dst_rect;
}
g_object_set (priv->conv, "dest-x", priv->viewport.x,
"dest-y", priv->viewport.y, "dest-width", priv->viewport.w,
"dest-height", priv->viewport.h, nullptr);
gst_d3d12_overlay_compositor_update_viewport (priv->comp, &priv->viewport);
priv->first_present = false;
}
gst_d3d12_overlay_compositor_upload (priv->comp, priv->cached_buf);
GstD3D12CommandAllocator *gst_ca;
if (!gst_d3d12_command_allocator_pool_acquire (priv->ca_pool, &gst_ca)) {
GST_ERROR_OBJECT (self, "Couldn't acquire command allocator");
return FALSE;
}
auto ca = gst_d3d12_command_allocator_get_handle (gst_ca);
auto hr = ca->Reset ();
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't reset command list");
gst_d3d12_command_allocator_unref (gst_ca);
return FALSE;
}
ComPtr < ID3D12GraphicsCommandList > cl;
if (!priv->cl) {
auto device_handle = gst_d3d12_device_get_device_handle (self->device);
hr = device_handle->CreateCommandList (0, D3D12_COMMAND_LIST_TYPE_DIRECT,
ca, nullptr, IID_PPV_ARGS (&cl));
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't create command list");
gst_d3d12_command_allocator_unref (gst_ca);
return FALSE;
}
priv->cl = cl;
} else {
cl = priv->cl;
hr = cl->Reset (ca, nullptr);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't reset command list");
gst_d3d12_command_allocator_unref (gst_ca);
return FALSE;
}
}
auto cur_idx = priv->swapchain->GetCurrentBackBufferIndex ();
auto backbuf = priv->backbuf[cur_idx]->backbuf;
GstD3D12FenceData *fence_data;
gst_d3d12_fence_data_pool_acquire (priv->fence_data_pool, &fence_data);
gst_d3d12_fence_data_push (fence_data, FENCE_NOTIFY_MINI_OBJECT (gst_ca));
auto mem = (GstD3D12Memory *) gst_buffer_peek_memory (backbuf, 0);
auto backbuf_texture = gst_d3d12_memory_get_resource_handle (mem);
D3D12_RESOURCE_BARRIER barrier =
CD3DX12_RESOURCE_BARRIER::Transition (backbuf_texture,
D3D12_RESOURCE_STATE_COMMON,
D3D12_RESOURCE_STATE_RENDER_TARGET);
cl->ResourceBarrier (1, &barrier);
if (priv->viewport.x != 0 || priv->viewport.y != 0 ||
(guint) priv->viewport.w != priv->width ||
(guint) priv->viewport.h != priv->height) {
auto rtv_heap = gst_d3d12_memory_get_render_target_view_heap (mem);
auto cpu_handle = GetCPUDescriptorHandleForHeapStart (rtv_heap);
cl->ClearRenderTargetView (cpu_handle, priv->border_color_val, 0, nullptr);
}
if (!gst_d3d12_converter_convert_buffer (priv->conv,
priv->cached_buf, backbuf, fence_data, cl.Get (), TRUE)) {
GST_ERROR_OBJECT (self, "Couldn't build convert command");
gst_d3d12_fence_data_unref (fence_data);
return FALSE;
}
if (!gst_d3d12_overlay_compositor_draw (priv->comp,
backbuf, fence_data, cl.Get ())) {
GST_ERROR_OBJECT (self, "Couldn't build overlay command");
gst_d3d12_fence_data_unref (fence_data);
return FALSE;
}
barrier =
CD3DX12_RESOURCE_BARRIER::Transition (backbuf_texture,
D3D12_RESOURCE_STATE_RENDER_TARGET, D3D12_RESOURCE_STATE_COMMON);
cl->ResourceBarrier (1, &barrier);
hr = cl->Close ();
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't close command list");
gst_d3d12_fence_data_unref (fence_data);
return FALSE;
}
ID3D12CommandList *cmd_list[] = { cl.Get () };
hr = gst_d3d12_command_queue_execute_command_lists (priv->cq,
1, cmd_list, &priv->fence_val);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Signal failed");
gst_d3d12_fence_data_unref (fence_data);
return FALSE;
}
gst_d3d12_command_queue_set_notify (priv->cq, priv->fence_val,
fence_data, (GDestroyNotify) gst_d3d12_fence_data_unref);
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_set_buffer (GstD3D12SwapChainSink * self,
GstBuffer * buffer, gboolean is_prepare)
{
auto priv = self->priv;
bool need_render = false;
bool update_converter = false;
std::lock_guard < std::recursive_mutex > lk (priv->lock);
if (!priv->swapchain || priv->backbuf.empty ()) {
GST_ERROR_OBJECT (self, "Swapchain was not configured");
return FALSE;
}
if (is_prepare) {
if (priv->caps_updated) {
need_render = false;
update_converter = false;
} else {
need_render = true;
update_converter = false;
}
} else {
if (priv->caps_updated) {
need_render = true;
update_converter = true;
priv->caps_updated = false;
} else {
need_render = false;
update_converter = false;
}
}
if (update_converter) {
gst_d3d12_command_queue_idle_for_swapchain (priv->cq, priv->fence_val);
auto format = GST_VIDEO_INFO_FORMAT (&priv->info);
if (priv->convert_format != format)
gst_clear_object (&priv->conv);
priv->display_width = GST_VIDEO_SINK_WIDTH (self);
priv->display_height = GST_VIDEO_SINK_HEIGHT (self);
priv->convert_format = format;
priv->crop_rect = CD3DX12_BOX (0, 0, priv->info.width, priv->info.height);
priv->prev_crop_rect = priv->crop_rect;
priv->first_present = true;
gst_clear_buffer (&priv->cached_buf);
if (!priv->conv) {
gst_structure_set (priv->convert_config,
GST_D3D12_CONVERTER_OPT_DEST_ALPHA_MODE,
GST_TYPE_D3D12_CONVERTER_ALPHA_MODE,
GST_VIDEO_INFO_HAS_ALPHA (&priv->info) ?
GST_D3D12_CONVERTER_ALPHA_MODE_PREMULTIPLIED :
GST_D3D12_CONVERTER_ALPHA_MODE_UNSPECIFIED, nullptr);
priv->conv = gst_d3d12_converter_new (self->device, nullptr, &priv->info,
&priv->display_info, nullptr, nullptr,
gst_structure_copy (priv->convert_config));
if (!priv->conv) {
GST_ERROR_OBJECT (self, "Couldn't create converter");
return FALSE;
}
} else {
g_object_set (priv->conv, "src-x", 0, "src-y", 0,
"src-width", priv->info.width, "src-height", priv->info.height,
nullptr);
}
}
if (!need_render)
return TRUE;
auto mem = gst_buffer_peek_memory (buffer, 0);
if (!gst_is_d3d12_memory (mem)) {
GstBuffer *upload = nullptr;
gst_buffer_pool_acquire_buffer (priv->pool, &upload, nullptr);
if (!upload) {
GST_ERROR_OBJECT (self, "Couldn't allocate upload buffer");
return FALSE;
}
GstVideoFrame in_frame, out_frame;
if (!gst_video_frame_map (&in_frame, &priv->info, buffer, GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Couldn't map input frame");
gst_buffer_unref (upload);
return FALSE;
}
if (!gst_video_frame_map (&out_frame, &priv->info, upload, GST_MAP_WRITE)) {
GST_ERROR_OBJECT (self, "Couldn't map upload frame");
gst_video_frame_unmap (&in_frame);
gst_buffer_unref (upload);
return FALSE;
}
auto copy_ret = gst_video_frame_copy (&out_frame, &in_frame);
gst_video_frame_unmap (&out_frame);
gst_video_frame_unmap (&in_frame);
if (!copy_ret) {
GST_ERROR_OBJECT (self, "Couldn't copy frame");
gst_buffer_unref (upload);
return FALSE;
}
gst_buffer_foreach_meta (buffer,
(GstBufferForeachMetaFunc) gst_d3d12_swapchain_sink_foreach_meta,
upload);
gst_clear_buffer (&priv->cached_buf);
priv->cached_buf = upload;
} else {
gst_buffer_replace (&priv->cached_buf, buffer);
}
return gst_d3d12_swapchain_sink_render (self);
}
static void
gst_d3d12_swapchain_sink_resize (GstD3D12SwapChainSink * self, guint width,
guint height)
{
auto priv = self->priv;
if (width == 0 || width > D3D12_REQ_TEXTURE2D_U_OR_V_DIMENSION) {
GST_WARNING_OBJECT (self, "Invalid width value %u", width);
return;
}
if (height == 0 || height > D3D12_REQ_TEXTURE2D_U_OR_V_DIMENSION) {
GST_WARNING_OBJECT (self, "Invalid height value %u", height);
return;
}
std::lock_guard < std::recursive_mutex > lk (priv->lock);
if (!gst_d3d12_swapchain_sink_resize_unlocked (self, width, height)) {
GST_ERROR_OBJECT (self, "Couldn't resize swapchain");
return;
}
if (priv->swapchain && priv->cached_buf &&
gst_d3d12_swapchain_sink_render (self)) {
GST_ERROR_OBJECT (self, "resize %ux%u", width, height);
auto hr = priv->swapchain->Present (0, 0);
if (!gst_d3d12_result (hr, self->device))
GST_ERROR_OBJECT (self, "Present failed");
gst_d3d12_command_queue_execute_command_lists (priv->cq,
0, nullptr, &priv->fence_val);
}
}
static gboolean
gst_d3d12_swapchain_sink_start (GstBaseSink * sink)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
auto priv = self->priv;
GST_DEBUG_OBJECT (self, "Start");
std::lock_guard < std::recursive_mutex > lk (priv->lock);
if (!gst_d3d12_swapchain_sink_ensure_swapchain (self)) {
GST_ERROR_OBJECT (self, "Couldn't create swapchain");
return FALSE;
}
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_stop (GstBaseSink * sink)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
auto priv = self->priv;
GST_DEBUG_OBJECT (self, "Stop");
priv->stop ();
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_propose_allocation (GstBaseSink * sink,
GstQuery * query)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
GstCaps *caps;
GstBufferPool *pool = nullptr;
GstVideoInfo info;
guint size;
gboolean need_pool;
if (!self->device) {
GST_WARNING_OBJECT (self, "No configured device");
return FALSE;
}
gst_query_parse_allocation (query, &caps, &need_pool);
if (!caps) {
GST_WARNING_OBJECT (self, "no caps specified");
return FALSE;
}
if (!gst_video_info_from_caps (&info, caps)) {
GST_ERROR_OBJECT (self, "Invalid caps %" GST_PTR_FORMAT, caps);
return FALSE;
}
size = info.size;
bool is_d3d12 = false;
auto features = gst_caps_get_features (caps, 0);
if (gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) {
is_d3d12 = true;
GST_DEBUG_OBJECT (self, "upstream support d3d12 memory");
}
if (need_pool) {
if (is_d3d12)
pool = gst_d3d12_buffer_pool_new (self->device);
else
pool = gst_video_buffer_pool_new ();
auto config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
if (!is_d3d12) {
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
}
gst_buffer_pool_config_set_params (config, caps, size, 2, 0);
if (!gst_buffer_pool_set_config (pool, config)) {
GST_ERROR_OBJECT (pool, "Couldn't set config");
gst_object_unref (pool);
return FALSE;
}
if (is_d3d12) {
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr,
nullptr);
gst_structure_free (config);
}
}
gst_query_add_allocation_pool (query, pool, size, 2, 0);
gst_clear_object (&pool);
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, nullptr);
gst_query_add_allocation_meta (query,
GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, nullptr);
if (is_d3d12) {
gst_query_add_allocation_meta (query,
GST_VIDEO_CROP_META_API_TYPE, nullptr);
}
return TRUE;
}
static gboolean
gst_d3d12_swapchain_sink_query (GstBaseSink * sink, GstQuery * query)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
if (gst_d3d12_handle_context_query (GST_ELEMENT (self), query,
self->device)) {
return TRUE;
}
break;
default:
break;
}
return GST_BASE_SINK_CLASS (parent_class)->query (sink, query);
}
static GstFlowReturn
gst_d3d12_swapchain_sink_prepare (GstBaseSink * sink, GstBuffer * buffer)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
auto priv = self->priv;
std::lock_guard < std::recursive_mutex > lk (priv->lock);
if (!gst_d3d12_swapchain_sink_set_buffer (self, buffer, TRUE)) {
GST_ERROR_OBJECT (self, "Set buffer failed");
return GST_FLOW_ERROR;
}
return GST_FLOW_OK;
}
static GstFlowReturn
gst_d3d12_swapchain_sink_show_frame (GstVideoSink * sink, GstBuffer * buf)
{
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
auto priv = self->priv;
std::lock_guard < std::recursive_mutex > lk (priv->lock);
if (!gst_d3d12_swapchain_sink_set_buffer (self, buf, FALSE)) {
GST_ERROR_OBJECT (self, "Set buffer failed");
return GST_FLOW_ERROR;
}
auto hr = priv->swapchain->Present (0, 0);
if (!gst_d3d12_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Present failed");
return GST_FLOW_ERROR;
}
/* To update fence value */
gst_d3d12_command_queue_execute_command_lists (priv->cq,
0, nullptr, &priv->fence_val);
return GST_FLOW_OK;
}
Reference in a new issue View git blame Copy permalink