diff --git a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json index 610205d985..497e40f636 100644 --- a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json +++ b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json @@ -11349,6 +11349,78 @@ }, "rank": "none" }, + "d3d12compositor": { + "author": "Seungha Yang ", + "description": "A Direct3D12 compositor", + "hierarchy": [ + "GstD3D12Compositor", + "GstVideoAggregator", + "GstAggregator", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "interfaces": [ + "GstChildProxy" + ], + "klass": "Filter/Editor/Video/Compositor", + "pad-templates": { + "sink_%%u": { + "caps": "video/x-raw(memory:D3D12Memory):\n format: { RGBA64_LE, RGB10A2_LE, Y410, VUYA, RGBA, BGRA, RBGA, P016_LE, P012_LE, P010_10LE, RGBx, BGRx, NV12, AYUV64, GBRA_12LE, GBRA_10LE, AYUV, ABGR, ARGB, GBRA, Y444_16LE, GBR_16LE, Y444_12LE, GBR_12LE, I422_12LE, I420_12LE, Y444_10LE, GBR_10LE, I422_10LE, I420_10LE, Y444, BGRP, GBR, RGBP, xBGR, xRGB, Y42B, NV21, I420, YV12, GRAY16_LE, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n format: { RGBA64_LE, RGB10A2_LE, Y410, VUYA, RGBA, BGRA, RBGA, P016_LE, P012_LE, P010_10LE, RGBx, BGRx, NV12, AYUV64, GBRA_12LE, GBRA_10LE, AYUV, ABGR, ARGB, GBRA, Y444_16LE, GBR_16LE, Y444_12LE, GBR_12LE, I422_12LE, I420_12LE, Y444_10LE, GBR_10LE, I422_10LE, I420_10LE, Y444, BGRP, GBR, RGBP, xBGR, xRGB, Y42B, NV21, I420, YV12, GRAY16_LE, GRAY8 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "sink", + "presence": "request", + "type": "GstD3D12CompositorPad" + }, + "src": { + "caps": "video/x-raw(memory:D3D12Memory):\n format: { RGBA64_LE, RGB10A2_LE, BGRA, RGBA, BGRx, RGBx, VUYA, NV12, NV21, P010_10LE, P012_LE, P016_LE, I420, YV12, Y42B, Y444, Y444_16LE, GRAY8, GRAY16_LE }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n format: { RGBA64_LE, RGB10A2_LE, BGRA, RGBA, BGRx, RGBx, VUYA, NV12, NV21, P010_10LE, P012_LE, P016_LE, I420, YV12, Y42B, Y444, Y444_16LE, GRAY8, GRAY16_LE }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n", + "direction": "src", + "presence": "always", + "type": "GstAggregatorPad" + } + }, + "properties": { + "adapter": { + "blurb": "Adapter index for creating device (-1 for default)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "-1", + "max": "2147483647", + "min": "-1", + "mutable": "ready", + "readable": true, + "type": "gint", + "writable": true + }, + "background": { + "blurb": "Background type", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "checker (0)", + "mutable": "null", + "readable": true, + "type": "GstD3D12CompositorBackground", + "writable": true + }, + "ignore-inactive-pads": { + "blurb": "Avoid timing out waiting for inactive pads", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + } + }, + "rank": "none" + }, "d3d12convert": { "author": "Seungha Yang ", "description": "Performs resizing, colorspace conversion, cropping and flipping/rotating using Direct3D12", @@ -11881,6 +11953,169 @@ ], "kind": "object" }, + "GstD3D12CompositorBackground": { + "kind": "enum", + "values": [ + { + "desc": "Checker pattern", + "name": "checker", + "value": "0" + }, + { + "desc": "Black", + "name": "black", + "value": "1" + }, + { + "desc": "White", + "name": "white", + "value": "2" + }, + { + "desc": "Transparent Background to enable further compositing", + "name": "transparent", + "value": "3" + } + ] + }, + "GstD3D12CompositorOperator": { + "kind": "enum", + "values": [ + { + "desc": "Source", + "name": "source", + "value": "0" + }, + { + "desc": "Over", + "name": "over", + "value": "1" + } + ] + }, + "GstD3D12CompositorPad": { + "hierarchy": [ + "GstD3D12CompositorPad", + "GstVideoAggregatorPad", + "GstAggregatorPad", + "GstPad", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "kind": "object", + "properties": { + "alpha": { + "blurb": "Alpha of the picture", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "1", + "max": "1", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gdouble", + "writable": true + }, + "height": { + "blurb": "Height of the picture", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "0", + "max": "2147483647", + "min": "-2147483648", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "operator": { + "blurb": "Blending operator to use for blending this pad over the previous ones", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "over (1)", + "mutable": "null", + "readable": true, + "type": "GstD3D12CompositorOperator", + "writable": true + }, + "sizing-policy": { + "blurb": "Sizing policy to use for image scaling", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "none (0)", + "mutable": "null", + "readable": true, + "type": "GstD3D12CompositorSizingPolicy", + "writable": true + }, + "width": { + "blurb": "Width of the picture", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "0", + "max": "2147483647", + "min": "-2147483648", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "xpos": { + "blurb": "X position of the picture", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "0", + "max": "2147483647", + "min": "-2147483648", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + }, + "ypos": { + "blurb": "Y position of the picture", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": true, + "default": "0", + "max": "2147483647", + "min": "-2147483648", + "mutable": "null", + "readable": true, + "type": "gint", + "writable": true + } + } + }, + "GstD3D12CompositorSizingPolicy": { + "kind": "enum", + "values": [ + { + "desc": "None: Image is scaled to fill configured destination rectangle without padding or keeping the aspect ratio", + "name": "none", + "value": "0" + }, + { + "desc": "Keep Aspect Ratio: Image is scaled to fit destination rectangle specified by GstD3D12CompositorPad:{xpos, ypos, width, height} with preserved aspect ratio. Resulting image will be centered in the destination rectangle with padding if necessary", + "name": "keep-aspect-ratio", + "value": "1" + } + ] + }, "GstD3D12SamplingMethod": { "kind": "enum", "values": [ diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.cpp b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.cpp new file mode 100644 index 0000000000..ef6900cf53 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.cpp @@ -0,0 +1,2546 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-d3d12compositor + * @title: d3d12compositor + * + * A Direct3D12 based video compositing element. + * + * ## Example launch line + * ``` + * gst-launch-1.0 d3d12compositor name=c ! d3d12videosink \ + * videotestsrc ! video/x-raw,width=320,height=240 ! c. \ + * videotestsrc pattern=ball ! video/x-raw,width=100,height=100 ! c. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d12compositor.h" +#include "gstd3d12pluginutils.h" +#include +#include +#include +#include +#include +#include +#include +#include "PSMain_checker_luma.h" +#include "PSMain_checker_rgb.h" +#include "PSMain_checker_vuya.h" +#include "VSMain_pos.h" + +GST_DEBUG_CATEGORY_STATIC (gst_d3d12_compositor_debug); +#define GST_CAT_DEFAULT gst_d3d12_compositor_debug + +/* *INDENT-OFF* */ +using namespace Microsoft::WRL; +/* *INDENT-ON* */ + +enum GstD3D12CompositorBackground +{ + GST_D3D12_COMPOSITOR_BACKGROUND_CHECKER, + GST_D3D12_COMPOSITOR_BACKGROUND_BLACK, + GST_D3D12_COMPOSITOR_BACKGROUND_WHITE, + GST_D3D12_COMPOSITOR_BACKGROUND_TRANSPARENT, +}; + +#define GST_TYPE_D3D12_COMPOSITOR_BACKGROUND (gst_d3d12_compositor_background_get_type()) +static GType +gst_d3d12_compositor_background_get_type (void) +{ + static GType compositor_background_type = 0; + static const GEnumValue compositor_background[] = { + {GST_D3D12_COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_D3D12_COMPOSITOR_BACKGROUND_BLACK, "Black", "black"}, + {GST_D3D12_COMPOSITOR_BACKGROUND_WHITE, "White", "white"}, + {GST_D3D12_COMPOSITOR_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, nullptr, nullptr}, + }; + + GST_D3D12_CALL_ONCE_BEGIN { + compositor_background_type = + g_enum_register_static ("GstD3D12CompositorBackground", + compositor_background); + } GST_D3D12_CALL_ONCE_END; + + return compositor_background_type; +} + +enum GstD3D12CompositorOperator +{ + GST_D3D12_COMPOSITOR_OPERATOR_SOURCE, + GST_D3D12_COMPOSITOR_OPERATOR_OVER, +}; + +#define GST_TYPE_D3D12_COMPOSITOR_OPERATOR (gst_d3d12_compositor_operator_get_type()) +static GType +gst_d3d12_compositor_operator_get_type (void) +{ + static GType compositor_operator_type = 0; + static const GEnumValue compositor_operator[] = { + {GST_D3D12_COMPOSITOR_OPERATOR_SOURCE, "Source", "source"}, + {GST_D3D12_COMPOSITOR_OPERATOR_OVER, "Over", "over"}, + {0, nullptr, nullptr}, + }; + + GST_D3D12_CALL_ONCE_BEGIN { + compositor_operator_type = + g_enum_register_static ("GstD3D12CompositorOperator", + compositor_operator); + } GST_D3D12_CALL_ONCE_END; + + return compositor_operator_type; +} + +enum GstD3D12CompositorSizingPolicy +{ + GST_D3D12_COMPOSITOR_SIZING_POLICY_NONE, + GST_D3D12_COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO, +}; + +#define GST_TYPE_D3D12_COMPOSITOR_SIZING_POLICY (gst_d3d12_compositor_sizing_policy_get_type()) +static GType +gst_d3d12_compositor_sizing_policy_get_type (void) +{ + static GType sizing_policy_type = 0; + + static const GEnumValue sizing_polices[] = { + {GST_D3D12_COMPOSITOR_SIZING_POLICY_NONE, + "None: Image is scaled to fill configured destination rectangle without " + "padding or keeping the aspect ratio", "none"}, + {GST_D3D12_COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO, + "Keep Aspect Ratio: Image is scaled to fit destination rectangle " + "specified by GstD3D12CompositorPad:{xpos, ypos, width, height} " + "with preserved aspect ratio. Resulting image will be centered in " + "the destination rectangle with padding if necessary", + "keep-aspect-ratio"}, + {0, nullptr, nullptr}, + }; + + GST_D3D12_CALL_ONCE_BEGIN { + sizing_policy_type = + g_enum_register_static ("GstD3D12CompositorSizingPolicy", + sizing_polices); + } GST_D3D12_CALL_ONCE_END; + + return sizing_policy_type; +} + +enum +{ + PROP_PAD_0, + PROP_PAD_XPOS, + PROP_PAD_YPOS, + PROP_PAD_WIDTH, + PROP_PAD_HEIGHT, + PROP_PAD_ALPHA, + PROP_PAD_OPERATOR, + PROP_PAD_SIZING_POLICY, + PROP_PAD_GAMMA_MODE, + PROP_PAD_PRIMARIES_MODE, +}; + +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 +#define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_OPERATOR GST_D3D12_COMPOSITOR_OPERATOR_OVER +#define DEFAULT_PAD_SIZING_POLICY GST_D3D12_COMPOSITOR_SIZING_POLICY_NONE + +enum +{ + PROP_0, + PROP_ADAPTER, + PROP_BACKGROUND, + PROP_IGNORE_INACTIVE_PADS, +}; + +#define DEFAULT_ADAPTER -1 +#define DEFAULT_BACKGROUND GST_D3D12_COMPOSITOR_BACKGROUND_CHECKER + +static const D3D12_RENDER_TARGET_BLEND_DESC g_blend_source = { + TRUE, + FALSE, + D3D12_BLEND_ONE, + D3D12_BLEND_ZERO, + D3D12_BLEND_OP_ADD, + D3D12_BLEND_ONE, + D3D12_BLEND_ZERO, + D3D12_BLEND_OP_ADD, + D3D12_LOGIC_OP_NOOP, + D3D12_COLOR_WRITE_ENABLE_ALL, +}; + +static const D3D12_RENDER_TARGET_BLEND_DESC g_blend_over = { + TRUE, + FALSE, + D3D12_BLEND_SRC_ALPHA, + D3D12_BLEND_INV_SRC_ALPHA, + D3D12_BLEND_OP_ADD, + D3D12_BLEND_ONE, + D3D12_BLEND_INV_SRC_ALPHA, + D3D12_BLEND_OP_ADD, + D3D12_LOGIC_OP_NOOP, + D3D12_COLOR_WRITE_ENABLE_ALL, +}; + +static const D3D12_RENDER_TARGET_BLEND_DESC g_blend_over_factor = { + TRUE, + FALSE, + D3D12_BLEND_BLEND_FACTOR, + D3D12_BLEND_INV_BLEND_FACTOR, + D3D12_BLEND_OP_ADD, + D3D12_BLEND_BLEND_FACTOR, + D3D12_BLEND_INV_BLEND_FACTOR, + D3D12_BLEND_OP_ADD, + D3D12_LOGIC_OP_NOOP, + D3D12_COLOR_WRITE_ENABLE_ALL, +}; + +static const D3D12_ROOT_SIGNATURE_FLAGS g_rs_flags = + D3D12_ROOT_SIGNATURE_FLAG_ALLOW_INPUT_ASSEMBLER_INPUT_LAYOUT | + D3D12_ROOT_SIGNATURE_FLAG_DENY_HULL_SHADER_ROOT_ACCESS | + D3D12_ROOT_SIGNATURE_FLAG_DENY_DOMAIN_SHADER_ROOT_ACCESS | + D3D12_ROOT_SIGNATURE_FLAG_DENY_GEOMETRY_SHADER_ROOT_ACCESS | + D3D12_ROOT_SIGNATURE_FLAG_DENY_AMPLIFICATION_SHADER_ROOT_ACCESS | + D3D12_ROOT_SIGNATURE_FLAG_DENY_MESH_SHADER_ROOT_ACCESS; + +struct PadContext +{ + PadContext (GstD3D12Device * dev) + { + event_handle = CreateEventEx (nullptr, nullptr, 0, EVENT_ALL_ACCESS); + device = (GstD3D12Device *) gst_object_ref (dev); + ca_pool = gst_d3d12_command_allocator_pool_new (device, + D3D12_COMMAND_LIST_TYPE_DIRECT); + gst_video_info_init (&info); + } + + PadContext () = delete; + + ~PadContext () { + gst_d3d12_device_fence_wait (device, D3D12_COMMAND_LIST_TYPE_DIRECT, + fence_val, event_handle); + + CloseHandle (event_handle); + + gst_clear_d3d12_fence_data (&fence_data); + gst_clear_object (&conv); + gst_clear_object (&ca_pool); + gst_clear_object (&device); + } + + GstVideoInfo info; + GstD3D12CommandAllocatorPool *ca_pool; + ComPtr < ID3D12GraphicsCommandList > cl; + GstD3D12FenceData *fence_data = nullptr; + GstD3D12Device *device; + GstD3D12Converter *conv = nullptr; + HANDLE event_handle; + guint64 fence_val = 0; +}; + +struct GstD3D12CompositorPadPrivate +{ + GstD3D12CompositorPadPrivate () + { + blend_desc = CD3DX12_BLEND_DESC (D3D12_DEFAULT); + blend_desc.RenderTarget[0] = g_blend_over; + for (guint i = 0; i < 4; i++) + blend_factor[i] = 1.0f; + } + + std::unique_ptr < PadContext > ctx; + std::future < gboolean > prepare_rst; + + gboolean position_updated = FALSE; + gboolean alpha_updated = FALSE; + gboolean blend_desc_updated = FALSE; + D3D12_BLEND_DESC blend_desc; + gfloat blend_factor[4]; + + std::recursive_mutex lock; + + /* properties */ + gint xpos = DEFAULT_PAD_XPOS; + gint ypos = DEFAULT_PAD_YPOS; + gint width = DEFAULT_PAD_WIDTH; + gint height = DEFAULT_PAD_HEIGHT; + gdouble alpha = DEFAULT_PAD_ALPHA; + GstD3D12CompositorOperator op = DEFAULT_PAD_OPERATOR; + GstD3D12CompositorSizingPolicy sizing_policy = DEFAULT_PAD_SIZING_POLICY; +}; + +struct _GstD3D12CompositorPad +{ + GstVideoAggregatorConvertPad parent; + + GstD3D12CompositorPadPrivate *priv; +}; + +struct VertexData +{ + struct + { + FLOAT x; + FLOAT y; + FLOAT z; + } position; + struct + { + FLOAT u; + FLOAT v; + } texture; +}; + +struct BackgroundRender +{ + BackgroundRender (GstD3D12Device * dev, const GstVideoInfo & info) + { + event_handle = CreateEventEx (nullptr, nullptr, 0, EVENT_ALL_ACCESS); + device = (GstD3D12Device *) gst_object_ref (dev); + ca_pool = gst_d3d12_command_allocator_pool_new (device, + D3D12_COMMAND_LIST_TYPE_DIRECT); + + D3D12_VERSIONED_ROOT_SIGNATURE_DESC rs_desc = { }; + CD3DX12_VERSIONED_ROOT_SIGNATURE_DESC::Init_1_0 (rs_desc, + 0, nullptr, 0, nullptr, g_rs_flags); + + ComPtr < ID3DBlob > rs_blob; + ComPtr < ID3DBlob > error_blob; + auto hr = D3DX12SerializeVersionedRootSignature (&rs_desc, + D3D_ROOT_SIGNATURE_VERSION_1_1, &rs_blob, &error_blob); + if (!gst_d3d12_result (hr, device)) { + const gchar *error_msg = nullptr; + if (error_blob) + error_msg = (const gchar *) error_blob->GetBufferPointer (); + + GST_ERROR_OBJECT (device, "Couldn't serialize root signature, error: %s", + GST_STR_NULL (error_msg)); + return; + } + + auto device_handle = gst_d3d12_device_get_device_handle (device); + hr = device_handle->CreateRootSignature (0, rs_blob->GetBufferPointer (), + rs_blob->GetBufferSize (), IID_PPV_ARGS (&rs)); + if (!gst_d3d12_result (hr, device)) { + GST_ERROR_OBJECT (device, "Couldn't create root signature"); + return; + } + + GstD3D12Format format; + gst_d3d12_device_get_format (device, GST_VIDEO_INFO_FORMAT (&info), + &format); + + D3D12_INPUT_ELEMENT_DESC input_desc; + input_desc.SemanticName = "POSITION"; + input_desc.SemanticIndex = 0; + input_desc.Format = DXGI_FORMAT_R32G32B32_FLOAT; + input_desc.InputSlot = 0; + input_desc.AlignedByteOffset = D3D12_APPEND_ALIGNED_ELEMENT; + input_desc.InputSlotClass = D3D12_INPUT_CLASSIFICATION_PER_VERTEX_DATA; + input_desc.InstanceDataStepRate = 0; + + D3D12_GRAPHICS_PIPELINE_STATE_DESC pso_desc = { }; + pso_desc.pRootSignature = rs.Get (); + pso_desc.VS.BytecodeLength = sizeof (g_VSMain_pos); + pso_desc.VS.pShaderBytecode = g_VSMain_pos; + if (GST_VIDEO_INFO_IS_RGB (&info)) { + pso_desc.PS.BytecodeLength = sizeof (g_PSMain_checker_rgb); + pso_desc.PS.pShaderBytecode = g_PSMain_checker_rgb; + } else if (GST_VIDEO_INFO_FORMAT (&info) == GST_VIDEO_FORMAT_VUYA) { + pso_desc.PS.BytecodeLength = sizeof (g_PSMain_checker_vuya); + pso_desc.PS.pShaderBytecode = g_PSMain_checker_vuya; + } else { + pso_desc.PS.BytecodeLength = sizeof (g_PSMain_checker_luma); + pso_desc.PS.pShaderBytecode = g_PSMain_checker_luma; + } + pso_desc.BlendState = CD3DX12_BLEND_DESC (D3D12_DEFAULT); + pso_desc.SampleMask = UINT_MAX; + pso_desc.RasterizerState = CD3DX12_RASTERIZER_DESC (D3D12_DEFAULT); + pso_desc.RasterizerState.CullMode = D3D12_CULL_MODE_NONE; + pso_desc.DepthStencilState.DepthEnable = FALSE; + pso_desc.DepthStencilState.StencilEnable = FALSE; + pso_desc.InputLayout.pInputElementDescs = &input_desc; + pso_desc.InputLayout.NumElements = 1; + pso_desc.PrimitiveTopologyType = D3D12_PRIMITIVE_TOPOLOGY_TYPE_TRIANGLE; + pso_desc.NumRenderTargets = 1; + pso_desc.RTVFormats[0] = format.resource_format[0]; + pso_desc.SampleDesc.Count = 1; + + hr = device_handle->CreateGraphicsPipelineState (&pso_desc, + IID_PPV_ARGS (&pso)); + if (!gst_d3d12_result (hr, device)) { + GST_ERROR_OBJECT (device, "Couldn't create pso"); + return; + } + + VertexData vertex_data[4]; + const WORD indices[6] = { 0, 1, 2, 3, 0, 2 }; + + /* bottom left */ + vertex_data[0].position.x = -1.0f; + vertex_data[0].position.y = -1.0f; + vertex_data[0].position.z = 0.0f; + vertex_data[0].texture.u = 0.0f; + vertex_data[0].texture.v = 1.0f; + + /* top left */ + vertex_data[1].position.x = -1.0f; + vertex_data[1].position.y = 1.0f; + vertex_data[1].position.z = 0.0f; + vertex_data[1].texture.u = 0.0f; + vertex_data[1].texture.v = 0.0f; + + /* top right */ + vertex_data[2].position.x = 1.0f; + vertex_data[2].position.y = 1.0f; + vertex_data[2].position.z = 0.0f; + vertex_data[2].texture.u = 1.0f; + vertex_data[2].texture.v = 0.0f; + + /* bottom right */ + vertex_data[3].position.x = 1.0f; + vertex_data[3].position.y = -1.0f; + vertex_data[3].position.z = 0.0f; + vertex_data[3].texture.u = 1.0f; + vertex_data[3].texture.v = 1.0f; + + D3D12_HEAP_PROPERTIES heap_prop = + CD3DX12_HEAP_PROPERTIES (D3D12_HEAP_TYPE_UPLOAD); + D3D12_RESOURCE_DESC buffer_desc = + CD3DX12_RESOURCE_DESC::Buffer (sizeof (VertexData) * 4 + + sizeof (indices)); + + hr = device_handle->CreateCommittedResource (&heap_prop, + D3D12_HEAP_FLAG_NONE, &buffer_desc, D3D12_RESOURCE_STATE_GENERIC_READ, + nullptr, IID_PPV_ARGS (&vertex_index_upload)); + if (!gst_d3d12_result (hr, device)) { + GST_ERROR_OBJECT (device, "Couldn't create vertex upload buf"); + return; + } + + guint8 *data; + CD3DX12_RANGE range (0, 0); + hr = vertex_index_upload->Map (0, &range, (void **) &data); + if (!gst_d3d12_result (hr, device)) { + GST_ERROR_OBJECT (device, "Couldn't map index buffer"); + return; + } + + memcpy (data, vertex_data, sizeof (VertexData) * 4); + memcpy (data + sizeof (VertexData) * 4, indices, sizeof (indices)); + vertex_index_upload->Unmap (0, nullptr); + + heap_prop = CD3DX12_HEAP_PROPERTIES (D3D12_HEAP_TYPE_DEFAULT); + hr = device_handle->CreateCommittedResource (&heap_prop, + D3D12_HEAP_FLAG_NONE, &buffer_desc, D3D12_RESOURCE_STATE_COPY_DEST, + nullptr, IID_PPV_ARGS (&vertex_index_buf)); + if (!gst_d3d12_result (hr, device)) { + GST_ERROR_OBJECT (device, "Couldn't create index buffer"); + return; + } + + vbv.BufferLocation = vertex_index_buf->GetGPUVirtualAddress (); + vbv.SizeInBytes = sizeof (VertexData) * 4; + vbv.StrideInBytes = sizeof (VertexData); + ibv.BufferLocation = vbv.BufferLocation + vbv.SizeInBytes; + ibv.SizeInBytes = sizeof (indices); + ibv.Format = DXGI_FORMAT_R16_UINT; + + viewport.TopLeftX = 0; + viewport.TopLeftY = 0; + viewport.Width = info.width; + viewport.Height = info.height; + viewport.MinDepth = 0; + viewport.MaxDepth = 1; + + scissor_rect.left = 0; + scissor_rect.top = 0; + scissor_rect.right = info.width; + scissor_rect.bottom = info.height; + + rtv_inc_size = + device_handle->GetDescriptorHandleIncrementSize + (D3D12_DESCRIPTOR_HEAP_TYPE_RTV); + + is_valid = true; + } + BackgroundRender () = delete; + + ~BackgroundRender () { + gst_d3d12_device_fence_wait (device, D3D12_COMMAND_LIST_TYPE_DIRECT, + fence_val, event_handle); + + CloseHandle (event_handle); + + gst_clear_object (&ca_pool); + gst_clear_object (&device); + } + + GstD3D12Device *device; + ComPtr < ID3D12RootSignature > rs; + ComPtr < ID3D12PipelineState > pso; + ComPtr < ID3D12Resource > vertex_index_buf; + ComPtr < ID3D12Resource > vertex_index_upload; + D3D12_VERTEX_BUFFER_VIEW vbv; + D3D12_INDEX_BUFFER_VIEW ibv; + ComPtr < ID3D12GraphicsCommandList > cl; + GstD3D12CommandAllocatorPool *ca_pool; + D3D12_VIEWPORT viewport; + D3D12_RECT scissor_rect; + guint rtv_inc_size; + bool need_upload = true; + bool is_valid = false; + HANDLE event_handle; + guint64 fence_val = 0; +}; + +struct ClearColor +{ + /* [rtv][colors] */ + FLOAT color[4][4]; +}; + +struct GStD3D12CompositorPrivate +{ + GStD3D12CompositorPrivate () + { + fence_data_pool = gst_d3d12_fence_data_pool_new (); + gst_video_info_init (&negotiated_info); + } + + ~GStD3D12CompositorPrivate () + { + gst_clear_buffer (&fallback_buf); + gst_clear_object (&fence_data_pool); + } + + GstBuffer *fallback_buf = nullptr; + GstBuffer *generated_output_buf = nullptr; + + std::unique_ptr < BackgroundRender > bg_render; + /* black/white/transparent */ + ClearColor clear_color[3]; + GstD3D12FenceDataPool *fence_data_pool; + std::vector < D3D12_CPU_DESCRIPTOR_HANDLE > rtv_handles; + std::queue < guint64 > scheduled; + + GstVideoInfo negotiated_info; + + gboolean downstream_supports_d3d12 = FALSE; + + std::recursive_mutex lock; + + /* properties */ + gint adapter = DEFAULT_ADAPTER; + GstD3D12CompositorBackground background = DEFAULT_BACKGROUND; +}; + +struct _GstD3D12Compositor +{ + GstVideoAggregator parent; + + GstD3D12Device *device; + + GStD3D12CompositorPrivate *priv; +}; + +static void gst_d3d12_compositor_pad_finalize (GObject * object); +static void gst_d3d12_compositor_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d12_compositor_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void +gst_d3d12_compositor_pad_update_conversion_info (GstVideoAggregatorPad * pad); +static void +gst_d3d12_compositor_pad_prepare_frame_start (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame); +static void +gst_d3d12_compositor_pad_prepare_frame_finish (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame); +static void gst_d3d12_compositor_pad_clean_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame); + +#define gst_d3d12_compositor_pad_parent_class parent_pad_class +G_DEFINE_TYPE (GstD3D12CompositorPad, gst_d3d12_compositor_pad, + GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_d3d12_compositor_pad_class_init (GstD3D12CompositorPadClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto vagg_pad_class = GST_VIDEO_AGGREGATOR_PAD_CLASS (klass); + GParamFlags param_flags = (GParamFlags) + (G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS); + + object_class->finalize = gst_d3d12_compositor_pad_finalize; + object_class->set_property = gst_d3d12_compositor_pad_set_property; + object_class->get_property = gst_d3d12_compositor_pad_get_property; + + g_object_class_install_property (object_class, PROP_PAD_XPOS, + g_param_spec_int ("xpos", "X Position", "X position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, param_flags)); + g_object_class_install_property (object_class, PROP_PAD_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, param_flags)); + g_object_class_install_property (object_class, PROP_PAD_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, param_flags)); + g_object_class_install_property (object_class, PROP_PAD_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, param_flags)); + g_object_class_install_property (object_class, PROP_PAD_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, param_flags)); + g_object_class_install_property (object_class, PROP_PAD_OPERATOR, + g_param_spec_enum ("operator", "Operator", + "Blending operator to use for blending this pad over the previous ones", + GST_TYPE_D3D12_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR, + param_flags)); + g_object_class_install_property (object_class, PROP_PAD_SIZING_POLICY, + g_param_spec_enum ("sizing-policy", "Sizing policy", + "Sizing policy to use for image scaling", + GST_TYPE_D3D12_COMPOSITOR_SIZING_POLICY, DEFAULT_PAD_SIZING_POLICY, + param_flags)); + + vagg_pad_class->update_conversion_info = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_pad_update_conversion_info); + vagg_pad_class->prepare_frame_start = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_pad_prepare_frame_start); + vagg_pad_class->prepare_frame_finish = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_pad_prepare_frame_finish); + vagg_pad_class->clean_frame = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_pad_clean_frame); + + gst_type_mark_as_plugin_api (GST_TYPE_D3D12_COMPOSITOR_OPERATOR, + (GstPluginAPIFlags) 0); + gst_type_mark_as_plugin_api (GST_TYPE_D3D12_COMPOSITOR_SIZING_POLICY, + (GstPluginAPIFlags) 0); +} + +static void +gst_d3d12_compositor_pad_init (GstD3D12CompositorPad * pad) +{ + pad->priv = new GstD3D12CompositorPadPrivate (); +} + +static void +gst_d3d12_compositor_pad_finalize (GObject * object) +{ + auto self = GST_D3D12_COMPOSITOR_PAD (object); + + delete self->priv; + + G_OBJECT_CLASS (parent_pad_class)->finalize (object); +} + +static void +gst_d3d12_compositor_pad_update_position (GstD3D12CompositorPad * pad, + gint * old, const GValue * value) +{ + auto priv = pad->priv; + gint tmp = g_value_get_int (value); + + if (*old != tmp) { + *old = tmp; + priv->position_updated = TRUE; + } +} + +static void +gst_d3d12_compositor_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + auto pad = GST_D3D12_COMPOSITOR_PAD (object); + auto priv = pad->priv; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + switch (prop_id) { + case PROP_PAD_XPOS: + gst_d3d12_compositor_pad_update_position (pad, &priv->xpos, value); + break; + case PROP_PAD_YPOS: + gst_d3d12_compositor_pad_update_position (pad, &priv->ypos, value); + break; + case PROP_PAD_WIDTH: + gst_d3d12_compositor_pad_update_position (pad, &priv->width, value); + break; + case PROP_PAD_HEIGHT: + gst_d3d12_compositor_pad_update_position (pad, &priv->height, value); + break; + case PROP_PAD_ALPHA:{ + gdouble alpha = g_value_get_double (value); + if (priv->alpha != alpha) { + priv->alpha_updated = TRUE; + priv->alpha = alpha; + } + break; + } + case PROP_PAD_OPERATOR:{ + GstD3D12CompositorOperator op = + (GstD3D12CompositorOperator) g_value_get_enum (value); + if (op != priv->op) { + priv->op = op; + priv->blend_desc_updated = TRUE; + } + break; + } + case PROP_PAD_SIZING_POLICY:{ + GstD3D12CompositorSizingPolicy policy = + (GstD3D12CompositorSizingPolicy) g_value_get_enum (value); + if (priv->sizing_policy != policy) { + priv->sizing_policy = policy; + priv->position_updated = TRUE; + } + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d12_compositor_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + auto pad = GST_D3D12_COMPOSITOR_PAD (object); + auto priv = pad->priv; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + switch (prop_id) { + case PROP_PAD_XPOS: + g_value_set_int (value, priv->xpos); + break; + case PROP_PAD_YPOS: + g_value_set_int (value, priv->ypos); + break; + case PROP_PAD_WIDTH: + g_value_set_int (value, priv->width); + break; + case PROP_PAD_HEIGHT: + g_value_set_int (value, priv->height); + break; + case PROP_PAD_ALPHA: + g_value_set_double (value, priv->alpha); + break; + case PROP_PAD_OPERATOR: + g_value_set_enum (value, priv->op); + break; + case PROP_PAD_SIZING_POLICY: + g_value_set_enum (value, priv->sizing_policy); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d12_compositor_pad_update_conversion_info (GstVideoAggregatorPad * pad) +{ + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + priv->position_updated = TRUE; +} + +static void +gst_d3d12_compositor_pad_get_output_size (GstD3D12CompositorPad * pad, + gint out_par_n, gint out_par_d, gint * width, gint * height, + gint * x_offset, gint * y_offset) +{ + auto priv = pad->priv; + auto vagg_pad = GST_VIDEO_AGGREGATOR_PAD (pad); + gint pad_width, pad_height; + guint dar_n, dar_d; + + *x_offset = 0; + *y_offset = 0; + *width = 0; + *height = 0; + + /* FIXME: Anything better we can do here? */ + if (!vagg_pad->info.finfo + || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) { + GST_DEBUG_OBJECT (pad, "Have no caps yet"); + return; + } + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + + pad_width = priv->width <= 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : + priv->width; + pad_height = priv->height <= 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : + priv->height; + + if (pad_width == 0 || pad_height == 0) + return; + + if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { + GST_WARNING_OBJECT (pad, "Cannot calculate display aspect ratio"); + return; + } + + GST_TRACE_OBJECT (pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", + pad_width, pad_height, dar_n, dar_d, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); + + switch (priv->sizing_policy) { + case GST_D3D12_COMPOSITOR_SIZING_POLICY_NONE: + /* Pick either height or width, whichever is an integer multiple of the + * display aspect ratio. However, prefer preserving the height to account + * for interlaced video. */ + if (pad_height % dar_n == 0) { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } else if (pad_width % dar_d == 0) { + pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); + } else { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } + break; + case GST_D3D12_COMPOSITOR_SIZING_POLICY_KEEP_ASPECT_RATIO:{ + gint from_dar_n, from_dar_d, to_dar_n, to_dar_d, num, den; + + /* Calculate DAR again with actual video size */ + if (!gst_util_fraction_multiply (GST_VIDEO_INFO_WIDTH (&vagg_pad->info), + GST_VIDEO_INFO_HEIGHT (&vagg_pad->info), + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), &from_dar_n, + &from_dar_d)) { + from_dar_n = from_dar_d = -1; + } + + if (!gst_util_fraction_multiply (pad_width, pad_height, + out_par_n, out_par_d, &to_dar_n, &to_dar_d)) { + to_dar_n = to_dar_d = -1; + } + + if (from_dar_n != to_dar_n || from_dar_d != to_dar_d) { + /* Calculate new output resolution */ + if (from_dar_n != -1 && from_dar_d != -1 + && gst_util_fraction_multiply (from_dar_n, from_dar_d, + out_par_d, out_par_n, &num, &den)) { + GstVideoRectangle src_rect, dst_rect, rst_rect; + + src_rect.h = gst_util_uint64_scale_int (pad_width, den, num); + if (src_rect.h == 0) { + pad_width = 0; + pad_height = 0; + break; + } + + src_rect.x = src_rect.y = 0; + src_rect.w = pad_width; + + dst_rect.x = dst_rect.y = 0; + dst_rect.w = pad_width; + dst_rect.h = pad_height; + + /* Scale rect to be centered in destination rect */ + gst_video_center_rect (&src_rect, &dst_rect, &rst_rect, TRUE); + + GST_LOG_OBJECT (pad, + "Re-calculated size %dx%d -> %dx%d (x-offset %d, y-offset %d)", + pad_width, pad_height, rst_rect.w, rst_rect.h, rst_rect.x, + rst_rect.h); + + *x_offset = rst_rect.x; + *y_offset = rst_rect.y; + pad_width = rst_rect.w; + pad_height = rst_rect.h; + } else { + GST_WARNING_OBJECT (pad, "Failed to calculate output size"); + + *x_offset = 0; + *y_offset = 0; + pad_width = 0; + pad_height = 0; + } + } + break; + } + } + + *width = pad_width; + *height = pad_height; +} + +static GstVideoRectangle +clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width, + gint outer_height) +{ + gint x2 = x + w; + gint y2 = y + h; + GstVideoRectangle clamped; + + /* Clamp the x/y coordinates of this frame to the output boundaries to cover + * the case where (say, with negative xpos/ypos or w/h greater than the output + * size) the non-obscured portion of the frame could be outside the bounds of + * the video itself and hence not visible at all */ + clamped.x = CLAMP (x, 0, outer_width); + clamped.y = CLAMP (y, 0, outer_height); + clamped.w = CLAMP (x2, 0, outer_width) - clamped.x; + clamped.h = CLAMP (y2, 0, outer_height) - clamped.y; + + return clamped; +} + +static gboolean +gst_d3d12_compositor_pad_check_frame_obscured (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + gint width, height; + GstVideoInfo *info = &vagg->info; + /* The rectangle representing this frame, clamped to the video's boundaries. + * Due to the clamping, this is different from the frame width/height above. */ + GstVideoRectangle frame_rect; + gint x_offset, y_offset; + + /* There's three types of width/height here: + * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: + * The frame width/height (same as pad->info.height/width; + * see gst_video_frame_map()) + * 2. cpad->width/height: + * The optional pad property for scaling the frame (if zero, the video is + * left unscaled) + */ + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + if (priv->alpha == 0) + return TRUE; + + gst_d3d12_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info), + GST_VIDEO_INFO_PAR_D (info), &width, &height, &x_offset, &y_offset); + + frame_rect = clamp_rectangle (priv->xpos + x_offset, priv->ypos + y_offset, + width, height, GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info)); + + if (frame_rect.w == 0 || frame_rect.h == 0) { + GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height " + "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); + return TRUE; + } + + return FALSE; +} + +static gboolean +gst_d3d12_compositor_pad_setup_converter (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + auto self = GST_D3D12_COMPOSITOR (vagg); + gint width, height; + GstVideoInfo *info = &vagg->info; + GstVideoRectangle frame_rect; + gboolean output_has_alpha_comp = FALSE; + gint x_offset, y_offset; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + + if (GST_VIDEO_INFO_HAS_ALPHA (info) || + GST_VIDEO_INFO_FORMAT (info) == GST_VIDEO_FORMAT_BGRx || + GST_VIDEO_INFO_FORMAT (info) == GST_VIDEO_FORMAT_RGBx) { + output_has_alpha_comp = TRUE; + } + + if (priv->ctx) { + if (GST_VIDEO_INFO_FORMAT (&priv->ctx->info) != + GST_VIDEO_INFO_FORMAT (&pad->info)) { + priv->ctx = nullptr; + } + } + + if (!priv->ctx || priv->blend_desc_updated) { + switch (priv->op) { + case GST_D3D12_COMPOSITOR_OPERATOR_SOURCE: + priv->blend_desc.RenderTarget[0] = g_blend_source; + break; + case GST_D3D12_COMPOSITOR_OPERATOR_OVER: + if (output_has_alpha_comp) + priv->blend_desc.RenderTarget[0] = g_blend_over; + else + priv->blend_desc.RenderTarget[0] = g_blend_over_factor; + break; + default: + g_assert_not_reached (); + return FALSE; + } + } + + if (!priv->ctx || priv->alpha_updated) { + for (guint i = 0; i < 4; i++) + priv->blend_factor[i] = priv->alpha; + } + + if (!priv->ctx) { + auto ctx = std::make_unique < PadContext > (self->device); + ctx->info = pad->info; + + ctx->conv = gst_d3d12_converter_new (self->device, &pad->info, info, + &priv->blend_desc, priv->blend_factor, nullptr); + if (!ctx->conv) { + GST_ERROR_OBJECT (pad, "Couldn't create converter"); + return FALSE; + } + + priv->ctx = std::move (ctx); + } + + if (priv->ctx->fence_val == 0 || priv->alpha_updated) { + g_object_set (priv->ctx->conv, "alpha", priv->alpha, nullptr); + gst_d3d12_converter_update_blend_state (priv->ctx->conv, + &priv->blend_desc, priv->blend_factor); + } + + priv->alpha_updated = FALSE; + priv->blend_desc_updated = FALSE; + + if (priv->ctx->fence_val != 0 && !priv->position_updated) + return TRUE; + + gst_d3d12_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info), + GST_VIDEO_INFO_PAR_D (info), &width, &height, &x_offset, &y_offset); + + frame_rect = clamp_rectangle (priv->xpos + x_offset, priv->ypos + y_offset, + width, height, GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info)); + +#ifndef GST_DISABLE_GST_DEBUG + guint zorder = 0; + g_object_get (pad, "zorder", &zorder, nullptr); + + GST_LOG_OBJECT (pad, "Update position, pad-xpos %d, pad-ypos %d, " + "pad-zorder %d, pad-width %d, pad-height %d, in-resolution %dx%d, " + "out-resoution %dx%d, dst-{x,y,width,height} %d-%d-%d-%d", + priv->xpos, priv->ypos, zorder, priv->width, priv->height, + GST_VIDEO_INFO_WIDTH (&pad->info), GST_VIDEO_INFO_HEIGHT (&pad->info), + GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), + frame_rect.x, frame_rect.y, frame_rect.w, frame_rect.h); +#endif + + priv->position_updated = FALSE; + + g_object_set (priv->ctx->conv, "dest-x", frame_rect.x, + "dest-y", frame_rect.y, "dest-width", frame_rect.w, + "dest-height", frame_rect.h, nullptr); + + return TRUE; +} + +static gboolean +gst_d3d12_compositor_preprare_func (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) +{ + auto self = GST_D3D12_COMPOSITOR (vagg); + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + + GST_LOG_OBJECT (pad, "Building command list"); + + if (!self->priv->generated_output_buf) { + GST_ERROR_OBJECT (cpad, "Have no generated output buf"); + return FALSE; + } + + /* Skip this frame */ + if (gst_d3d12_compositor_pad_check_frame_obscured (pad, vagg)) + return TRUE; + + if (!gst_d3d12_compositor_pad_setup_converter (pad, vagg)) + return FALSE; + + gint x, y, w, h; + auto crop_meta = gst_buffer_get_video_crop_meta (buffer); + if (crop_meta) { + x = crop_meta->x; + y = crop_meta->y; + w = crop_meta->width; + h = crop_meta->height; + } else { + x = y = 0; + w = pad->info.width; + h = pad->info.height; + } + + g_assert (priv->ctx); + + g_object_set (priv->ctx->conv, "src-x", x, "src-y", y, "src-width", w, + "src-height", h, nullptr); + + GstD3D12CommandAllocator *gst_ca; + if (!gst_d3d12_command_allocator_pool_acquire (priv->ctx->ca_pool, &gst_ca)) { + GST_ERROR_OBJECT (cpad, "Couldn't acquire command allocator"); + return FALSE; + } + + GstD3D12FenceData *fence_data; + gst_d3d12_fence_data_pool_acquire (self->priv->fence_data_pool, &fence_data); + gst_d3d12_fence_data_add_notify (fence_data, gst_ca, + (GDestroyNotify) gst_d3d12_command_allocator_unref); + + ComPtr < ID3D12CommandAllocator > ca; + gst_d3d12_command_allocator_get_handle (gst_ca, &ca); + + auto hr = ca->Reset (); + if (!gst_d3d12_result (hr, priv->ctx->device)) { + GST_ERROR_OBJECT (cpad, "Couldn't reset command allocator"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + if (!priv->ctx->cl) { + auto device = gst_d3d12_device_get_device_handle (priv->ctx->device); + hr = device->CreateCommandList (0, D3D12_COMMAND_LIST_TYPE_DIRECT, + ca.Get (), nullptr, IID_PPV_ARGS (&priv->ctx->cl)); + if (!gst_d3d12_result (hr, priv->ctx->device)) { + GST_ERROR_OBJECT (cpad, "Couldn't create command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + } else { + hr = priv->ctx->cl->Reset (ca.Get (), nullptr); + if (!gst_d3d12_result (hr, priv->ctx->device)) { + GST_ERROR_OBJECT (self, "Couldn't reset command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + } + + if (!gst_d3d12_converter_convert_buffer (priv->ctx->conv, + buffer, self->priv->generated_output_buf, fence_data, + priv->ctx->cl.Get ())) { + GST_ERROR_OBJECT (self, "Couldn't build command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + hr = priv->ctx->cl->Close (); + if (!gst_d3d12_result (hr, priv->ctx->device)) { + GST_ERROR_OBJECT (self, "Couldn't close command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + prepared_frame->buffer = buffer; + + priv->ctx->fence_data = fence_data; + + GST_LOG_OBJECT (pad, "Command list prepared"); + + return TRUE; +} + +static void +gst_d3d12_compositor_pad_prepare_frame_start (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) +{ + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + + GST_LOG_OBJECT (cpad, "Prepare start"); + + priv->prepare_rst = std::async (std::launch::async, + gst_d3d12_compositor_preprare_func, pad, vagg, buffer, prepared_frame); +} + +static void +gst_d3d12_compositor_pad_prepare_frame_finish (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) +{ + /* Will wait on aggregate() function */ +} + +static void +gst_d3d12_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) +{ + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto priv = cpad->priv; + + if (priv->prepare_rst.valid ()) { + GST_WARNING_OBJECT (cpad, "Async task still pending"); + priv->prepare_rst.get (); + } + + memset (prepared_frame, 0, sizeof (GstVideoFrame)); + + if (priv->ctx && priv->ctx->fence_data) { + gst_d3d12_device_set_fence_notify (priv->ctx->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, priv->ctx->fence_val, + priv->ctx->fence_data, (GDestroyNotify) gst_d3d12_fence_data_unref); + priv->ctx->fence_data = nullptr; + } +} + +static GstStaticPadTemplate sink_template = + GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, GST_D3D12_ALL_FORMATS) "; " + GST_VIDEO_CAPS_MAKE (GST_D3D12_ALL_FORMATS))); + +/* formats we can output without conversion. + * Excludes 10/12 bits planar YUV (needs bitshift) and + * AYUV/AYUV64 (d3d12 runtime does not understand the ayuv order) */ +#define COMPOSITOR_SRC_FORMATS \ + "{ RGBA64_LE, RGB10A2_LE, BGRA, RGBA, BGRx, RGBx, VUYA, NV12, NV21, " \ + "P010_10LE, P012_LE, P016_LE, I420, YV12, Y42B, Y444, Y444_16LE, " \ + "GRAY8, GRAY16_LE }" + +static GstStaticPadTemplate src_template = + GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY, COMPOSITOR_SRC_FORMATS) "; " + GST_VIDEO_CAPS_MAKE (COMPOSITOR_SRC_FORMATS))); + +static void gst_d3d12_compositor_child_proxy_init (gpointer g_iface, + gpointer iface_data); +static void gst_d3d12_compositor_finalize (GObject * object); +static void gst_d3d12_compositor_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d12_compositor_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +static GstPad *gst_d3d12_compositor_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); +static void gst_d3d12_compositor_release_pad (GstElement * element, + GstPad * pad); +static void gst_d3d12_compositor_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_d3d12_compositor_start (GstAggregator * agg); +static gboolean gst_d3d12_compositor_stop (GstAggregator * agg); +static gboolean gst_d3d12_compositor_sink_query (GstAggregator * agg, + GstAggregatorPad * pad, GstQuery * query); +static gboolean gst_d3d12_compositor_src_query (GstAggregator * agg, + GstQuery * query); +static GstCaps *gst_d3d12_compositor_fixate_src_caps (GstAggregator * agg, + GstCaps * caps); +static gboolean gst_d3d12_compositor_negotiated_src_caps (GstAggregator * agg, + GstCaps * caps); +static gboolean +gst_d3d12_compositor_propose_allocation (GstAggregator * agg, + GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query); +static gboolean gst_d3d12_compositor_decide_allocation (GstAggregator * agg, + GstQuery * query); +static GstFlowReturn +gst_d3d12_compositor_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuf); +static GstFlowReturn +gst_d3d12_compositor_create_output_buffer (GstVideoAggregator * vagg, + GstBuffer ** outbuffer); + +#define gst_d3d12_compositor_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstD3D12Compositor, gst_d3d12_compositor, + GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_d3d12_compositor_child_proxy_init)); + +static void +gst_d3d12_compositor_class_init (GstD3D12CompositorClass * klass) +{ + auto object_class = G_OBJECT_CLASS (klass); + auto element_class = GST_ELEMENT_CLASS (klass); + auto agg_class = GST_AGGREGATOR_CLASS (klass); + auto vagg_class = GST_VIDEO_AGGREGATOR_CLASS (klass); + + object_class->finalize = gst_d3d12_compositor_finalize; + object_class->set_property = gst_d3d12_compositor_set_property; + object_class->get_property = gst_d3d12_compositor_get_property; + + g_object_class_install_property (object_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "Adapter index for creating device (-1 for default)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, PROP_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_TYPE_D3D12_COMPOSITOR_BACKGROUND, + DEFAULT_BACKGROUND, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property (object_class, + PROP_IGNORE_INACTIVE_PADS, g_param_spec_boolean ("ignore-inactive-pads", + "Ignore inactive pads", + "Avoid timing out waiting for inactive pads", FALSE, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + element_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_request_new_pad); + element_class->release_pad = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_release_pad); + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_set_context); + + agg_class->start = GST_DEBUG_FUNCPTR (gst_d3d12_compositor_start); + agg_class->stop = GST_DEBUG_FUNCPTR (gst_d3d12_compositor_stop); + agg_class->sink_query = GST_DEBUG_FUNCPTR (gst_d3d12_compositor_sink_query); + agg_class->src_query = GST_DEBUG_FUNCPTR (gst_d3d12_compositor_src_query); + agg_class->fixate_src_caps = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_fixate_src_caps); + agg_class->negotiated_src_caps = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_negotiated_src_caps); + agg_class->propose_allocation = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_propose_allocation); + agg_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_decide_allocation); + + vagg_class->aggregate_frames = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_aggregate_frames); + vagg_class->create_output_buffer = + GST_DEBUG_FUNCPTR (gst_d3d12_compositor_create_output_buffer); + + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_template, GST_TYPE_D3D12_COMPOSITOR_PAD); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_template, GST_TYPE_AGGREGATOR_PAD); + + gst_element_class_set_static_metadata (element_class, "Direct3D12 Compositor", + "Filter/Editor/Video/Compositor", "A Direct3D12 compositor", + "Seungha Yang "); + + gst_type_mark_as_plugin_api (GST_TYPE_D3D12_COMPOSITOR_BACKGROUND, + (GstPluginAPIFlags) 0); + gst_type_mark_as_plugin_api (GST_TYPE_D3D12_COMPOSITOR_PAD, + (GstPluginAPIFlags) 0); + + GST_DEBUG_CATEGORY_INIT (gst_d3d12_compositor_debug, + "d3d12compositor", 0, "d3d12compositor element"); +} + +static void +gst_d3d12_compositor_init (GstD3D12Compositor * self) +{ + self->priv = new GStD3D12CompositorPrivate (); +} + +static void +gst_d3d12_compositor_finalize (GObject * object) +{ + auto self = GST_D3D12_COMPOSITOR (object); + + delete self->priv; + + gst_clear_object (&self->device); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_d3d12_compositor_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + auto self = GST_D3D12_COMPOSITOR (object); + auto priv = self->priv; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + switch (prop_id) { + case PROP_ADAPTER: + priv->adapter = g_value_get_int (value); + break; + case PROP_BACKGROUND: + priv->background = + (GstD3D12CompositorBackground) g_value_get_enum (value); + break; + case PROP_IGNORE_INACTIVE_PADS: + gst_aggregator_set_ignore_inactive_pads (GST_AGGREGATOR (object), + g_value_get_boolean (value)); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d12_compositor_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + auto self = GST_D3D12_COMPOSITOR (object); + auto priv = self->priv; + + std::lock_guard < std::recursive_mutex > lk (priv->lock); + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_int (value, priv->adapter); + break; + case PROP_BACKGROUND: + g_value_set_enum (value, priv->background); + break; + case PROP_IGNORE_INACTIVE_PADS: + g_value_set_boolean (value, + gst_aggregator_get_ignore_inactive_pads (GST_AGGREGATOR (object))); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GObject * +gst_d3d12_compositor_child_proxy_get_child_by_index (GstChildProxy * proxy, + guint index) +{ + auto self = GST_D3D12_COMPOSITOR (proxy); + GObject *obj = nullptr; + + GST_OBJECT_LOCK (self); + obj = (GObject *) g_list_nth_data (GST_ELEMENT_CAST (self)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (self); + + return obj; +} + +static guint +gst_d3d12_compositor_child_proxy_get_children_count (GstChildProxy * proxy) +{ + auto self = GST_D3D12_COMPOSITOR (proxy); + guint count = 0; + + GST_OBJECT_LOCK (self); + count = GST_ELEMENT_CAST (self)->numsinkpads; + GST_OBJECT_UNLOCK (self); + GST_INFO_OBJECT (self, "Children Count: %d", count); + + return count; +} + +static void +gst_d3d12_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = (GstChildProxyInterface *) g_iface; + + iface->get_child_by_index = + gst_d3d12_compositor_child_proxy_get_child_by_index; + iface->get_children_count = + gst_d3d12_compositor_child_proxy_get_children_count; +} + +static GstPad * +gst_d3d12_compositor_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps) +{ + GstPad *pad; + + pad = GST_ELEMENT_CLASS (parent_class)->request_new_pad (element, + templ, name, caps); + + if (!pad) { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return nullptr; + } + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_DEBUG_OBJECT (element, "Created new pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + return pad; +} + +static void +gst_d3d12_compositor_release_pad (GstElement * element, GstPad * pad) +{ + auto self = GST_D3D12_COMPOSITOR (element); + + GST_DEBUG_OBJECT (self, "Releasing pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad); +} + +static void +gst_d3d12_compositor_set_context (GstElement * element, GstContext * context) +{ + auto self = GST_D3D12_COMPOSITOR (element); + auto priv = self->priv; + + { + std::lock_guard < std::recursive_mutex > lk (priv->lock); + gst_d3d12_handle_set_context (element, context, priv->adapter, + &self->device); + } + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +static gboolean +gst_d3d12_compositor_start (GstAggregator * agg) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + auto priv = self->priv; + + { + std::lock_guard < std::recursive_mutex > lk (priv->lock); + if (!gst_d3d12_ensure_element_data (GST_ELEMENT_CAST (self), + priv->adapter, &self->device)) { + GST_ERROR_OBJECT (self, "Failed to get D3D12 device"); + return FALSE; + } + } + + priv->scheduled = { }; + + return GST_AGGREGATOR_CLASS (parent_class)->start (agg); +} + +static gboolean +gst_d3d12_compositor_stop (GstAggregator * agg) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + auto priv = self->priv; + + { + std::lock_guard < std::recursive_mutex > lk (priv->lock); + priv->bg_render = nullptr; + gst_clear_object (&self->device); + } + + return GST_AGGREGATOR_CLASS (parent_class)->stop (agg); +} + +static GstCaps * +gst_d3d12_compositor_sink_getcaps (GstPad * pad, GstCaps * filter) +{ + GstCaps *sinkcaps; + GstCaps *template_caps; + GstCaps *filtered_caps; + GstCaps *returned_caps; + + template_caps = gst_pad_get_pad_template_caps (pad); + + sinkcaps = gst_pad_get_current_caps (pad); + if (sinkcaps == nullptr) { + sinkcaps = gst_caps_ref (template_caps); + } else { + sinkcaps = gst_caps_merge (sinkcaps, gst_caps_ref (template_caps)); + } + + if (filter) { + filtered_caps = gst_caps_intersect (sinkcaps, filter); + gst_caps_unref (sinkcaps); + } else { + filtered_caps = sinkcaps; /* pass ownership */ + } + + returned_caps = gst_caps_intersect (filtered_caps, template_caps); + + gst_caps_unref (template_caps); + gst_caps_unref (filtered_caps); + + GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps); + + return returned_caps; +} + +static gboolean +gst_d3d12_compositor_sink_acceptcaps (GstPad * pad, GstCaps * caps) +{ + gboolean ret; + GstCaps *template_caps; + + GST_DEBUG_OBJECT (pad, "try accept caps of %" GST_PTR_FORMAT, caps); + + template_caps = gst_pad_get_pad_template_caps (pad); + template_caps = gst_caps_make_writable (template_caps); + + ret = gst_caps_can_intersect (caps, template_caps); + GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, + (ret ? "" : "not "), caps); + gst_caps_unref (template_caps); + + return ret; +} + +static gboolean +gst_d3d12_compositor_sink_query (GstAggregator * agg, + GstAggregatorPad * pad, GstQuery * query) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + auto priv = self->priv; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + std::lock_guard < std::recursive_mutex > lk (priv->lock); + if (gst_d3d12_handle_context_query (GST_ELEMENT (agg), query, + self->device)) { + return TRUE; + } + break; + } + case GST_QUERY_CAPS:{ + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_d3d12_compositor_sink_getcaps (GST_PAD (pad), filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + return TRUE; + } + case GST_QUERY_ACCEPT_CAPS:{ + GstCaps *caps; + gboolean ret; + + gst_query_parse_accept_caps (query, &caps); + ret = gst_d3d12_compositor_sink_acceptcaps (GST_PAD (pad), caps); + gst_query_set_accept_caps_result (query, ret); + return TRUE; + } + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, pad, query); +} + +static gboolean +gst_d3d12_compositor_src_query (GstAggregator * agg, GstQuery * query) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d12_handle_context_query (GST_ELEMENT (agg), query, + self->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query); +} + +static GstCaps * +gst_d3d12_compositor_fixate_src_caps (GstAggregator * agg, GstCaps * caps) +{ + auto vagg = GST_VIDEO_AGGREGATOR (agg); + GList *l; + gint best_width = -1, best_height = -1; + gint best_fps_n = -1, best_fps_d = -1; + gint par_n, par_d; + gdouble best_fps = 0.; + GstCaps *ret = nullptr; + GstStructure *s; + + ret = gst_caps_make_writable (caps); + + /* we need this to calculate how large to make the output frame */ + s = gst_caps_get_structure (ret, 0); + if (gst_structure_has_field (s, "pixel-aspect-ratio")) { + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); + } else { + par_n = par_d = 1; + } + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + auto vaggpad = GST_VIDEO_AGGREGATOR_PAD (l->data); + auto cpad = GST_D3D12_COMPOSITOR_PAD (vaggpad); + auto priv = cpad->priv; + gint this_width, this_height; + gint width, height; + gint fps_n, fps_d; + gdouble cur_fps; + gint x_offset; + gint y_offset; + + fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info); + gst_d3d12_compositor_pad_get_output_size (cpad, + par_n, par_d, &width, &height, &x_offset, &y_offset); + + if (width == 0 || height == 0) + continue; + + /* {x,y}_offset represent padding size of each top and left area. + * To calculate total resolution, count bottom and right padding area + * as well here */ + this_width = width + MAX (priv->xpos + 2 * x_offset, 0); + this_height = height + MAX (priv->ypos + 2 * y_offset, 0); + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + } + GST_OBJECT_UNLOCK (vagg); + + if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; + } + + if (best_width <= 0 || best_height <= 0) { + best_width = 320; + best_height = 240; + } + + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + ret = gst_caps_fixate (ret); + + GST_LOG_OBJECT (agg, "Fixated caps %" GST_PTR_FORMAT, ret); + + return ret; +} + +static void +convert_info_gray_to_yuv (const GstVideoInfo * gray, GstVideoInfo * yuv) +{ + GstVideoInfo tmp; + + if (GST_VIDEO_INFO_IS_YUV (gray)) { + *yuv = *gray; + return; + } + + if (gray->finfo->depth[0] == 8) { + gst_video_info_set_format (&tmp, + GST_VIDEO_FORMAT_Y444, gray->width, gray->height); + } else { + gst_video_info_set_format (&tmp, + GST_VIDEO_FORMAT_Y444_16LE, gray->width, gray->height); + } + + tmp.colorimetry.range = gray->colorimetry.range; + if (tmp.colorimetry.range == GST_VIDEO_COLOR_RANGE_UNKNOWN) + tmp.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255; + + tmp.colorimetry.primaries = gray->colorimetry.primaries; + if (tmp.colorimetry.primaries == GST_VIDEO_COLOR_PRIMARIES_UNKNOWN) + tmp.colorimetry.primaries = GST_VIDEO_COLOR_PRIMARIES_BT709; + + tmp.colorimetry.transfer = gray->colorimetry.transfer; + if (tmp.colorimetry.transfer == GST_VIDEO_TRANSFER_UNKNOWN) + tmp.colorimetry.transfer = GST_VIDEO_TRANSFER_BT709; + + tmp.colorimetry.matrix = gray->colorimetry.matrix; + if (tmp.colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN) + tmp.colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT709; + + *yuv = tmp; +} + +static void +gst_d3d12_compositor_calculate_background_color (GstD3D12Compositor * self, + const GstVideoInfo * info) +{ + auto priv = self->priv; + GstD3D12ColorMatrix clear_color_matrix; + gdouble rgb[3]; + gdouble converted[3]; + GstVideoFormat format = GST_VIDEO_INFO_FORMAT (info); + + if (GST_VIDEO_INFO_IS_RGB (info)) { + GstVideoInfo rgb_info = *info; + rgb_info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255; + + gst_d3d12_color_range_adjust_matrix_unorm (&rgb_info, info, + &clear_color_matrix); + } else { + GstVideoInfo rgb_info; + GstVideoInfo yuv_info; + + gst_video_info_set_format (&rgb_info, GST_VIDEO_FORMAT_RGBA64_LE, + info->width, info->height); + convert_info_gray_to_yuv (info, &yuv_info); + + if (yuv_info.colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN || + yuv_info.colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_RGB) { + GST_WARNING_OBJECT (self, "Invalid matrix is detected"); + yuv_info.colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT709; + } + + gst_d3d12_rgb_to_yuv_matrix_unorm (&rgb_info, + &yuv_info, &clear_color_matrix); + } + + /* Calculate black and white color values */ + for (guint i = 0; i < 2; i++) { + ClearColor *clear_color = &priv->clear_color[i]; + rgb[0] = rgb[1] = rgb[2] = (gdouble) i; + + for (guint j = 0; j < 3; j++) { + converted[j] = 0; + for (guint k = 0; k < 3; k++) { + converted[j] += clear_color_matrix.matrix[j][k] * rgb[k]; + } + converted[j] += clear_color_matrix.offset[j]; + converted[j] = CLAMP (converted[j], + clear_color_matrix.min[j], clear_color_matrix.max[j]); + } + + GST_DEBUG_OBJECT (self, "Calculated background color RGB: %f, %f, %f", + converted[0], converted[1], converted[2]); + + if (GST_VIDEO_INFO_IS_RGB (info) || GST_VIDEO_INFO_IS_GRAY (info)) { + for (guint j = 0; j < 3; j++) + clear_color->color[0][j] = converted[j]; + clear_color->color[0][3] = 1.0; + } else { + switch (format) { + case GST_VIDEO_FORMAT_VUYA: + clear_color->color[0][0] = converted[2]; + clear_color->color[0][1] = converted[1]; + clear_color->color[0][2] = converted[0]; + clear_color->color[0][3] = 1.0; + break; + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_NV21: + case GST_VIDEO_FORMAT_P010_10LE: + case GST_VIDEO_FORMAT_P012_LE: + case GST_VIDEO_FORMAT_P016_LE: + clear_color->color[0][0] = converted[0]; + clear_color->color[0][1] = 0; + clear_color->color[0][2] = 0; + clear_color->color[0][3] = 1.0; + if (format == GST_VIDEO_FORMAT_NV21) { + clear_color->color[1][0] = converted[2]; + clear_color->color[1][1] = converted[1]; + } else { + clear_color->color[1][0] = converted[1]; + clear_color->color[1][1] = converted[2]; + } + clear_color->color[1][2] = 0; + clear_color->color[1][3] = 1.0; + break; + case GST_VIDEO_FORMAT_I420: + case GST_VIDEO_FORMAT_YV12: + case GST_VIDEO_FORMAT_I420_10LE: + case GST_VIDEO_FORMAT_I420_12LE: + case GST_VIDEO_FORMAT_Y42B: + case GST_VIDEO_FORMAT_I422_10LE: + case GST_VIDEO_FORMAT_I422_12LE: + case GST_VIDEO_FORMAT_Y444: + case GST_VIDEO_FORMAT_Y444_10LE: + case GST_VIDEO_FORMAT_Y444_12LE: + case GST_VIDEO_FORMAT_Y444_16LE: + clear_color->color[0][0] = converted[0]; + clear_color->color[0][1] = 0; + clear_color->color[0][2] = 0; + clear_color->color[0][3] = 1.0; + if (format == GST_VIDEO_FORMAT_YV12) { + clear_color->color[1][0] = converted[2]; + clear_color->color[2][0] = converted[1]; + } else { + clear_color->color[1][0] = converted[1]; + clear_color->color[2][0] = converted[2]; + } + clear_color->color[1][1] = 0; + clear_color->color[1][2] = 0; + clear_color->color[1][3] = 1.0; + clear_color->color[2][1] = 0; + clear_color->color[2][2] = 0; + clear_color->color[2][3] = 1.0; + break; + default: + g_assert_not_reached (); + break; + } + } + } +} + +static gboolean +gst_d3d12_compositor_clear_pad_context (GstD3D12Compositor * self, + GstD3D12CompositorPad * cpad, gpointer user_data) +{ + auto priv = cpad->priv; + priv->ctx = nullptr; + + return TRUE; +} + +static gboolean +gst_d3d12_compositor_negotiated_src_caps (GstAggregator * agg, GstCaps * caps) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + auto priv = self->priv; + GstVideoInfo info; + + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (self, "Failed to convert caps to info"); + return FALSE; + } + + auto features = gst_caps_get_features (caps, 0); + if (features + && gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) { + GST_DEBUG_OBJECT (self, "Negotiated with D3D12 memory caps"); + priv->downstream_supports_d3d12 = TRUE; + } else { + GST_DEBUG_OBJECT (self, "Negotiated with system memory caps"); + priv->downstream_supports_d3d12 = FALSE; + } + + if (GST_VIDEO_INFO_FORMAT (&info) != + GST_VIDEO_INFO_FORMAT (&priv->negotiated_info)) { + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (self), + (GstElementForeachPadFunc) gst_d3d12_compositor_clear_pad_context, + nullptr); + priv->bg_render = nullptr; + } + gst_clear_buffer (&priv->fallback_buf); + + priv->negotiated_info = info; + gst_d3d12_compositor_calculate_background_color (self, &info); + + if (!priv->bg_render) { + auto bg_render = std::make_unique < BackgroundRender > (self->device, info); + if (!bg_render->is_valid) { + GST_ERROR_OBJECT (self, "Couldn't configure background render object"); + return FALSE; + } + + priv->bg_render = std::move (bg_render); + } else { + priv->bg_render->viewport.Width = info.width; + priv->bg_render->viewport.Height = info.height; + priv->bg_render->scissor_rect.right = info.width; + priv->bg_render->scissor_rect.bottom = info.height; + } + + if (!priv->downstream_supports_d3d12) { + auto pool = gst_d3d12_buffer_pool_new (self->device); + auto config = gst_buffer_pool_get_config (pool); + auto params = gst_d3d12_allocation_params_new (self->device, &info, + GST_D3D12_ALLOCATION_FLAG_DEFAULT, + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS); + gst_buffer_pool_config_set_d3d12_allocation_params (config, params); + gst_d3d12_allocation_params_free (params); + gst_buffer_pool_config_set_params (config, caps, info.size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Couldn't set pool config"); + gst_object_unref (pool); + return FALSE; + } + + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ERROR_OBJECT (self, "Failed to set active"); + gst_object_unref (pool); + return FALSE; + } + + gst_buffer_pool_acquire_buffer (pool, &priv->fallback_buf, nullptr); + gst_buffer_pool_set_active (pool, FALSE); + gst_object_unref (pool); + + if (!priv->fallback_buf) { + GST_ERROR_OBJECT (self, "Couldn't acquire fallback buf"); + return FALSE; + } + } + + return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); +} + +static gboolean +gst_d3d12_compositor_propose_allocation (GstAggregator * agg, + GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + GstVideoInfo info; + GstBufferPool *pool; + GstCaps *caps; + gboolean is_d3d12 = FALSE; + guint size; + + gst_query_parse_allocation (query, &caps, nullptr); + + if (!caps) + return FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; + + auto features = gst_caps_get_features (caps, 0); + if (gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY)) { + GST_DEBUG_OBJECT (pad, "Upstream support d3d12 memory"); + is_d3d12 = TRUE; + } + + if (gst_query_get_n_allocation_pools (query) == 0) { + if (is_d3d12) + pool = gst_d3d12_buffer_pool_new (self->device); + else + pool = gst_video_buffer_pool_new (); + + if (!pool) { + GST_ERROR_OBJECT (self, "Failed to create buffer pool"); + return FALSE; + } + + auto config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + + size = GST_VIDEO_INFO_SIZE (&info); + if (is_d3d12) { + auto params = gst_d3d12_allocation_params_new (self->device, + &info, GST_D3D12_ALLOCATION_FLAG_DEFAULT, + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS); + + gst_buffer_pool_config_set_d3d12_allocation_params (config, params); + gst_d3d12_allocation_params_free (params); + } else { + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + } + + gst_buffer_pool_config_set_params (config, caps, (guint) size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (pool, "Couldn't set config"); + gst_object_unref (pool); + + return FALSE; + } + + /* d3d12 buffer pool will update buffer size based on allocated texture, + * get size from config again */ + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, + nullptr, &size, nullptr, nullptr); + gst_structure_free (config); + + gst_query_add_allocation_pool (query, pool, size, 0, 0); + gst_object_unref (pool); + } + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, nullptr); + if (is_d3d12) { + gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, + nullptr); + } + + return TRUE; +} + +static gboolean +gst_d3d12_compositor_decide_allocation (GstAggregator * agg, GstQuery * query) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + auto priv = self->priv; + GstCaps *caps; + GstBufferPool *pool = nullptr; + guint n, size, min, max; + GstVideoInfo info; + gboolean use_d3d12_pool; + + gst_query_parse_allocation (query, &caps, nullptr); + + if (!caps) { + GST_DEBUG_OBJECT (self, "No output caps"); + return FALSE; + } + + if (!gst_video_info_from_caps (&info, caps)) { + GST_ERROR_OBJECT (self, "Invalid caps"); + return FALSE; + } + + use_d3d12_pool = priv->downstream_supports_d3d12; + + n = gst_query_get_n_allocation_pools (query); + if (n > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + /* create our own pool */ + if (pool && use_d3d12_pool) { + if (!GST_IS_D3D12_BUFFER_POOL (pool)) { + GST_DEBUG_OBJECT (self, + "Downstream pool is not d3d12, will create new one"); + gst_clear_object (&pool); + } else { + GstD3D12BufferPool *dpool = GST_D3D12_BUFFER_POOL (pool); + if (dpool->device != self->device) { + GST_DEBUG_OBJECT (self, "Different device, will create new one"); + gst_clear_object (&pool); + } + } + } + + size = (guint) info.size; + + if (!pool) { + if (use_d3d12_pool) + pool = gst_d3d12_buffer_pool_new (self->device); + else + pool = gst_video_buffer_pool_new (); + + min = 0; + max = 0; + } + + auto config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + if (use_d3d12_pool) { + auto params = gst_buffer_pool_config_get_d3d12_allocation_params (config); + if (!params) { + params = gst_d3d12_allocation_params_new (self->device, &info, + GST_D3D12_ALLOCATION_FLAG_DEFAULT, + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS); + } else { + gst_d3d12_allocation_params_set_resource_flags (params, + D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET | + D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS); + } + + gst_buffer_pool_config_set_d3d12_allocation_params (config, params); + gst_d3d12_allocation_params_free (params); + } + + gst_buffer_pool_set_config (pool, config); + + /* d3d12 buffer pool will update buffer size based on allocated texture, + * get size from config again */ + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, nullptr, &size, nullptr, nullptr); + gst_structure_free (config); + + if (n > 0) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + gst_object_unref (pool); + + return TRUE; +} + +static void +resource_free_func (ID3D12Resource * resource) +{ + if (resource) + resource->Release (); +} + +static gboolean +gst_d3d12_compositor_draw_background (GstD3D12Compositor * self) +{ + auto priv = self->priv; + ClearColor *color = &priv->clear_color[0]; + auto bg_render = priv->bg_render.get (); + auto & rtv_handles = priv->rtv_handles; + + rtv_handles.clear (); + for (guint i = 0; i < gst_buffer_n_memory (priv->generated_output_buf); i++) { + auto mem = (GstD3D12Memory *) + gst_buffer_peek_memory (priv->generated_output_buf, i); + auto num_planes = gst_d3d12_memory_get_plane_count (mem); + ComPtr < ID3D12DescriptorHeap > rtv_heap; + + if (!gst_d3d12_memory_get_render_target_view_heap (mem, &rtv_heap)) { + GST_ERROR_OBJECT (self, "Couldn't get rtv heap"); + return FALSE; + } + + auto cpu_handle = + CD3DX12_CPU_DESCRIPTOR_HANDLE + (rtv_heap->GetCPUDescriptorHandleForHeapStart ()); + + for (guint plane = 0; plane < num_planes; plane++) { + rtv_handles.push_back (cpu_handle); + cpu_handle.Offset (bg_render->rtv_inc_size); + } + } + + GstD3D12CommandAllocator *gst_ca; + if (!gst_d3d12_command_allocator_pool_acquire (bg_render->ca_pool, &gst_ca)) { + GST_ERROR_OBJECT (self, "Couldn't acquire command allocator"); + return FALSE; + } + + GstD3D12FenceData *fence_data; + gst_d3d12_fence_data_pool_acquire (priv->fence_data_pool, &fence_data); + gst_d3d12_fence_data_add_notify (fence_data, gst_ca, + (GDestroyNotify) gst_d3d12_command_allocator_unref); + + ComPtr < ID3D12CommandAllocator > ca; + gst_d3d12_command_allocator_get_handle (gst_ca, &ca); + + auto hr = ca->Reset (); + if (!gst_d3d12_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't reset command allocator"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + if (!bg_render->cl) { + auto device = gst_d3d12_device_get_device_handle (self->device); + hr = device->CreateCommandList (0, D3D12_COMMAND_LIST_TYPE_DIRECT, + ca.Get (), bg_render->pso.Get (), IID_PPV_ARGS (&bg_render->cl)); + if (!gst_d3d12_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't create command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + } else { + hr = bg_render->cl->Reset (ca.Get (), bg_render->pso.Get ()); + if (!gst_d3d12_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't reset command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + } + + auto cl = bg_render->cl; + if (bg_render->vertex_index_upload) { + cl->CopyResource (bg_render->vertex_index_buf.Get (), + bg_render->vertex_index_upload.Get ()); + D3D12_RESOURCE_BARRIER barrier = + CD3DX12_RESOURCE_BARRIER::Transition (bg_render-> + vertex_index_buf.Get (), + D3D12_RESOURCE_STATE_COPY_DEST, + D3D12_RESOURCE_STATE_VERTEX_AND_CONSTANT_BUFFER | + D3D12_RESOURCE_STATE_INDEX_BUFFER); + cl->ResourceBarrier (1, &barrier); + } + + if (priv->background == GST_D3D12_COMPOSITOR_BACKGROUND_CHECKER) { + cl->SetGraphicsRootSignature (bg_render->rs.Get ()); + cl->IASetPrimitiveTopology (D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST); + cl->IASetIndexBuffer (&bg_render->ibv); + cl->IASetVertexBuffers (0, 1, &bg_render->vbv); + cl->RSSetViewports (1, &bg_render->viewport); + cl->RSSetScissorRects (1, &bg_render->scissor_rect); + cl->OMSetRenderTargets (1, rtv_handles.data (), FALSE, nullptr); + cl->DrawIndexedInstanced (6, 1, 0, 0, 0); + + /* clear U and V components if needed */ + for (size_t i = 1; i < rtv_handles.size (); i++) + cl->ClearRenderTargetView (rtv_handles[i], color->color[i], 0, nullptr); + } else { + switch (priv->background) { + case GST_D3D12_COMPOSITOR_BACKGROUND_BLACK: + color = &priv->clear_color[0]; + break; + case GST_D3D12_COMPOSITOR_BACKGROUND_WHITE: + color = &priv->clear_color[1]; + break; + case GST_D3D12_COMPOSITOR_BACKGROUND_TRANSPARENT: + color = &priv->clear_color[2]; + break; + default: + g_assert_not_reached (); + return FALSE; + } + + for (size_t i = 0; i < priv->rtv_handles.size (); i++) { + cl->ClearRenderTargetView (priv->rtv_handles[i], color->color[i], + 0, nullptr); + } + } + + hr = cl->Close (); + if (!gst_d3d12_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't close command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + ID3D12CommandList *cmd_list[] = { cl.Get () }; + + if (!gst_d3d12_device_execute_command_lists (self->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, 1, cmd_list, + &priv->bg_render->fence_val)) { + GST_ERROR_OBJECT (self, "Couldn't execute command list"); + gst_d3d12_fence_data_unref (fence_data); + return FALSE; + } + + gst_d3d12_buffer_after_write (priv->generated_output_buf, + bg_render->fence_val); + + if (bg_render->vertex_index_upload) { + gst_d3d12_fence_data_add_notify (fence_data, + bg_render->vertex_index_upload.Detach (), + (GDestroyNotify) resource_free_func); + } + + gst_d3d12_device_set_fence_notify (self->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, priv->bg_render->fence_val, + fence_data, (GDestroyNotify) gst_d3d12_fence_data_unref); + + return TRUE; +} + +static GstFlowReturn +gst_d3d12_compositor_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuf) +{ + auto self = GST_D3D12_COMPOSITOR (vagg); + auto priv = self->priv; + GList *iter; + GstFlowReturn ret = GST_FLOW_OK; + + GST_LOG_OBJECT (self, "aggregate"); + + if (!priv->generated_output_buf) { + GST_ERROR_OBJECT (self, "No generated output buffer"); + return GST_FLOW_ERROR; + } + + auto completed = gst_d3d12_device_get_completed_value (self->device, + D3D12_COMMAND_LIST_TYPE_DIRECT); + while (!priv->scheduled.empty ()) { + if (priv->scheduled.front () > completed) + break; + + priv->scheduled.pop (); + } + + /* avoid too large buffering */ + if (priv->scheduled.size () > 2) { + auto fence_to_wait = priv->scheduled.front (); + priv->scheduled.pop (); + GST_LOG_OBJECT (self, "Waiting for previous command, %" G_GUINT64_FORMAT, + fence_to_wait); + gst_d3d12_device_fence_wait (self->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, fence_to_wait, + priv->bg_render->event_handle); + } + + if (!gst_d3d12_compositor_draw_background (self)) { + GST_ERROR_OBJECT (self, "Couldn't draw background"); + return GST_FLOW_ERROR; + } + + guint64 fence_val = priv->bg_render->fence_val; + GST_OBJECT_LOCK (self); + for (iter = GST_ELEMENT (vagg)->sinkpads; iter; iter = g_list_next (iter)) { + auto pad = GST_VIDEO_AGGREGATOR_PAD (iter->data); + auto cpad = GST_D3D12_COMPOSITOR_PAD (pad); + auto pad_priv = cpad->priv; + + /* Might be a case where pad was added between prepare_frame() and + * aggregate_frames() */ + if (!pad_priv->prepare_rst.valid ()) { + GST_DEBUG_OBJECT (pad, "Ignoring non-prepared pad"); + continue; + } + + GST_LOG_OBJECT (cpad, "Waiting for command list building thread"); + auto prepare_ret = pad_priv->prepare_rst.get (); + if (!prepare_ret) { + GST_ERROR_OBJECT (pad, "Couldn't build command list"); + ret = GST_FLOW_ERROR; + break; + } + + if (!gst_video_aggregator_pad_get_prepared_frame (pad)) + continue; + + GST_LOG_OBJECT (cpad, "Command list prepared"); + + ID3D12CommandList *cmd_list[] = { pad_priv->ctx->cl.Get () }; + if (!gst_d3d12_device_execute_command_lists (self->device, + D3D12_COMMAND_LIST_TYPE_DIRECT, 1, cmd_list, + &pad_priv->ctx->fence_val)) { + GST_ERROR_OBJECT (self, "Couldn't execute command list"); + ret = GST_FLOW_ERROR; + break; + } + + fence_val = pad_priv->ctx->fence_val; + gst_d3d12_buffer_after_write (priv->generated_output_buf, fence_val); + } + GST_OBJECT_UNLOCK (self); + + if (ret != GST_FLOW_OK) + return ret; + + priv->scheduled.push (fence_val); + if (priv->generated_output_buf != outbuf) { + GstVideoFrame out_frame, in_frame; + if (!gst_video_frame_map (&in_frame, &vagg->info, + priv->generated_output_buf, GST_MAP_READ)) { + GST_ERROR_OBJECT (self, "Couldn't map generated buffer"); + return GST_FLOW_ERROR; + } + + if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Couldn't map output buffer"); + gst_video_frame_unmap (&in_frame); + return GST_FLOW_ERROR; + } + + auto copy_ret = gst_video_frame_copy (&out_frame, &in_frame); + gst_video_frame_unmap (&out_frame); + gst_video_frame_unmap (&in_frame); + if (!copy_ret) { + GST_ERROR_OBJECT (self, "Couldn't copy frame"); + return GST_FLOW_ERROR; + } + } + + return GST_FLOW_OK; +} + +struct DeviceCheckData +{ + /* without holding ref */ + GstD3D12Device *other_device = nullptr; + gboolean have_same_device = FALSE; +}; + +static gboolean +gst_d3d12_compositor_check_device_update (GstElement * agg, + GstVideoAggregatorPad * vpad, DeviceCheckData * data) +{ + auto self = GST_D3D12_COMPOSITOR (agg); + GstBuffer *buf; + GstMemory *mem; + GstD3D12Memory *dmem; + + buf = gst_video_aggregator_pad_get_current_buffer (vpad); + if (!buf) + return TRUE; + + /* Ignore gap buffer */ + if (GST_BUFFER_FLAG_IS_SET (buf, GST_BUFFER_FLAG_GAP) || + gst_buffer_get_size (buf) == 0) { + return TRUE; + } + + mem = gst_buffer_peek_memory (buf, 0); + if (!gst_is_d3d12_memory (mem)) + return TRUE; + + dmem = GST_D3D12_MEMORY_CAST (mem); + + /* We can use existing device */ + if (dmem->device == self->device) { + data->have_same_device = TRUE; + return FALSE; + } + + data->other_device = dmem->device; + + /* Keep iterate since there might be one buffer which holds the same device + * as ours */ + return TRUE; +} + +static GstFlowReturn +gst_d3d12_compositor_create_output_buffer (GstVideoAggregator * vagg, + GstBuffer ** outbuffer) +{ + auto self = GST_D3D12_COMPOSITOR (vagg); + auto priv = self->priv; + DeviceCheckData data; + + /* Check whether there is at least one sinkpad which holds d3d12 buffer + * with compatible device, and if not, update our device */ + data.other_device = nullptr; + data.have_same_device = FALSE; + + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), + (GstElementForeachPadFunc) gst_d3d12_compositor_check_device_update, + &data); + + priv->generated_output_buf = nullptr; + if (data.have_same_device || !data.other_device) { + GstBuffer *buf = nullptr; + auto ret = + GST_VIDEO_AGGREGATOR_CLASS (parent_class)->create_output_buffer (vagg, + &buf); + if (ret != GST_FLOW_OK) + return ret; + + if (priv->downstream_supports_d3d12) + priv->generated_output_buf = buf; + else + priv->generated_output_buf = priv->fallback_buf; + + *outbuffer = buf; + return GST_FLOW_OK; + } + + /* Clear all device dependent resources */ + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), + (GstElementForeachPadFunc) gst_d3d12_compositor_clear_pad_context, + nullptr); + + gst_clear_buffer (&priv->fallback_buf); + priv->bg_render = nullptr; + priv->scheduled = { }; + + GST_INFO_OBJECT (self, "Updating device %" GST_PTR_FORMAT " -> %" + GST_PTR_FORMAT, self->device, data.other_device); + { + std::lock_guard < std::recursive_mutex > lk (priv->lock); + gst_object_unref (self->device); + self->device = (GstD3D12Device *) gst_object_ref (data.other_device); + } + + /* We cannot call gst_aggregator_negotiate() here, since GstVideoAggregator + * is holding GST_VIDEO_AGGREGATOR_LOCK() already. + * Mark reconfigure and do reconfigure later */ + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + + return GST_AGGREGATOR_FLOW_NEED_DATA; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.h b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.h new file mode 100644 index 0000000000..db23bc737c --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/gstd3d12compositor.h @@ -0,0 +1,38 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#pragma once + +#include +#include +#include +#include "gstd3d12.h" + +G_BEGIN_DECLS + +#define GST_TYPE_D3D12_COMPOSITOR_PAD (gst_d3d12_compositor_pad_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D12CompositorPad, gst_d3d12_compositor_pad, + GST, D3D12_COMPOSITOR_PAD, GstVideoAggregatorPad) + +#define GST_TYPE_D3D12_COMPOSITOR (gst_d3d12_compositor_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D12Compositor, gst_d3d12_compositor, + GST, D3D12_COMPOSITOR, GstVideoAggregator) + +G_END_DECLS + diff --git a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_luma.hlsl b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_luma.hlsl new file mode 100644 index 0000000000..990c71891d --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_luma.hlsl @@ -0,0 +1,49 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +static const float blocksize = 8.0; +static const float4 high = float4 (0.667, 0.0, 0.0, 1.0); +static const float4 low = float4 (0.333, 0.0, 0.0, 1.0); + +struct PS_INPUT +{ + float4 Position : SV_POSITION; +}; + +struct PS_OUTPUT +{ + float4 Plane : SV_TARGET; +}; + +PS_OUTPUT PSMain_checker_luma (PS_INPUT input) +{ + PS_OUTPUT output; + if ((input.Position.x % (blocksize * 2.0)) >= blocksize) { + if ((input.Position.y % (blocksize * 2.0)) >= blocksize) + output.Plane = low; + else + output.Plane = high; + } else { + if ((input.Position.y % (blocksize * 2.0)) < blocksize) + output.Plane = low; + else + output.Plane = high; + } + return output; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_rgb.hlsl b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_rgb.hlsl new file mode 100644 index 0000000000..32d5eb960b --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_rgb.hlsl @@ -0,0 +1,49 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +static const float blocksize = 8.0; +static const float4 high = float4 (0.667, 0.667, 0.667, 1.0); +static const float4 low = float4 (0.333, 0.333, 0.333, 1.0); + +struct PS_INPUT +{ + float4 Position : SV_POSITION; +}; + +struct PS_OUTPUT +{ + float4 Plane : SV_TARGET; +}; + +PS_OUTPUT PSMain_checker_rgb (PS_INPUT input) +{ + PS_OUTPUT output; + if ((input.Position.x % (blocksize * 2.0)) >= blocksize) { + if ((input.Position.y % (blocksize * 2.0)) >= blocksize) + output.Plane = low; + else + output.Plane = high; + } else { + if ((input.Position.y % (blocksize * 2.0)) < blocksize) + output.Plane = low; + else + output.Plane = high; + } + return output; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_vuya.hlsl b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_vuya.hlsl new file mode 100644 index 0000000000..22f139fe61 --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/PSMain_checker_vuya.hlsl @@ -0,0 +1,49 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +static const float blocksize = 8.0; +static const float4 high = float4 (0.5, 0.5, 0.667, 1.0); +static const float4 low = float4 (0.5, 0.5, 0.333, 1.0); + +struct PS_INPUT +{ + float4 Position : SV_POSITION; +}; + +struct PS_OUTPUT +{ + float4 Plane : SV_TARGET; +}; + +PS_OUTPUT PSMain_checker_vuya (PS_INPUT input) +{ + PS_OUTPUT output; + if ((input.Position.x % (blocksize * 2.0)) >= blocksize) { + if ((input.Position.y % (blocksize * 2.0)) >= blocksize) + output.Plane = low; + else + output.Plane = high; + } else { + if ((input.Position.y % (blocksize * 2.0)) < blocksize) + output.Plane = low; + else + output.Plane = high; + } + return output; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/VSMain_pos.hlsl b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/VSMain_pos.hlsl new file mode 100644 index 0000000000..df069fd5de --- /dev/null +++ b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/VSMain_pos.hlsl @@ -0,0 +1,33 @@ +/* GStreamer + * Copyright (C) 2023 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +struct VS_INPUT +{ + float4 Position : POSITION; +}; + +struct VS_OUTPUT +{ + float4 Position : SV_POSITION; +}; + +VS_OUTPUT VSMain_pos (VS_INPUT input) +{ + return input; +} diff --git a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/meson.build b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/meson.build index 66b6a8c3c2..6b52dc408a 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/hlsl/meson.build +++ b/subprojects/gst-plugins-bad/sys/d3d12/hlsl/meson.build @@ -148,6 +148,10 @@ hlsl_sources = [ ['VSMain_color', 'vs_6_0'], ['PSMain_snow', 'ps_6_0'], ['PSMain_checker', 'ps_6_0'], + ['PSMain_checker_luma', 'ps_6_0'], + ['PSMain_checker_rgb', 'ps_6_0'], + ['PSMain_checker_vuya', 'ps_6_0'], + ['VSMain_pos', 'vs_6_0'], ] foreach shader : hlsl_sources diff --git a/subprojects/gst-plugins-bad/sys/d3d12/meson.build b/subprojects/gst-plugins-bad/sys/d3d12/meson.build index 532db7fbaf..dee0e52d8a 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/meson.build +++ b/subprojects/gst-plugins-bad/sys/d3d12/meson.build @@ -6,6 +6,7 @@ d3d12_sources = [ 'gstd3d12commandallocatorpool.cpp', 'gstd3d12commandlistpool.cpp', 'gstd3d12commandqueue.cpp', + 'gstd3d12compositor.cpp', 'gstd3d12converter-builder.cpp', 'gstd3d12converter.cpp', 'gstd3d12convert.cpp', diff --git a/subprojects/gst-plugins-bad/sys/d3d12/plugin.cpp b/subprojects/gst-plugins-bad/sys/d3d12/plugin.cpp index c7152fffe5..460b0ec2ac 100644 --- a/subprojects/gst-plugins-bad/sys/d3d12/plugin.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d12/plugin.cpp @@ -34,6 +34,7 @@ #include "gstd3d12upload.h" #include "gstd3d12videosink.h" #include "gstd3d12testsrc.h" +#include "gstd3d12compositor.h" #include "gstd3d12h264dec.h" #include "gstd3d12h265dec.h" #include "gstd3d12vp9dec.h" @@ -116,6 +117,8 @@ plugin_init (GstPlugin * plugin) "d3d12videosink", GST_RANK_NONE, GST_TYPE_D3D12_VIDEO_SINK); gst_element_register (plugin, "d3d12testsrc", GST_RANK_NONE, GST_TYPE_D3D12_TEST_SRC); + gst_element_register (plugin, + "d3d12compositor", GST_RANK_NONE, GST_TYPE_D3D12_COMPOSITOR); return TRUE; }