mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-27 20:21:24 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/sys/d3d11/gstd3d11compositor.c
Seungha Yang 6e30cbb3fc d3d11: Introduce d3d11compositor element 
Add new video composition element which is equivalent to compositor
and glvideomixer elements. When d3d11 decoder elements are used,
d3d11compositor can do efficient graphics memory handling
(zero copying or at least copying memory on GPU memory space).

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1323>
2020-10-16 17:02:15 +00:00

2221 lines
70 KiB
C
Raw Blame History

/*
* GStreamer
* Copyright (C) 2004, 2008 Wim Taymans <wim@fluendo.com>
* Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
* Copyright (C) 2014 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
* Copyright (C) 2014 Thibault Saunier <tsaunier@gnome.org>
* Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstd3d11_fwd.h"
#include "gstd3d11compositor.h"
#include "gstd3d11device.h"
#include "gstd3d11memory.h"
#include "gstd3d11bufferpool.h"
#include "gstd3d11utils.h"
#include "gstd3d11format.h"
#include "gstd3d11colorconverter.h"
#include "gstd3d11shader.h"
#include <string.h>
GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_compositor_debug);
#define GST_CAT_DEFAULT gst_d3d11_compositor_debug
/**
* GstD3D11CompositorBlendOperation:
* @GST_D3D11_COMPOSITOR_BLEND_OP_ADD:
* Add source 1 and source 2
* @GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT:
* Subtract source 1 from source 2
* @GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT:
* Subtract source 2 from source 1
* @GST_D3D11_COMPOSITOR_BLEND_OP_MIN:
* Find the minimum of source 1 and source 2
* @GST_D3D11_COMPOSITOR_BLEND_OP_MAX:
* Find the maximum of source 1 and source 2
*/
GType
gst_d3d11_compositor_blend_operation_get_type (void)
{
static GType blend_operation_type = 0;
static const GEnumValue blend_operator[] = {
{GST_D3D11_COMPOSITOR_BLEND_OP_ADD, "Add source and background",
"add"},
{GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT,
"Subtract source from background",
"subtract"},
{GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT,
"Subtract background from source",
"rev-subtract"},
{GST_D3D11_COMPOSITOR_BLEND_OP_MIN,
"Minimum of source and background", "min"},
{GST_D3D11_COMPOSITOR_BLEND_OP_MAX,
"Maximum of source and background", "max"},
{0, NULL, NULL},
};
if (!blend_operation_type) {
blend_operation_type =
g_enum_register_static ("GstD3D11CompositorBlendOperation",
blend_operator);
}
return blend_operation_type;
}
static GstD3D11CompositorBlendOperation
gst_d3d11_compositor_blend_operation_from_native (D3D11_BLEND_OP blend_op)
{
switch (blend_op) {
case D3D11_BLEND_OP_ADD:
return GST_D3D11_COMPOSITOR_BLEND_OP_ADD;
case D3D11_BLEND_OP_SUBTRACT:
return GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT;
case D3D11_BLEND_OP_REV_SUBTRACT:
return GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT;
case D3D11_BLEND_OP_MIN:
return GST_D3D11_COMPOSITOR_BLEND_OP_MIN;
case D3D11_BLEND_OP_MAX:
return GST_D3D11_COMPOSITOR_BLEND_OP_MAX;
default:
g_assert_not_reached ();
break;
}
return GST_D3D11_COMPOSITOR_BLEND_OP_ADD;
}
static D3D11_BLEND_OP
gst_d3d11_compositor_blend_operation_to_native (GstD3D11CompositorBlendOperation
op)
{
switch (op) {
case GST_D3D11_COMPOSITOR_BLEND_OP_ADD:
return D3D11_BLEND_OP_ADD;
case GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT:
return D3D11_BLEND_OP_SUBTRACT;
case GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT:
return D3D11_BLEND_OP_REV_SUBTRACT;
case GST_D3D11_COMPOSITOR_BLEND_OP_MIN:
return D3D11_BLEND_OP_MIN;
case GST_D3D11_COMPOSITOR_BLEND_OP_MAX:
return D3D11_BLEND_OP_MAX;
default:
g_assert_not_reached ();
break;
}
return D3D11_BLEND_OP_ADD;
}
/**
* GstD3D11CompositorBlend:
* @GST_D3D11_COMPOSITOR_BLEND_ZERO:
* The blend factor is (0, 0, 0, 0). No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_ONE:
* The blend factor is (1, 1, 1, 1). No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR:
* The blend factor is (Rs, Gs, Bs, As),
* that is color data (RGB) from a pixel shader. No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR:
* The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As),
* that is color data (RGB) from a pixel shader.
* The pre-blend operation inverts the data, generating 1 - RGB.
* @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA:
* The blend factor is (As, As, As, As),
* that is alpha data (A) from a pixel shader. No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA:
* The blend factor is ( 1 - As, 1 - As, 1 - As, 1 - As),
* that is alpha data (A) from a pixel shader.
* The pre-blend operation inverts the data, generating 1 - A.
* @GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA:
* The blend factor is (Ad, Ad, Ad, Ad),
* that is alpha data from a render target. No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA:
* The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad),
* that is alpha data from a render target.
* The pre-blend operation inverts the data, generating 1 - A.
* @GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR:
* The blend factor is (Rd, Gd, Bd, Ad),
* that is color data from a render target. No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR:
* The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad),
* that is color data from a render target.
* The pre-blend operation inverts the data, generating 1 - RGB.
* @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT:
* The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad).
* The pre-blend operation clamps the data to 1 or less.
* @GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR:
* The blend factor is the blend factor set with
* ID3D11DeviceContext::OMSetBlendState. No pre-blend operation.
* @GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR:
* The blend factor is the blend factor set with
* ID3D11DeviceContext::OMSetBlendState.
* The pre-blend operation inverts the blend factor,
* generating 1 - blend_factor.
*/
GType
gst_d3d11_compositor_blend_get_type (void)
{
static GType blend_type = 0;
static const GEnumValue blend[] = {
{GST_D3D11_COMPOSITOR_BLEND_ZERO,
"The blend factor is (0, 0, 0, 0)", "zero"},
{GST_D3D11_COMPOSITOR_BLEND_ONE,
"The blend factor is (1, 1, 1, 1)", "one"},
{GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR,
"The blend factor is (Rs, Gs, Bs, As)", "src-color"},
{GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR,
"The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As)",
"inv-src-color"},
{GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA,
"The blend factor is (As, As, As, As)", "src-alpha"},
{GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA,
"The blend factor is (1 - As, 1 - As, 1 - As, 1 - As)",
"inv-src-alpha"},
{GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA,
"The blend factor is (Ad, Ad, Ad, Ad)", "dest-alpha"},
{GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA,
"The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad)",
"inv-dest-alpha"},
{GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR,
"The blend factor is (Rd, Gd, Bd, Ad)", "dest-color"},
{GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR,
"The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad)",
"inv-dest-color"},
{GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT,
"The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad)",
"src-alpha-sat"},
{GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR,
"User defined blend factor", "blend-factor"},
{GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR,
"Inverse of user defined blend factor", "inv-blend-factor"},
{0, NULL, NULL},
};
if (!blend_type) {
blend_type = g_enum_register_static ("GstD3D11CompositorBlend", blend);
}
return blend_type;
}
static GstD3D11CompositorBlend
gst_d3d11_compositor_blend_from_native (D3D11_BLEND blend)
{
switch (blend) {
case D3D11_BLEND_ZERO:
return GST_D3D11_COMPOSITOR_BLEND_ZERO;
case D3D11_BLEND_ONE:
return GST_D3D11_COMPOSITOR_BLEND_ONE;
case D3D11_BLEND_SRC_COLOR:
return GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR;
case D3D11_BLEND_INV_SRC_COLOR:
return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR;
case D3D11_BLEND_SRC_ALPHA:
return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA;
case D3D11_BLEND_INV_SRC_ALPHA:
return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA;
case D3D11_BLEND_DEST_ALPHA:
return GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA;
case D3D11_BLEND_INV_DEST_ALPHA:
return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA;
case D3D11_BLEND_DEST_COLOR:
return GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR;
case D3D11_BLEND_INV_DEST_COLOR:
return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR;
case D3D11_BLEND_SRC_ALPHA_SAT:
return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT;
case D3D11_BLEND_BLEND_FACTOR:
return GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR;
case D3D11_BLEND_INV_BLEND_FACTOR:
return GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR;
default:
g_assert_not_reached ();
break;
}
return GST_D3D11_COMPOSITOR_BLEND_ZERO;
}
static D3D11_BLEND
gst_d3d11_compositor_blend_to_native (GstD3D11CompositorBlend blend)
{
switch (blend) {
case GST_D3D11_COMPOSITOR_BLEND_ZERO:
return D3D11_BLEND_ZERO;
case GST_D3D11_COMPOSITOR_BLEND_ONE:
return D3D11_BLEND_ONE;
case GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR:
return D3D11_BLEND_SRC_COLOR;
case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR:
return D3D11_BLEND_INV_SRC_COLOR;
case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA:
return D3D11_BLEND_SRC_ALPHA;
case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA:
return D3D11_BLEND_INV_SRC_ALPHA;
case GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA:
return D3D11_BLEND_DEST_ALPHA;
case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA:
return D3D11_BLEND_INV_DEST_ALPHA;
case GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR:
return D3D11_BLEND_DEST_COLOR;
case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR:
return D3D11_BLEND_INV_DEST_COLOR;
case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT:
return D3D11_BLEND_SRC_ALPHA_SAT;
case GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR:
return D3D11_BLEND_BLEND_FACTOR;
case GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR:
return D3D11_BLEND_INV_BLEND_FACTOR;
default:
g_assert_not_reached ();
break;
}
return D3D11_BLEND_ZERO;
}
GType
gst_d3d11_compositor_background_get_type (void)
{
static GType compositor_background_type = 0;
static const GEnumValue compositor_background[] = {
{GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"},
{GST_D3D11_COMPOSITOR_BACKGROUND_BLACK, "Black", "black"},
{GST_D3D11_COMPOSITOR_BACKGROUND_WHITE, "White", "white"},
{GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT,
"Transparent Background to enable further compositing", "transparent"},
{0, NULL, NULL},
};
if (!compositor_background_type) {
compositor_background_type =
g_enum_register_static ("GstD3D11CompositorBackground",
compositor_background);
}
return compositor_background_type;
}
/* *INDENT-OFF* */
static const gchar checker_vs_src[] =
"struct VS_INPUT\n"
"{\n"
" float4 Position : POSITION;\n"
"};\n"
"\n"
"struct VS_OUTPUT\n"
"{\n"
" float4 Position: SV_POSITION;\n"
"};\n"
"\n"
"VS_OUTPUT main(VS_INPUT input)\n"
"{\n"
" return input;\n"
"}\n";
static const gchar checker_ps_src[] =
"static const float blocksize = 8.0;\n"
"static const float4 high = float4(0.667, 0.667, 0.667, 1.0);\n"
"static const float4 low = float4(0.333, 0.333, 0.333, 1.0);\n"
"struct PS_INPUT\n"
"{\n"
" float4 Position: SV_POSITION;\n"
"};\n"
"struct PS_OUTPUT\n"
"{\n"
" float4 Plane: SV_TARGET;\n"
"};\n"
"PS_OUTPUT main(PS_INPUT input)\n"
"{\n"
" PS_OUTPUT output;\n"
" if ((input.Position.x % (blocksize * 2.0)) >= blocksize) {\n"
" if ((input.Position.y % (blocksize * 2.0)) >= blocksize)\n"
" output.Plane = low;\n"
" else\n"
" output.Plane = high;\n"
" } else {\n"
" if ((input.Position.y % (blocksize * 2.0)) < blocksize)\n"
" output.Plane = low;\n"
" else\n"
" output.Plane = high;\n"
" }\n"
" return output;\n"
"}\n";
/* *INDENT-ON* */
struct _GstD3D11CompositorPad
{
GstVideoAggregatorConvertPad parent;
GstD3D11ColorConverter *convert;
GstBufferPool *fallback_pool;
GstBuffer *fallback_buf;
gboolean position_updated;
gboolean alpha_updated;
gboolean blend_desc_updated;
ID3D11BlendState *blend;
/* properties */
gint xpos;
gint ypos;
gint width;
gint height;
gdouble alpha;
D3D11_RENDER_TARGET_BLEND_DESC desc;
gfloat blend_factor[4];
};
struct _GstD3D11Compositor
{
GstVideoAggregator parent;
GstD3D11Device *device;
GstBufferPool *fallback_pool;
GstBuffer *fallback_buf;
GstD3D11Quad *checker_background;
D3D11_VIEWPORT viewport;
/* properties */
gint adapter;
GstD3D11CompositorBackground background;
};
enum
{
PROP_PAD_0,
PROP_PAD_XPOS,
PROP_PAD_YPOS,
PROP_PAD_WIDTH,
PROP_PAD_HEIGHT,
PROP_PAD_ALPHA,
PROP_PAD_BLEND_OP_RGB,
PROP_PAD_BLEND_OP_ALPHA,
PROP_PAD_BLEND_SRC_RGB,
PROP_PAD_BLEND_SRC_ALPHA,
PROP_PAD_BLEND_DEST_RGB,
PROP_PAD_BLEND_DEST_ALPHA,
PROP_PAD_BLEND_FACTOR_RED,
PROP_PAD_BLEND_FACTOR_GREEN,
PROP_PAD_BLEND_FACTOR_BLUE,
PROP_PAD_BLEND_FACTOR_ALPHA,
};
#define DEFAULT_PAD_XPOS 0
#define DEFAULT_PAD_YPOS 0
#define DEFAULT_PAD_WIDTH 0
#define DEFAULT_PAD_HEIGHT 0
#define DEFAULT_PAD_ALPHA 1.0
#define DEFAULT_PAD_BLEND_OP_RGB GST_D3D11_COMPOSITOR_BLEND_OP_ADD
#define DEFAULT_PAD_BLEND_OP_ALPHA GST_D3D11_COMPOSITOR_BLEND_OP_ADD
#define DEFAULT_PAD_BLEND_SRC_RGB GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA
#define DEFAULT_PAD_BLEND_SRC_ALPHA GST_D3D11_COMPOSITOR_BLEND_ONE
#define DEFAULT_PAD_BLEND_DEST_RGB GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA
#define DEFAULT_PAD_BLEND_DEST_ALPHA GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA
static void gst_d3d11_compositor_pad_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_d3d11_compositor_pad_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static gboolean
gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg, GstBuffer * buffer,
GstVideoFrame * prepared_frame);
static void
gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg, GstVideoFrame * prepared_frame);
static void
gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad);
#define gst_d3d11_compositor_pad_parent_class parent_pad_class
G_DEFINE_TYPE (GstD3D11CompositorPad, gst_d3d11_compositor_pad,
GST_TYPE_VIDEO_AGGREGATOR_PAD);
static void
gst_d3d11_compositor_pad_class_init (GstD3D11CompositorPadClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstVideoAggregatorPadClass *vaggpadclass =
GST_VIDEO_AGGREGATOR_PAD_CLASS (klass);
gobject_class->set_property = gst_d3d11_compositor_pad_set_property;
gobject_class->get_property = gst_d3d11_compositor_pad_get_property;
g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
g_param_spec_int ("xpos", "X Position", "X position of the picture",
G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
g_param_spec_int ("ypos", "Y Position", "Y position of the picture",
G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
g_param_spec_int ("width", "Width", "Width of the picture",
G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
g_param_spec_int ("height", "Height", "Height of the picture",
G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
DEFAULT_PAD_ALPHA,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_RGB,
g_param_spec_enum ("blend-op-rgb", "Blend Operation RGB",
"Blend equation for RGB", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION,
DEFAULT_PAD_BLEND_OP_RGB,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_ALPHA,
g_param_spec_enum ("blend-op-alpha", "Blend Operation Alpha",
"Blend equation for alpha", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION,
DEFAULT_PAD_BLEND_OP_ALPHA,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_PAD_BLEND_SRC_RGB,
g_param_spec_enum ("blend-src-rgb", "Blend Source RGB",
"Blend factor for source RGB",
GST_TYPE_D3D11_COMPOSITOR_BLEND,
DEFAULT_PAD_BLEND_SRC_RGB,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_PAD_BLEND_SRC_ALPHA,
g_param_spec_enum ("blend-src-alpha",
"Blend Source Alpha",
"Blend factor for source alpha, \"*-color\" values are not allowed",
GST_TYPE_D3D11_COMPOSITOR_BLEND,
DEFAULT_PAD_BLEND_SRC_ALPHA,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_PAD_BLEND_DEST_RGB,
g_param_spec_enum ("blend-dest-rgb",
"Blend Destination RGB",
"Blend factor for destination RGB",
GST_TYPE_D3D11_COMPOSITOR_BLEND,
DEFAULT_PAD_BLEND_DEST_RGB,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_PAD_BLEND_DEST_ALPHA,
g_param_spec_enum ("blend-dest-alpha",
"Blend Destination Alpha",
"Blend factor for destination alpha, "
"\"*-color\" values are not allowed",
GST_TYPE_D3D11_COMPOSITOR_BLEND,
DEFAULT_PAD_BLEND_DEST_ALPHA,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_RED,
g_param_spec_float ("blend-factor-red", "Blend Factor Red",
"Blend factor for red component "
"when blend type is \"blend-factor\" or \"inv-blend-factor\"",
0.0, 1.0, 1.0,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_GREEN,
g_param_spec_float ("blend-factor-green", "Blend Factor Green",
"Blend factor for green component "
"when blend type is \"blend-factor\" or \"inv-blend-factor\"",
0.0, 1.0, 1.0,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_BLUE,
g_param_spec_float ("blend-factor-blue", "Blend Factor Blue",
"Blend factor for blue component "
"when blend type is \"blend-factor\" or \"inv-blend-factor\"",
0.0, 1.0, 1.0,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_ALPHA,
g_param_spec_float ("blend-factor-alpha", "Blend Factor Alpha",
"Blend factor for alpha component "
"when blend type is \"blend-factor\" or \"inv-blend-factor\"",
0.0, 1.0, 1.0,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
vaggpadclass->prepare_frame =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_prepare_frame);
vaggpadclass->clean_frame =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_clean_frame);
}
static void
gst_d3d11_compositor_pad_init (GstD3D11CompositorPad * pad)
{
pad->xpos = DEFAULT_PAD_XPOS;
pad->ypos = DEFAULT_PAD_YPOS;
pad->width = DEFAULT_PAD_WIDTH;
pad->height = DEFAULT_PAD_HEIGHT;
pad->alpha = DEFAULT_PAD_ALPHA;
gst_d3d11_compositor_pad_init_blend_options (pad);
}
static void
gst_d3d11_compositor_pad_update_blend_function (GstD3D11CompositorPad * pad,
D3D11_BLEND * value, GstD3D11CompositorBlend new_value)
{
D3D11_BLEND temp = gst_d3d11_compositor_blend_to_native (new_value);
if (temp == *value)
return;
*value = temp;
pad->blend_desc_updated = TRUE;
}
static void
gst_d3d11_compositor_pad_update_blend_equation (GstD3D11CompositorPad * pad,
D3D11_BLEND_OP * value, GstD3D11CompositorBlend new_value)
{
D3D11_BLEND_OP temp =
gst_d3d11_compositor_blend_operation_to_native (new_value);
if (temp == *value)
return;
*value = temp;
pad->blend_desc_updated = TRUE;
}
static void
gst_d3d11_compositor_pad_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object);
switch (prop_id) {
case PROP_PAD_XPOS:
pad->xpos = g_value_get_int (value);
pad->position_updated = TRUE;
break;
case PROP_PAD_YPOS:
pad->ypos = g_value_get_int (value);
pad->position_updated = TRUE;
break;
case PROP_PAD_WIDTH:
pad->width = g_value_get_int (value);
pad->position_updated = TRUE;
break;
case PROP_PAD_HEIGHT:
pad->height = g_value_get_int (value);
pad->position_updated = TRUE;
break;
case PROP_PAD_ALPHA:
{
gdouble alpha = g_value_get_double (value);
if (pad->alpha != alpha) {
pad->alpha_updated = TRUE;
pad->alpha = alpha;
}
break;
}
case PROP_PAD_BLEND_OP_RGB:
gst_d3d11_compositor_pad_update_blend_equation (pad, &pad->desc.BlendOp,
g_value_get_enum (value));
break;
case PROP_PAD_BLEND_OP_ALPHA:
gst_d3d11_compositor_pad_update_blend_equation (pad,
&pad->desc.BlendOpAlpha, g_value_get_enum (value));
break;
case PROP_PAD_BLEND_SRC_RGB:
gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.SrcBlend,
g_value_get_enum (value));
break;
case PROP_PAD_BLEND_SRC_ALPHA:
{
GstD3D11CompositorBlend blend = g_value_get_enum (value);
if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) {
g_warning ("%d is not allowed for %s", blend, pspec->name);
} else {
gst_d3d11_compositor_pad_update_blend_function (pad,
&pad->desc.SrcBlendAlpha, blend);
}
break;
}
case PROP_PAD_BLEND_DEST_RGB:
gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.DestBlend,
g_value_get_enum (value));
break;
case PROP_PAD_BLEND_DEST_ALPHA:
{
GstD3D11CompositorBlend blend = g_value_get_enum (value);
if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR ||
blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) {
g_warning ("%d is not allowed for %s", blend, pspec->name);
} else {
gst_d3d11_compositor_pad_update_blend_function (pad,
&pad->desc.DestBlendAlpha, blend);
}
break;
}
case PROP_PAD_BLEND_FACTOR_RED:
pad->blend_factor[0] = g_value_get_float (value);
break;
case PROP_PAD_BLEND_FACTOR_GREEN:
pad->blend_factor[1] = g_value_get_float (value);
break;
case PROP_PAD_BLEND_FACTOR_BLUE:
pad->blend_factor[2] = g_value_get_float (value);
break;
case PROP_PAD_BLEND_FACTOR_ALPHA:
pad->blend_factor[3] = g_value_get_float (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_d3d11_compositor_pad_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object);
switch (prop_id) {
case PROP_PAD_XPOS:
g_value_set_int (value, pad->xpos);
break;
case PROP_PAD_YPOS:
g_value_set_int (value, pad->ypos);
break;
case PROP_PAD_WIDTH:
g_value_set_int (value, pad->width);
break;
case PROP_PAD_HEIGHT:
g_value_set_int (value, pad->height);
break;
case PROP_PAD_ALPHA:
g_value_set_double (value, pad->alpha);
break;
case PROP_PAD_BLEND_OP_RGB:
g_value_set_enum (value,
gst_d3d11_compositor_blend_operation_from_native (pad->desc.BlendOp));
break;
case PROP_PAD_BLEND_OP_ALPHA:
g_value_set_enum (value,
gst_d3d11_compositor_blend_operation_from_native (pad->
desc.BlendOpAlpha));
break;
case PROP_PAD_BLEND_SRC_RGB:
g_value_set_enum (value,
gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlend));
break;
case PROP_PAD_BLEND_SRC_ALPHA:
g_value_set_enum (value,
gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlendAlpha));
break;
case PROP_PAD_BLEND_DEST_RGB:
g_value_set_enum (value,
gst_d3d11_compositor_blend_from_native (pad->desc.DestBlend));
break;
case PROP_PAD_BLEND_DEST_ALPHA:
g_value_set_enum (value,
gst_d3d11_compositor_blend_from_native (pad->desc.DestBlendAlpha));
break;
case PROP_PAD_BLEND_FACTOR_RED:
g_value_set_float (value, pad->blend_factor[0]);
break;
case PROP_PAD_BLEND_FACTOR_GREEN:
g_value_set_float (value, pad->blend_factor[1]);
break;
case PROP_PAD_BLEND_FACTOR_BLUE:
g_value_set_float (value, pad->blend_factor[2]);
break;
case PROP_PAD_BLEND_FACTOR_ALPHA:
g_value_set_float (value, pad->blend_factor[3]);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad)
{
gint i;
pad->desc.BlendEnable = TRUE;
pad->desc.SrcBlend =
gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_RGB);
pad->desc.DestBlend =
gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_RGB);
pad->desc.BlendOp =
gst_d3d11_compositor_blend_operation_to_native (DEFAULT_PAD_BLEND_OP_RGB);
pad->desc.SrcBlendAlpha =
gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_ALPHA);
pad->desc.DestBlendAlpha =
gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_ALPHA);
pad->desc.BlendOpAlpha =
gst_d3d11_compositor_blend_operation_to_native
(DEFAULT_PAD_BLEND_OP_ALPHA);
pad->desc.RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL;
for (i = 0; i < G_N_ELEMENTS (pad->blend_factor); i++)
pad->blend_factor[i] = 1.0f;
}
static gboolean
gst_d3d11_compositor_configure_fallback_pool (GstD3D11Compositor * self,
GstVideoInfo * info, gint bind_flags, GstBufferPool ** pool)
{
GstStructure *config;
GstD3D11AllocationParams *d3d11_params;
GstCaps *caps;
if (*pool) {
gst_buffer_pool_set_active (*pool, FALSE);
gst_clear_object (pool);
}
caps = gst_video_info_to_caps (info);
if (!caps) {
GST_ERROR_OBJECT (self, "Couldn't create caps from info");
return FALSE;
}
*pool = gst_d3d11_buffer_pool_new (self->device);
config = gst_buffer_pool_get_config (*pool);
gst_buffer_pool_config_set_params (config,
caps, GST_VIDEO_INFO_SIZE (info), 0, 0);
gst_caps_unref (caps);
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (self->device,
info, 0, bind_flags);
} else {
gint i;
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++)
d3d11_params->desc[i].BindFlags |= bind_flags;
}
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
gst_buffer_pool_set_config (*pool, config);
gst_buffer_pool_set_active (*pool, TRUE);
return TRUE;
}
static gboolean
gst_d3d11_compsitor_prepare_fallback_buffer (GstD3D11Compositor * self,
GstVideoInfo * info, gboolean is_input, GstBufferPool ** pool,
GstBuffer ** fallback_buffer)
{
GstBuffer *new_buf = NULL;
gint bind_flags = D3D11_BIND_SHADER_RESOURCE;
gint i;
gst_clear_buffer (fallback_buffer);
if (!is_input)
bind_flags = D3D11_BIND_RENDER_TARGET;
if (*pool == NULL &&
!gst_d3d11_compositor_configure_fallback_pool (self, info,
bind_flags, pool)) {
GST_ERROR_OBJECT (self, "Couldn't configure fallback buffer pool");
return FALSE;
}
if (gst_buffer_pool_acquire_buffer (*pool, &new_buf, NULL)
!= GST_FLOW_OK) {
GST_ERROR_OBJECT (self, "Couldn't get fallback buffer from pool");
return FALSE;
}
for (i = 0; i < gst_buffer_n_memory (new_buf); i++) {
GstD3D11Memory *new_mem =
(GstD3D11Memory *) gst_buffer_peek_memory (new_buf, i);
if (is_input && !gst_d3d11_memory_ensure_shader_resource_view (new_mem)) {
GST_ERROR_OBJECT (self, "Couldn't prepare shader resource view");
gst_buffer_unref (new_buf);
return FALSE;
} else if (!is_input &&
!gst_d3d11_memory_ensure_render_target_view (new_mem)) {
GST_ERROR_OBJECT (self, "Couldn't prepare render target view");
gst_buffer_unref (new_buf);
return FALSE;
}
}
*fallback_buffer = new_buf;
return TRUE;
}
static gboolean
gst_d3d11_compositor_copy_buffer (GstD3D11Compositor * self,
GstVideoInfo * info, GstBuffer * src_buf, GstBuffer * dest_buf,
gboolean do_device_copy)
{
gint i;
if (do_device_copy) {
return gst_d3d11_buffer_copy_into (dest_buf, src_buf);
} else {
GstVideoFrame src_frame, dest_frame;
if (!gst_video_frame_map (&src_frame, info, src_buf,
GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF)) {
GST_ERROR_OBJECT (self, "Couldn't map input buffer");
return FALSE;
}
if (!gst_video_frame_map (&dest_frame, info, dest_buf,
GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF)) {
GST_ERROR_OBJECT (self, "Couldn't fallback buffer");
gst_video_frame_unmap (&src_frame);
return FALSE;
}
for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) {
if (!gst_video_frame_copy_plane (&dest_frame, &src_frame, i)) {
GST_ERROR_OBJECT (self, "Couldn't copy %dth plane", i);
gst_video_frame_unmap (&dest_frame);
gst_video_frame_unmap (&src_frame);
return FALSE;
}
}
gst_video_frame_unmap (&dest_frame);
gst_video_frame_unmap (&src_frame);
}
return TRUE;
}
static gboolean
gst_d3d11_compositor_check_d3d11_memory (GstD3D11Compositor * self,
GstBuffer * buffer, gboolean is_input, gboolean * view_available)
{
gint i;
gboolean ret = TRUE;
*view_available = TRUE;
for (i = 0; i < gst_buffer_n_memory (buffer); i++) {
GstMemory *mem = gst_buffer_peek_memory (buffer, i);
GstD3D11Memory *dmem;
if (!gst_is_d3d11_memory (mem)) {
ret = FALSE;
goto done;
}
dmem = (GstD3D11Memory *) mem;
if (dmem->device != self->device) {
ret = FALSE;
goto done;
}
if (is_input) {
if (!gst_d3d11_memory_ensure_shader_resource_view (dmem))
*view_available = FALSE;
} else {
if (!gst_d3d11_memory_ensure_render_target_view (dmem))
*view_available = FALSE;
}
}
done:
if (!ret)
*view_available = FALSE;
return ret;
}
static void
gst_d3d11_compositor_pad_get_output_size (GstD3D11CompositorPad * comp_pad,
gint out_par_n, gint out_par_d, gint * width, gint * height)
{
GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad);
gint pad_width, pad_height;
guint dar_n, dar_d;
/* FIXME: Anything better we can do here? */
if (!vagg_pad->info.finfo
|| vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_DEBUG_OBJECT (comp_pad, "Have no caps yet");
*width = 0;
*height = 0;
return;
}
pad_width =
comp_pad->width <=
0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width;
pad_height =
comp_pad->height <=
0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height;
if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio");
*width = *height = 0;
return;
}
GST_TRACE_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)",
pad_width, pad_height, dar_n, dar_d,
GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
/* Pick either height or width, whichever is an integer multiple of the
* display aspect ratio. However, prefer preserving the height to account
* for interlaced video. */
if (pad_height % dar_n == 0) {
pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
} else if (pad_width % dar_d == 0) {
pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
} else {
pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
}
*width = pad_width;
*height = pad_height;
}
static GstVideoRectangle
clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width,
gint outer_height)
{
gint x2 = x + w;
gint y2 = y + h;
GstVideoRectangle clamped;
/* Clamp the x/y coordinates of this frame to the output boundaries to cover
* the case where (say, with negative xpos/ypos or w/h greater than the output
* size) the non-obscured portion of the frame could be outside the bounds of
* the video itself and hence not visible at all */
clamped.x = CLAMP (x, 0, outer_width);
clamped.y = CLAMP (y, 0, outer_height);
clamped.w = CLAMP (x2, 0, outer_width) - clamped.x;
clamped.h = CLAMP (y2, 0, outer_height) - clamped.y;
return clamped;
}
static gboolean
gst_d3d11_compositor_pad_check_frame_obscured (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg)
{
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
gint width, height;
GstVideoInfo *info = &vagg->info;
/* The rectangle representing this frame, clamped to the video's boundaries.
* Due to the clamping, this is different from the frame width/height above. */
GstVideoRectangle frame_rect;
/* There's three types of width/height here:
* 1. GST_VIDEO_FRAME_WIDTH/HEIGHT:
* The frame width/height (same as pad->info.height/width;
* see gst_video_frame_map())
* 2. cpad->width/height:
* The optional pad property for scaling the frame (if zero, the video is
* left unscaled)
*/
gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info),
GST_VIDEO_INFO_PAR_D (info), &width, &height);
frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height,
GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
if (frame_rect.w == 0 || frame_rect.h == 0) {
GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height "
"(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h);
return TRUE;
}
return FALSE;
}
static gboolean
gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg, GstBuffer * buffer,
GstVideoFrame * prepared_frame)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
GstBuffer *target_buf = buffer;
gboolean do_device_copy = FALSE;
/* Skip this frame */
if (gst_d3d11_compositor_pad_check_frame_obscured (pad, vagg))
return TRUE;
/* Use fallback buffer when input buffer is:
* - non-d3d11 memory
* - or, from different d3d11 device
* - or not bound to shader resource
*/
if (!gst_d3d11_compositor_check_d3d11_memory (self,
buffer, TRUE, &do_device_copy) || !do_device_copy) {
if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &pad->info, TRUE,
&cpad->fallback_pool, &cpad->fallback_buf)) {
GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer");
return FALSE;
}
if (!gst_d3d11_compositor_copy_buffer (self, &pad->info, buffer,
cpad->fallback_buf, do_device_copy)) {
GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer");
gst_clear_buffer (&cpad->fallback_buf);
return FALSE;
}
target_buf = cpad->fallback_buf;
}
if (!gst_video_frame_map (prepared_frame, &pad->info, target_buf,
GST_MAP_READ | GST_MAP_D3D11)) {
GST_WARNING_OBJECT (pad, "Couldn't map input buffer");
return FALSE;
}
return TRUE;
}
static void
gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg, GstVideoFrame * prepared_frame)
{
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
GST_VIDEO_AGGREGATOR_PAD_CLASS (parent_pad_class)->clean_frame (pad,
vagg, prepared_frame);
gst_clear_buffer (&cpad->fallback_buf);
}
static gboolean
gst_d3d11_compositor_pad_setup_converter (GstVideoAggregatorPad * pad,
GstVideoAggregator * vagg)
{
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
RECT rect;
gint width, height;
GstVideoInfo *info = &vagg->info;
GstVideoRectangle frame_rect;
gboolean is_first = FALSE;
#ifndef GST_DISABLE_GST_DEBUG
guint zorder = 0;
#endif
if (!cpad->convert || cpad->alpha_updated) {
if (cpad->convert)
gst_d3d11_color_converter_free (cpad->convert);
cpad->convert =
gst_d3d11_color_converter_new_with_alpha (self->device,
&pad->info, &vagg->info, cpad->alpha);
cpad->alpha_updated = FALSE;
if (!cpad->convert) {
GST_ERROR_OBJECT (pad, "Couldn't create converter");
return FALSE;
}
is_first = TRUE;
}
if (!cpad->blend || cpad->blend_desc_updated) {
HRESULT hr;
D3D11_BLEND_DESC desc = { 0, };
ID3D11BlendState *blend = NULL;
ID3D11Device *device_handle =
gst_d3d11_device_get_device_handle (self->device);
if (cpad->blend) {
ID3D11BlendState_Release (cpad->blend);
cpad->blend = NULL;
}
desc.AlphaToCoverageEnable = FALSE;
desc.IndependentBlendEnable = FALSE;
desc.RenderTarget[0] = cpad->desc;
hr = ID3D11Device_CreateBlendState (device_handle, &desc, &blend);
if (!gst_d3d11_result (hr, self->device)) {
GST_ERROR_OBJECT (pad, "Couldn't create blend staten, hr: 0x%x",
(guint) hr);
return FALSE;
}
cpad->blend = blend;
}
if (!is_first && !cpad->position_updated)
return TRUE;
gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info),
GST_VIDEO_INFO_PAR_D (info), &width, &height);
frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height,
GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info));
rect.left = frame_rect.x;
rect.top = frame_rect.y;
rect.right = frame_rect.x + frame_rect.w;
rect.bottom = frame_rect.y + frame_rect.h;
#ifndef GST_DISABLE_GST_DEBUG
g_object_get (pad, "zorder", &zorder, NULL);
GST_LOG_OBJECT (pad, "Update position, pad-xpos %d, pad-ypos %d, "
"pad-zorder %d, pad-width %d, pad-height %d, in-resolution %dx%d, "
"out-resoution %dx%d, dst-{left,top,right,bottom} %d-%d-%d-%d",
cpad->xpos, cpad->ypos, zorder, cpad->width, cpad->height,
GST_VIDEO_INFO_WIDTH (&pad->info), GST_VIDEO_INFO_HEIGHT (&pad->info),
GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info),
(gint) rect.left, (gint) rect.top, (gint) rect.right, (gint) rect.bottom);
#endif
cpad->position_updated = FALSE;
return gst_d3d11_color_converter_update_dest_rect (cpad->convert, &rect);
}
static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink_%u",
GST_PAD_SINK,
GST_PAD_REQUEST,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ RGBA, BGRA }")
));
static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ RGBA, BGRA }")
));
enum
{
PROP_0,
PROP_ADAPTER,
PROP_BACKGROUND,
};
#define DEFAULT_ADAPTER -1
#define DEFAULT_BACKGROUND GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER
static void gst_d3d11_compositor_child_proxy_init (gpointer g_iface,
gpointer iface_data);
static void gst_d3d11_compositor_dispose (GObject * object);
static void gst_d3d11_compositor_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_d3d11_compositor_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static GstPad *gst_d3d11_compositor_request_new_pad (GstElement * element,
GstPadTemplate * templ, const gchar * name, const GstCaps * caps);
static void gst_d3d11_compositor_release_pad (GstElement * element,
GstPad * pad);
static void gst_d3d11_compositor_set_context (GstElement * element,
GstContext * context);
static gboolean gst_d3d11_compositor_start (GstAggregator * aggregator);
static gboolean gst_d3d11_compositor_stop (GstAggregator * aggregator);
static gboolean gst_d3d11_compositor_sink_query (GstAggregator * aggregator,
GstAggregatorPad * pad, GstQuery * query);
static gboolean gst_d3d11_compositor_src_query (GstAggregator * aggregator,
GstQuery * query);
static GstCaps *gst_d3d11_compositor_fixate_src_caps (GstAggregator *
aggregator, GstCaps * caps);
static gboolean gst_d3d11_compositor_propose_allocation (GstAggregator *
aggregator, GstAggregatorPad * pad, GstQuery * decide_query,
GstQuery * query);
static gboolean gst_d3d11_compositor_decide_allocation (GstAggregator *
aggregator, GstQuery * query);
static GstFlowReturn
gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg,
GstBuffer * outbuf);
#define gst_d3d11_compositor_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstD3D11Compositor, gst_d3d11_compositor,
GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
gst_d3d11_compositor_child_proxy_init));
static void
gst_d3d11_compositor_class_init (GstD3D11CompositorClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstAggregatorClass *aggregator_class = GST_AGGREGATOR_CLASS (klass);
GstVideoAggregatorClass *vagg_class = GST_VIDEO_AGGREGATOR_CLASS (klass);
gobject_class->dispose = gst_d3d11_compositor_dispose;
gobject_class->set_property = gst_d3d11_compositor_set_property;
gobject_class->get_property = gst_d3d11_compositor_get_property;
g_object_class_install_property (gobject_class, PROP_ADAPTER,
g_param_spec_int ("adapter", "Adapter",
"Adapter index for creating device (-1 for default)",
-1, G_MAXINT32, DEFAULT_ADAPTER,
G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_BACKGROUND,
g_param_spec_enum ("background", "Background", "Background type",
GST_TYPE_COMPOSITOR_BACKGROUND,
DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
element_class->request_new_pad =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_request_new_pad);
element_class->release_pad =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_release_pad);
element_class->set_context =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_set_context);
aggregator_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_start);
aggregator_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_stop);
aggregator_class->sink_query =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_sink_query);
aggregator_class->src_query =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_src_query);
aggregator_class->fixate_src_caps =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_fixate_src_caps);
aggregator_class->propose_allocation =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_propose_allocation);
aggregator_class->decide_allocation =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_decide_allocation);
vagg_class->aggregate_frames =
GST_DEBUG_FUNCPTR (gst_d3d11_compositor_aggregate_frames);
gst_element_class_add_static_pad_template_with_gtype (element_class,
&src_template, GST_TYPE_AGGREGATOR_PAD);
gst_element_class_add_static_pad_template_with_gtype (element_class,
&sink_template, GST_TYPE_D3D11_COMPOSITOR_PAD);
gst_element_class_set_static_metadata (element_class, "Direct3D11 Compositor",
"Filter/Editor/Video/Compositor",
"Composite multiple video streams via D3D11 API",
"Seungha Yang <seungha@centricular.com>");
}
static void
gst_d3d11_compositor_init (GstD3D11Compositor * self)
{
self->adapter = DEFAULT_ADAPTER;
self->background = DEFAULT_BACKGROUND;
}
static void
gst_d3d11_compositor_dispose (GObject * object)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
gst_clear_object (&self->device);
gst_clear_buffer (&self->fallback_buf);
gst_clear_object (&self->fallback_pool);
g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_d3d11_compositor_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
switch (prop_id) {
case PROP_ADAPTER:
self->adapter = g_value_get_int (value);
break;
case PROP_BACKGROUND:
self->background = g_value_get_enum (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_d3d11_compositor_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object);
switch (prop_id) {
case PROP_ADAPTER:
g_value_set_int (value, self->adapter);
break;
case PROP_BACKGROUND:
g_value_set_enum (value, self->background);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static GObject *
gst_d3d11_compositor_child_proxy_get_child_by_index (GstChildProxy * proxy,
guint index)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy);
GObject *obj = NULL;
GST_OBJECT_LOCK (self);
obj = g_list_nth_data (GST_ELEMENT_CAST (self)->sinkpads, index);
if (obj)
gst_object_ref (obj);
GST_OBJECT_UNLOCK (self);
return obj;
}
static guint
gst_d3d11_compositor_child_proxy_get_children_count (GstChildProxy * proxy)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy);
guint count = 0;
GST_OBJECT_LOCK (self);
count = GST_ELEMENT_CAST (self)->numsinkpads;
GST_OBJECT_UNLOCK (self);
GST_INFO_OBJECT (self, "Children Count: %d", count);
return count;
}
static void
gst_d3d11_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data)
{
GstChildProxyInterface *iface = g_iface;
iface->get_child_by_index =
gst_d3d11_compositor_child_proxy_get_child_by_index;
iface->get_children_count =
gst_d3d11_compositor_child_proxy_get_children_count;
}
static GstPad *
gst_d3d11_compositor_request_new_pad (GstElement * element,
GstPadTemplate * templ, const gchar * name, const GstCaps * caps)
{
GstPad *pad;
pad = GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
templ, name, caps);
if (pad == NULL)
goto could_not_create;
gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (pad),
GST_OBJECT_NAME (pad));
GST_DEBUG_OBJECT (element, "Created new pad %s:%s", GST_DEBUG_PAD_NAME (pad));
return pad;
could_not_create:
{
GST_DEBUG_OBJECT (element, "could not create/add pad");
return NULL;
}
}
static void
gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element);
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
GST_DEBUG_OBJECT (self, "Releasing pad %s:%s", GST_DEBUG_PAD_NAME (pad));
gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad),
GST_OBJECT_NAME (pad));
gst_clear_buffer (&cpad->fallback_buf);
gst_clear_object (&cpad->fallback_pool);
g_clear_pointer (&cpad->convert, gst_d3d11_color_converter_free);
if (cpad->blend) {
ID3D11BlendState_Release (cpad->blend);
cpad->blend = NULL;
}
GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
}
static void
gst_d3d11_compositor_set_context (GstElement * element, GstContext * context)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element);
gst_d3d11_handle_set_context (element, context, self->adapter, &self->device);
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
static gboolean
gst_d3d11_compositor_start (GstAggregator * aggregator)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self),
self->adapter, &self->device)) {
GST_ERROR_OBJECT (self, "Failed to get D3D11 device");
return FALSE;
}
return GST_AGGREGATOR_CLASS (parent_class)->start (aggregator);
}
static gboolean
gst_d3d11_compositor_stop (GstAggregator * aggregator)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
g_clear_pointer (&self->checker_background, gst_d3d11_quad_free);
gst_clear_object (&self->device);
return GST_AGGREGATOR_CLASS (parent_class)->stop (aggregator);
}
static GstCaps *
gst_d3d11_compositor_sink_getcaps (GstPad * pad, GstCaps * filter)
{
GstCaps *sinkcaps;
GstCaps *template_caps;
GstCaps *filtered_caps;
GstCaps *returned_caps;
template_caps = gst_pad_get_pad_template_caps (pad);
sinkcaps = gst_pad_get_current_caps (pad);
if (sinkcaps == NULL) {
sinkcaps = gst_caps_ref (template_caps);
} else {
sinkcaps = gst_caps_merge (sinkcaps, gst_caps_ref (template_caps));
}
if (filter) {
filtered_caps = gst_caps_intersect (sinkcaps, filter);
gst_caps_unref (sinkcaps);
} else {
filtered_caps = sinkcaps; /* pass ownership */
}
returned_caps = gst_caps_intersect (filtered_caps, template_caps);
gst_caps_unref (template_caps);
gst_caps_unref (filtered_caps);
GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps);
return returned_caps;
}
static gboolean
gst_d3d11_compositor_sink_acceptcaps (GstPad * pad, GstCaps * caps)
{
gboolean ret;
GstCaps *template_caps;
GST_DEBUG_OBJECT (pad, "try accept caps of %" GST_PTR_FORMAT, caps);
template_caps = gst_pad_get_pad_template_caps (pad);
template_caps = gst_caps_make_writable (template_caps);
ret = gst_caps_can_intersect (caps, template_caps);
GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT,
(ret ? "" : "not "), caps);
gst_caps_unref (template_caps);
return ret;
}
static gboolean
gst_d3d11_compositor_sink_query (GstAggregator * aggregator,
GstAggregatorPad * pad, GstQuery * query)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
{
gboolean ret;
ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query,
self->device);
if (ret)
return TRUE;
break;
}
case GST_QUERY_CAPS:
{
GstCaps *filter, *caps;
gst_query_parse_caps (query, &filter);
caps = gst_d3d11_compositor_sink_getcaps (GST_PAD (pad), filter);
gst_query_set_caps_result (query, caps);
gst_caps_unref (caps);
return TRUE;
}
case GST_QUERY_ACCEPT_CAPS:
{
GstCaps *caps;
gboolean ret;
gst_query_parse_accept_caps (query, &caps);
ret = gst_d3d11_compositor_sink_acceptcaps (GST_PAD (pad), caps);
gst_query_set_accept_caps_result (query, ret);
return TRUE;
}
default:
break;
}
return GST_AGGREGATOR_CLASS (parent_class)->sink_query (aggregator,
pad, query);
}
static gboolean
gst_d3d11_compositor_src_query (GstAggregator * aggregator, GstQuery * query)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:
{
gboolean ret;
ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query,
self->device);
if (ret)
return TRUE;
break;
}
default:
break;
}
return GST_AGGREGATOR_CLASS (parent_class)->src_query (aggregator, query);
}
static GstCaps *
gst_d3d11_compositor_fixate_src_caps (GstAggregator * aggregator,
GstCaps * caps)
{
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator);
GList *l;
gint best_width = -1, best_height = -1;
gint best_fps_n = -1, best_fps_d = -1;
gint par_n, par_d;
gdouble best_fps = 0.;
GstCaps *ret = NULL;
GstStructure *s;
ret = gst_caps_make_writable (caps);
/* we need this to calculate how large to make the output frame */
s = gst_caps_get_structure (ret, 0);
if (gst_structure_has_field (s, "pixel-aspect-ratio")) {
gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
} else {
par_n = par_d = 1;
}
GST_OBJECT_LOCK (vagg);
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
GstVideoAggregatorPad *vaggpad = l->data;
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (vaggpad);
gint this_width, this_height;
gint width, height;
gint fps_n, fps_d;
gdouble cur_fps;
fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
gst_d3d11_compositor_pad_get_output_size (cpad,
par_n, par_d, &width, &height);
if (width == 0 || height == 0)
continue;
this_width = width + MAX (cpad->xpos, 0);
this_height = height + MAX (cpad->ypos, 0);
if (best_width < this_width)
best_width = this_width;
if (best_height < this_height)
best_height = this_height;
if (fps_d == 0)
cur_fps = 0.0;
else
gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
if (best_fps < cur_fps) {
best_fps = cur_fps;
best_fps_n = fps_n;
best_fps_d = fps_d;
}
}
GST_OBJECT_UNLOCK (vagg);
if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
best_fps_n = 25;
best_fps_d = 1;
best_fps = 25.0;
}
gst_structure_fixate_field_nearest_int (s, "width", best_width);
gst_structure_fixate_field_nearest_int (s, "height", best_height);
gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
best_fps_d);
ret = gst_caps_fixate (ret);
GST_LOG_OBJECT (aggregator, "Fixated caps %" GST_PTR_FORMAT, ret);
return ret;
}
static gboolean
gst_d3d11_compositor_propose_allocation (GstAggregator * aggregator,
GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
GstVideoInfo info;
GstBufferPool *pool;
GstCaps *caps;
guint size;
gst_query_parse_allocation (query, &caps, NULL);
if (caps == NULL)
return FALSE;
if (!gst_video_info_from_caps (&info, caps))
return FALSE;
if (gst_query_get_n_allocation_pools (query) == 0) {
GstStructure *config;
GstD3D11AllocationParams *d3d11_params;
gint i;
pool = gst_d3d11_buffer_pool_new (self->device);
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_META);
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (self->device, &info, 0,
D3D11_BIND_SHADER_RESOURCE);
} else {
/* Set bind flag */
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE;
}
}
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
size = GST_VIDEO_INFO_SIZE (&info);
gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
if (!gst_buffer_pool_set_config (pool, config))
goto config_failed;
/* d3d11 buffer pool might update buffer size by self */
size = GST_D3D11_BUFFER_POOL (pool)->buffer_size;
gst_query_add_allocation_pool (query, pool, size, 0, 0);
gst_object_unref (pool);
}
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
return TRUE;
/* ERRORS */
config_failed:
{
GST_ERROR_OBJECT (self, "failed to set config");
gst_object_unref (pool);
return FALSE;
}
}
static gboolean
gst_d3d11_compositor_decide_allocation (GstAggregator * aggregator,
GstQuery * query)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator);
GstCaps *caps;
GstBufferPool *pool = NULL;
guint n, size, min, max;
GstVideoInfo info;
GstStructure *config;
GstD3D11AllocationParams *d3d11_params;
gst_query_parse_allocation (query, &caps, NULL);
if (!caps) {
GST_DEBUG_OBJECT (self, "No output caps");
return FALSE;
}
gst_video_info_from_caps (&info, caps);
n = gst_query_get_n_allocation_pools (query);
if (n > 0)
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
/* create our own pool */
if (pool && !GST_D3D11_BUFFER_POOL (pool)) {
gst_object_unref (pool);
pool = NULL;
}
if (!pool) {
pool = gst_d3d11_buffer_pool_new (self->device);
min = max = 0;
size = (guint) info.size;
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config);
if (!d3d11_params) {
d3d11_params = gst_d3d11_allocation_params_new (self->device,
&info, 0, D3D11_BIND_RENDER_TARGET);
} else {
gint i;
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) {
d3d11_params->desc[i].BindFlags |= D3D11_BIND_RENDER_TARGET;
}
}
gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params);
gst_d3d11_allocation_params_free (d3d11_params);
gst_buffer_pool_set_config (pool, config);
/* d3d11 buffer pool might update buffer size by self */
size = GST_D3D11_BUFFER_POOL (pool)->buffer_size;
if (n > 0)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
gst_object_unref (pool);
return TRUE;
}
typedef struct
{
struct
{
FLOAT x;
FLOAT y;
FLOAT z;
} position;
struct
{
FLOAT u;
FLOAT v;
} texture;
} VertexData;
static GstD3D11Quad *
gst_d3d11_compositor_create_checker_quad (GstD3D11Compositor * self)
{
ID3D11PixelShader *ps = NULL;
ID3D11VertexShader *vs = NULL;
ID3D11InputLayout *layout = NULL;
GstD3D11Quad *quad = NULL;
VertexData *vertex_data;
WORD *indices;
ID3D11Device *device_handle;
ID3D11DeviceContext *context_handle;
D3D11_MAPPED_SUBRESOURCE map;
D3D11_INPUT_ELEMENT_DESC input_desc;
D3D11_BUFFER_DESC buffer_desc;
ID3D11Buffer *vertex_buffer = NULL;
ID3D11Buffer *index_buffer = NULL;
gboolean ret = FALSE;
HRESULT hr;
device_handle = gst_d3d11_device_get_device_handle (self->device);
context_handle = gst_d3d11_device_get_device_context_handle (self->device);
ret = gst_d3d11_create_pixel_shader (self->device, checker_ps_src, &ps);
if (!ret) {
GST_ERROR_OBJECT (self, "Couldn't setup pixel shader");
goto done;
}
memset (&input_desc, 0, sizeof (D3D11_INPUT_ELEMENT_DESC));
input_desc.SemanticName = "POSITION";
input_desc.SemanticIndex = 0;
input_desc.Format = DXGI_FORMAT_R32G32B32_FLOAT;
input_desc.InputSlot = 0;
input_desc.AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT;
input_desc.InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA;
input_desc.InstanceDataStepRate = 0;
if (!gst_d3d11_create_vertex_shader (self->device, checker_vs_src,
&input_desc, 1, &vs, &layout)) {
GST_ERROR_OBJECT (self, "Couldn't setup vertex shader");
goto done;
}
memset (&buffer_desc, 0, sizeof (D3D11_BUFFER_DESC));
buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
buffer_desc.ByteWidth = sizeof (VertexData) * 4;
buffer_desc.BindFlags = D3D11_BIND_VERTEX_BUFFER;
buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL,
&vertex_buffer);
if (!gst_d3d11_result (hr, self->device)) {
GST_ERROR_OBJECT (self,
"Couldn't create vertex buffer, hr: 0x%x", (guint) hr);
goto done;
}
hr = ID3D11DeviceContext_Map (context_handle,
(ID3D11Resource *) vertex_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map);
if (!gst_d3d11_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't map vertex buffer, hr: 0x%x", (guint) hr);
goto done;
}
vertex_data = (VertexData *) map.pData;
/* bottom left */
/* bottom left */
vertex_data[0].position.x = -1.0f;
vertex_data[0].position.y = -1.0f;
vertex_data[0].position.z = 0.0f;
vertex_data[0].texture.u = 0.0f;
vertex_data[0].texture.v = 1.0f;
/* top left */
vertex_data[1].position.x = -1.0f;
vertex_data[1].position.y = 1.0f;
vertex_data[1].position.z = 0.0f;
vertex_data[1].texture.u = 0.0f;
vertex_data[1].texture.v = 0.0f;
/* top right */
vertex_data[2].position.x = 1.0f;
vertex_data[2].position.y = 1.0f;
vertex_data[2].position.z = 0.0f;
vertex_data[2].texture.u = 1.0f;
vertex_data[2].texture.v = 0.0f;
/* bottom right */
vertex_data[3].position.x = 1.0f;
vertex_data[3].position.y = -1.0f;
vertex_data[3].position.z = 0.0f;
vertex_data[3].texture.u = 1.0f;
vertex_data[3].texture.v = 1.0f;
ID3D11DeviceContext_Unmap (context_handle,
(ID3D11Resource *) vertex_buffer, 0);
buffer_desc.Usage = D3D11_USAGE_DYNAMIC;
buffer_desc.ByteWidth = sizeof (WORD) * 6;
buffer_desc.BindFlags = D3D11_BIND_INDEX_BUFFER;
buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE;
hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL,
&index_buffer);
if (!gst_d3d11_result (hr, self->device)) {
GST_ERROR_OBJECT (self,
"Couldn't create index buffer, hr: 0x%x", (guint) hr);
goto done;
}
hr = ID3D11DeviceContext_Map (context_handle,
(ID3D11Resource *) index_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map);
if (!gst_d3d11_result (hr, self->device)) {
GST_ERROR_OBJECT (self, "Couldn't map index buffer, hr: 0x%x", (guint) hr);
goto done;
}
indices = (WORD *) map.pData;
/* clockwise indexing */
indices[0] = 0; /* bottom left */
indices[1] = 1; /* top left */
indices[2] = 2; /* top right */
indices[3] = 3; /* bottom right */
indices[4] = 0; /* bottom left */
indices[5] = 2; /* top right */
ID3D11DeviceContext_Unmap (context_handle,
(ID3D11Resource *) index_buffer, 0);
quad = gst_d3d11_quad_new (self->device,
ps, vs, layout, NULL, NULL, NULL, NULL, vertex_buffer,
sizeof (VertexData), index_buffer, DXGI_FORMAT_R16_UINT, 6);
if (!quad) {
GST_ERROR_OBJECT (self, "Couldn't setup quad");
goto done;
}
done:
if (ps)
ID3D11PixelShader_Release (ps);
if (vs)
ID3D11VertexShader_Release (vs);
if (layout)
ID3D11InputLayout_Release (layout);
if (vertex_buffer)
ID3D11Buffer_Release (vertex_buffer);
if (index_buffer)
ID3D11Buffer_Release (index_buffer);
return quad;
}
static gboolean
gst_d3d11_compositor_draw_background_checker (GstD3D11Compositor * self,
ID3D11RenderTargetView * rtv)
{
if (!self->checker_background) {
GstVideoInfo *info = &GST_VIDEO_AGGREGATOR_CAST (self)->info;
self->checker_background = gst_d3d11_compositor_create_checker_quad (self);
if (!self->checker_background)
return FALSE;
self->viewport.TopLeftX = 0;
self->viewport.TopLeftY = 0;
self->viewport.Width = GST_VIDEO_INFO_WIDTH (info);
self->viewport.Height = GST_VIDEO_INFO_HEIGHT (info);
self->viewport.MinDepth = 0.0f;
self->viewport.MaxDepth = 1.0f;
}
return gst_d3d11_draw_quad_unlocked (self->checker_background,
&self->viewport, 1, NULL, 0, &rtv, 1, NULL, NULL, NULL);
}
/* Must be called with d3d11 device lock */
static gboolean
gst_d3d11_compositor_draw_background (GstD3D11Compositor * self,
ID3D11RenderTargetView * rtv)
{
ID3D11DeviceContext *device_context =
gst_d3d11_device_get_device_context_handle (self->device);
FLOAT rgba[4] = { 0.0f, 0.0f, 0.0f, 1.0f };
switch (self->background) {
case GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER:
return gst_d3d11_compositor_draw_background_checker (self, rtv);
case GST_D3D11_COMPOSITOR_BACKGROUND_BLACK:
/* {0, 0, 0, 1} */
break;
case GST_D3D11_COMPOSITOR_BACKGROUND_WHITE:
rgba[0] = 1.0f;
rgba[1] = 1.0f;
rgba[2] = 1.0f;
break;
case GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT:
rgba[3] = 0.0f;
break;
default:
g_assert_not_reached ();
return FALSE;
}
ID3D11DeviceContext_ClearRenderTargetView (device_context, rtv, rgba);
return TRUE;
}
static GstFlowReturn
gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg,
GstBuffer * outbuf)
{
GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg);
GList *iter;
GstBuffer *target_buf = outbuf;
gboolean need_copy = FALSE;
gboolean do_device_copy = FALSE;
GstFlowReturn ret = GST_FLOW_OK;
ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES] = { NULL, };
gint i, j;
gint view_idx;
/* Use fallback buffer when output buffer is:
* - non-d3d11 memory
* - or, from different d3d11 device
* - or not bound to render target
*/
if (!gst_d3d11_compositor_check_d3d11_memory (self,
outbuf, FALSE, &do_device_copy) || !do_device_copy) {
if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &vagg->info, FALSE,
&self->fallback_pool, &self->fallback_buf)) {
GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer");
return GST_FLOW_ERROR;
}
GST_TRACE_OBJECT (self, "Will draw on fallback texture");
need_copy = TRUE;
target_buf = self->fallback_buf;
}
view_idx = 0;
for (i = 0; i < gst_buffer_n_memory (target_buf); i++) {
GstMemory *mem = gst_buffer_peek_memory (target_buf, i);
GstD3D11Memory *dmem;
if (!gst_is_d3d11_memory (mem)) {
GST_ERROR_OBJECT (self, "Invalid output memory");
return GST_FLOW_ERROR;
}
dmem = (GstD3D11Memory *) mem;
if (!gst_d3d11_memory_ensure_render_target_view (dmem)) {
GST_ERROR_OBJECT (self, "Render target view is unavailable");
return GST_FLOW_ERROR;
}
for (j = 0; j < dmem->num_render_target_views; j++) {
g_assert (view_idx < GST_VIDEO_MAX_PLANES);
rtv[view_idx] = dmem->render_target_view[j];
view_idx++;
}
/* Mark need-download for fallback buffer use case */
GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD);
}
gst_d3d11_device_lock (self->device);
/* XXX: the number of render target view must be one here, since we support
* only RGBA or BGRA */
if (!gst_d3d11_compositor_draw_background (self, rtv[0])) {
GST_ERROR_OBJECT (self, "Couldn't draw background");
gst_d3d11_device_unlock (self->device);
ret = GST_FLOW_ERROR;
goto done;
}
GST_OBJECT_LOCK (self);
for (iter = GST_ELEMENT (vagg)->sinkpads; iter; iter = g_list_next (iter)) {
GstVideoAggregatorPad *pad = iter->data;
GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad);
GstVideoFrame *prepared_frame =
gst_video_aggregator_pad_get_prepared_frame (pad);
ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES] = { NULL, };
GstBuffer *buffer;
if (!prepared_frame)
continue;
if (!gst_d3d11_compositor_pad_setup_converter (pad, vagg)) {
GST_ERROR_OBJECT (self, "Couldn't setup converter");
ret = GST_FLOW_ERROR;
break;
}
buffer = prepared_frame->buffer;
view_idx = 0;
for (i = 0; i < gst_buffer_n_memory (buffer); i++) {
GstD3D11Memory *dmem =
(GstD3D11Memory *) gst_buffer_peek_memory (buffer, i);
for (j = 0; j < dmem->num_shader_resource_views; j++) {
g_assert (view_idx < GST_VIDEO_MAX_PLANES);
srv[view_idx] = dmem->shader_resource_view[j];
view_idx++;
}
}
if (!gst_d3d11_color_converter_convert_unlocked (cpad->convert, srv, rtv,
cpad->blend, cpad->blend_factor)) {
GST_ERROR_OBJECT (self, "Couldn't convert frame");
ret = GST_FLOW_ERROR;
break;
}
}
GST_OBJECT_UNLOCK (self);
gst_d3d11_device_unlock (self->device);
if (ret != GST_FLOW_OK)
goto done;
if (need_copy && !gst_d3d11_compositor_copy_buffer (self, &vagg->info,
target_buf, outbuf, do_device_copy)) {
GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer");
ret = GST_FLOW_ERROR;
}
done:
gst_clear_buffer (&self->fallback_buf);
return ret;
}
Reference in a new issue View git blame Copy permalink