2014-05-22 17:46:02 +00:00
|
|
|
/* Video compositor plugin
|
|
|
|
* Copyright (C) 2004, 2008 Wim Taymans <wim@fluendo.com>
|
|
|
|
* Copyright (C) 2010 Sebastian Dröge <sebastian.droege@collabora.co.uk>
|
|
|
|
* Copyright (C) 2014 Mathieu Duponchelle <mathieu.duponchelle@opencreed.com>
|
|
|
|
* Copyright (C) 2014 Thibault Saunier <tsaunier@gnome.org>
|
|
|
|
*
|
|
|
|
* This library is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Library General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* This library is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Library General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Library General Public
|
|
|
|
* License along with this library; if not, write to the
|
|
|
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
|
|
|
* Boston, MA 02110-1301, USA.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* SECTION:element-compositor
|
2017-03-08 18:01:13 +00:00
|
|
|
* @title: compositor
|
2014-05-22 17:46:02 +00:00
|
|
|
*
|
2019-08-20 14:12:22 +00:00
|
|
|
* Compositor can accept AYUV, VUYA, ARGB and BGRA video streams. For each of the requested
|
2014-05-22 17:46:02 +00:00
|
|
|
* sink pads it will compare the incoming geometry and framerate to define the
|
|
|
|
* output parameters. Indeed output video frames will have the geometry of the
|
|
|
|
* biggest incoming video stream and the framerate of the fastest incoming one.
|
|
|
|
*
|
|
|
|
* Compositor will do colorspace conversion.
|
2017-03-08 18:01:13 +00:00
|
|
|
*
|
2014-05-22 17:46:02 +00:00
|
|
|
* Individual parameters for each input stream can be configured on the
|
2014-12-27 10:19:47 +00:00
|
|
|
* #GstCompositorPad:
|
|
|
|
*
|
2017-03-08 18:01:13 +00:00
|
|
|
* * "xpos": The x-coordinate position of the top-left corner of the picture (#gint)
|
|
|
|
* * "ypos": The y-coordinate position of the top-left corner of the picture (#gint)
|
|
|
|
* * "width": The width of the picture; the input will be scaled if necessary (#gint)
|
|
|
|
* * "height": The height of the picture; the input will be scaled if necessary (#gint)
|
|
|
|
* * "alpha": The transparency of the picture; between 0.0 and 1.0. The blending
|
|
|
|
* is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). (#gdouble)
|
|
|
|
* * "zorder": The z-order position of the picture in the composition (#guint)
|
2014-05-22 17:46:02 +00:00
|
|
|
*
|
2017-03-08 18:01:13 +00:00
|
|
|
* ## Sample pipelines
|
2014-05-22 17:46:02 +00:00
|
|
|
* |[
|
|
|
|
* gst-launch-1.0 \
|
|
|
|
* videotestsrc pattern=1 ! \
|
|
|
|
* video/x-raw,format=AYUV,framerate=\(fraction\)10/1,width=100,height=100 ! \
|
|
|
|
* videobox border-alpha=0 top=-70 bottom=-70 right=-220 ! \
|
|
|
|
* compositor name=comp sink_0::alpha=0.7 sink_1::alpha=0.5 ! \
|
|
|
|
* videoconvert ! xvimagesink \
|
|
|
|
* videotestsrc ! \
|
|
|
|
* video/x-raw,format=AYUV,framerate=\(fraction\)5/1,width=320,height=240 ! comp.
|
|
|
|
* ]| A pipeline to demonstrate compositor used together with videobox.
|
|
|
|
* This should show a 320x240 pixels video test source with some transparency
|
|
|
|
* showing the background checker pattern. Another video test source with just
|
2016-04-15 20:22:51 +00:00
|
|
|
* the snow pattern of 100x100 pixels is overlaid on top of the first one on
|
2014-05-22 17:46:02 +00:00
|
|
|
* the left vertically centered with a small transparency showing the first
|
|
|
|
* video test source behind and the checker pattern under it. Note that the
|
|
|
|
* framerate of the output video is 10 frames per second.
|
|
|
|
* |[
|
|
|
|
* gst-launch-1.0 videotestsrc pattern=1 ! \
|
|
|
|
* video/x-raw, framerate=\(fraction\)10/1, width=100, height=100 ! \
|
|
|
|
* compositor name=comp ! videoconvert ! ximagesink \
|
|
|
|
* videotestsrc ! \
|
|
|
|
* video/x-raw, framerate=\(fraction\)5/1, width=320, height=240 ! comp.
|
2019-08-29 17:42:39 +00:00
|
|
|
* ]| A pipeline to demonstrate bgra comping. (This does not demonstrate alpha blending).
|
2014-05-22 17:46:02 +00:00
|
|
|
* |[
|
|
|
|
* gst-launch-1.0 videotestsrc pattern=1 ! \
|
|
|
|
* video/x-raw,format =I420, framerate=\(fraction\)10/1, width=100, height=100 ! \
|
|
|
|
* compositor name=comp ! videoconvert ! ximagesink \
|
|
|
|
* videotestsrc ! \
|
|
|
|
* video/x-raw,format=I420, framerate=\(fraction\)5/1, width=320, height=240 ! comp.
|
|
|
|
* ]| A pipeline to test I420
|
|
|
|
* |[
|
|
|
|
* gst-launch-1.0 compositor name=comp sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50 ! \
|
|
|
|
* videoconvert ! ximagesink \
|
|
|
|
* videotestsrc pattern=snow timestamp-offset=3000000000 ! \
|
|
|
|
* "video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1" ! \
|
|
|
|
* timeoverlay ! queue2 ! comp. \
|
|
|
|
* videotestsrc pattern=smpte ! \
|
|
|
|
* "video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1" ! \
|
|
|
|
* timeoverlay ! queue2 ! comp.
|
|
|
|
* ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds)
|
2017-03-08 18:01:13 +00:00
|
|
|
*
|
2014-05-22 17:46:02 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
|
|
#include "config.h"
|
|
|
|
#endif
|
|
|
|
|
|
|
|
#include <string.h>
|
|
|
|
|
|
|
|
#include "compositor.h"
|
|
|
|
|
|
|
|
#ifdef DISABLE_ORC
|
|
|
|
#define orc_memset memset
|
|
|
|
#else
|
|
|
|
#include <orc/orcfunctions.h>
|
|
|
|
#endif
|
|
|
|
|
|
|
|
GST_DEBUG_CATEGORY_STATIC (gst_compositor_debug);
|
|
|
|
#define GST_CAT_DEFAULT gst_compositor_debug
|
|
|
|
|
2019-08-20 14:12:22 +00:00
|
|
|
#define FORMATS " { AYUV, VUYA, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\
|
2014-05-22 17:46:02 +00:00
|
|
|
" YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\
|
|
|
|
" RGBx, BGRx } "
|
|
|
|
|
|
|
|
static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
|
|
|
|
GST_PAD_SRC,
|
|
|
|
GST_PAD_ALWAYS,
|
|
|
|
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS))
|
|
|
|
);
|
|
|
|
|
|
|
|
static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
|
|
|
|
GST_PAD_SINK,
|
|
|
|
GST_PAD_REQUEST,
|
2020-10-30 16:56:16 +00:00
|
|
|
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (GST_VIDEO_FORMATS_ALL))
|
2014-05-22 17:46:02 +00:00
|
|
|
);
|
|
|
|
|
2018-05-04 14:13:16 +00:00
|
|
|
static void gst_compositor_child_proxy_init (gpointer g_iface,
|
|
|
|
gpointer iface_data);
|
|
|
|
|
2018-10-27 15:44:27 +00:00
|
|
|
#define GST_TYPE_COMPOSITOR_OPERATOR (gst_compositor_operator_get_type())
|
|
|
|
static GType
|
|
|
|
gst_compositor_operator_get_type (void)
|
|
|
|
{
|
|
|
|
static GType compositor_operator_type = 0;
|
|
|
|
|
|
|
|
static const GEnumValue compositor_operator[] = {
|
|
|
|
{COMPOSITOR_OPERATOR_SOURCE, "Source", "source"},
|
|
|
|
{COMPOSITOR_OPERATOR_OVER, "Over", "over"},
|
|
|
|
{COMPOSITOR_OPERATOR_ADD, "Add", "add"},
|
|
|
|
{0, NULL, NULL},
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!compositor_operator_type) {
|
|
|
|
compositor_operator_type =
|
|
|
|
g_enum_register_static ("GstCompositorOperator", compositor_operator);
|
|
|
|
}
|
|
|
|
return compositor_operator_type;
|
|
|
|
}
|
|
|
|
|
2020-05-29 00:43:59 +00:00
|
|
|
#define GST_TYPE_COMPOSITOR_BACKGROUND (gst_compositor_background_get_type())
|
|
|
|
static GType
|
|
|
|
gst_compositor_background_get_type (void)
|
|
|
|
{
|
|
|
|
static GType compositor_background_type = 0;
|
|
|
|
|
|
|
|
static const GEnumValue compositor_background[] = {
|
|
|
|
{COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"},
|
|
|
|
{COMPOSITOR_BACKGROUND_BLACK, "Black", "black"},
|
|
|
|
{COMPOSITOR_BACKGROUND_WHITE, "White", "white"},
|
|
|
|
{COMPOSITOR_BACKGROUND_TRANSPARENT,
|
|
|
|
"Transparent Background to enable further compositing", "transparent"},
|
|
|
|
{0, NULL, NULL},
|
|
|
|
};
|
|
|
|
|
|
|
|
if (!compositor_background_type) {
|
|
|
|
compositor_background_type =
|
|
|
|
g_enum_register_static ("GstCompositorBackground",
|
|
|
|
compositor_background);
|
|
|
|
}
|
|
|
|
return compositor_background_type;
|
|
|
|
}
|
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
#define DEFAULT_PAD_XPOS 0
|
|
|
|
#define DEFAULT_PAD_YPOS 0
|
2020-11-07 01:59:09 +00:00
|
|
|
#define DEFAULT_PAD_WIDTH -1
|
|
|
|
#define DEFAULT_PAD_HEIGHT -1
|
2014-05-22 17:46:02 +00:00
|
|
|
#define DEFAULT_PAD_ALPHA 1.0
|
2018-10-27 15:44:27 +00:00
|
|
|
#define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER
|
2014-05-22 17:46:02 +00:00
|
|
|
enum
|
|
|
|
{
|
|
|
|
PROP_PAD_0,
|
|
|
|
PROP_PAD_XPOS,
|
|
|
|
PROP_PAD_YPOS,
|
2014-11-26 14:02:14 +00:00
|
|
|
PROP_PAD_WIDTH,
|
|
|
|
PROP_PAD_HEIGHT,
|
2017-07-06 18:26:21 +00:00
|
|
|
PROP_PAD_ALPHA,
|
2018-10-27 15:44:27 +00:00
|
|
|
PROP_PAD_OPERATOR,
|
2014-05-22 17:46:02 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad,
|
2018-05-06 13:21:24 +00:00
|
|
|
GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD);
|
2014-05-22 17:46:02 +00:00
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_pad_get_property (GObject * object, guint prop_id,
|
|
|
|
GValue * value, GParamSpec * pspec)
|
|
|
|
{
|
|
|
|
GstCompositorPad *pad = GST_COMPOSITOR_PAD (object);
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
case PROP_PAD_XPOS:
|
|
|
|
g_value_set_int (value, pad->xpos);
|
|
|
|
break;
|
|
|
|
case PROP_PAD_YPOS:
|
|
|
|
g_value_set_int (value, pad->ypos);
|
|
|
|
break;
|
2014-11-26 14:02:14 +00:00
|
|
|
case PROP_PAD_WIDTH:
|
|
|
|
g_value_set_int (value, pad->width);
|
|
|
|
break;
|
|
|
|
case PROP_PAD_HEIGHT:
|
|
|
|
g_value_set_int (value, pad->height);
|
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
case PROP_PAD_ALPHA:
|
|
|
|
g_value_set_double (value, pad->alpha);
|
|
|
|
break;
|
2018-10-27 15:44:27 +00:00
|
|
|
case PROP_PAD_OPERATOR:
|
|
|
|
g_value_set_enum (value, pad->op);
|
2017-07-06 18:26:21 +00:00
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
default:
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_pad_set_property (GObject * object, guint prop_id,
|
|
|
|
const GValue * value, GParamSpec * pspec)
|
|
|
|
{
|
|
|
|
GstCompositorPad *pad = GST_COMPOSITOR_PAD (object);
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
case PROP_PAD_XPOS:
|
|
|
|
pad->xpos = g_value_get_int (value);
|
|
|
|
break;
|
|
|
|
case PROP_PAD_YPOS:
|
|
|
|
pad->ypos = g_value_get_int (value);
|
|
|
|
break;
|
2014-11-26 14:02:14 +00:00
|
|
|
case PROP_PAD_WIDTH:
|
|
|
|
pad->width = g_value_get_int (value);
|
2018-07-18 11:05:26 +00:00
|
|
|
gst_video_aggregator_convert_pad_update_conversion_info
|
|
|
|
(GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad));
|
2014-11-26 14:02:14 +00:00
|
|
|
break;
|
|
|
|
case PROP_PAD_HEIGHT:
|
|
|
|
pad->height = g_value_get_int (value);
|
2018-07-18 11:05:26 +00:00
|
|
|
gst_video_aggregator_convert_pad_update_conversion_info
|
|
|
|
(GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad));
|
2014-11-26 14:02:14 +00:00
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
case PROP_PAD_ALPHA:
|
|
|
|
pad->alpha = g_value_get_double (value);
|
|
|
|
break;
|
2018-10-27 15:44:27 +00:00
|
|
|
case PROP_PAD_OPERATOR:
|
|
|
|
pad->op = g_value_get_enum (value);
|
2018-05-05 14:14:14 +00:00
|
|
|
gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad),
|
2018-10-27 15:44:27 +00:00
|
|
|
pad->op == COMPOSITOR_OPERATOR_ADD);
|
2017-07-06 18:26:21 +00:00
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
default:
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-05-14 03:04:21 +00:00
|
|
|
static void
|
2020-11-07 01:59:09 +00:00
|
|
|
_mixer_pad_get_output_size (GstCompositor * comp, GstCompositorPad * comp_pad,
|
|
|
|
gint out_par_n, gint out_par_d, gint * width, gint * height)
|
2015-05-14 03:04:21 +00:00
|
|
|
{
|
|
|
|
GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad);
|
|
|
|
gint pad_width, pad_height;
|
2015-05-20 14:56:01 +00:00
|
|
|
guint dar_n, dar_d;
|
2015-05-14 03:04:21 +00:00
|
|
|
|
2015-05-21 13:24:48 +00:00
|
|
|
/* FIXME: Anything better we can do here? */
|
|
|
|
if (!vagg_pad->info.finfo
|
|
|
|
|| vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) {
|
|
|
|
GST_DEBUG_OBJECT (comp_pad, "Have no caps yet");
|
|
|
|
*width = 0;
|
|
|
|
*height = 0;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2020-11-07 01:59:09 +00:00
|
|
|
if (comp->zero_size_is_unscaled) {
|
|
|
|
pad_width =
|
|
|
|
comp_pad->width <=
|
|
|
|
0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width;
|
|
|
|
pad_height =
|
|
|
|
comp_pad->height <=
|
|
|
|
0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height;
|
|
|
|
} else {
|
|
|
|
pad_width =
|
|
|
|
comp_pad->width <
|
|
|
|
0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width;
|
|
|
|
pad_height =
|
|
|
|
comp_pad->height <
|
|
|
|
0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (pad_width == 0 || pad_height == 0) {
|
|
|
|
*width = 0;
|
|
|
|
*height = 0;
|
|
|
|
return;
|
|
|
|
}
|
2015-05-14 03:04:21 +00:00
|
|
|
|
2016-05-15 12:34:33 +00:00
|
|
|
if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height,
|
|
|
|
GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
|
|
|
|
GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) {
|
|
|
|
GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio");
|
|
|
|
*width = *height = 0;
|
2017-02-28 11:06:41 +00:00
|
|
|
return;
|
2016-05-15 12:34:33 +00:00
|
|
|
}
|
2015-05-14 03:04:21 +00:00
|
|
|
GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width,
|
|
|
|
pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info),
|
2015-10-14 10:13:57 +00:00
|
|
|
GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d);
|
2015-05-14 03:04:21 +00:00
|
|
|
|
2019-05-26 13:23:25 +00:00
|
|
|
/* Pick either height or width, whichever is an integer multiple of the
|
|
|
|
* display aspect ratio. However, prefer preserving the height to account
|
|
|
|
* for interlaced video. */
|
2015-05-14 03:04:21 +00:00
|
|
|
if (pad_height % dar_n == 0) {
|
2015-05-20 14:56:01 +00:00
|
|
|
pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
|
2015-05-14 03:04:21 +00:00
|
|
|
} else if (pad_width % dar_d == 0) {
|
2015-05-20 14:56:01 +00:00
|
|
|
pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n);
|
2015-05-14 03:04:21 +00:00
|
|
|
} else {
|
2015-05-20 14:56:01 +00:00
|
|
|
pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d);
|
2015-05-14 03:04:21 +00:00
|
|
|
}
|
|
|
|
|
2016-05-15 12:34:33 +00:00
|
|
|
*width = pad_width;
|
|
|
|
*height = pad_height;
|
2015-05-14 03:04:21 +00:00
|
|
|
}
|
|
|
|
|
2015-04-18 09:39:02 +00:00
|
|
|
/* Test whether rectangle2 contains rectangle 1 (geometrically) */
|
|
|
|
static gboolean
|
2019-06-13 15:00:03 +00:00
|
|
|
is_rectangle_contained (const GstVideoRectangle rect1,
|
|
|
|
const GstVideoRectangle rect2)
|
2015-04-18 09:39:02 +00:00
|
|
|
{
|
|
|
|
if ((rect2.x <= rect1.x) && (rect2.y <= rect1.y) &&
|
|
|
|
((rect2.x + rect2.w) >= (rect1.x + rect1.w)) &&
|
|
|
|
((rect2.y + rect2.h) >= (rect1.y + rect1.h)))
|
|
|
|
return TRUE;
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
2015-08-26 10:10:16 +00:00
|
|
|
static GstVideoRectangle
|
2015-09-04 20:20:41 +00:00
|
|
|
clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width,
|
|
|
|
gint outer_height)
|
2015-08-26 10:10:16 +00:00
|
|
|
{
|
2015-08-29 11:21:08 +00:00
|
|
|
gint x2 = x + w;
|
|
|
|
gint y2 = y + h;
|
2015-08-26 10:10:16 +00:00
|
|
|
GstVideoRectangle clamped;
|
|
|
|
|
|
|
|
/* Clamp the x/y coordinates of this frame to the output boundaries to cover
|
|
|
|
* the case where (say, with negative xpos/ypos or w/h greater than the output
|
|
|
|
* size) the non-obscured portion of the frame could be outside the bounds of
|
|
|
|
* the video itself and hence not visible at all */
|
2015-08-29 11:21:08 +00:00
|
|
|
clamped.x = CLAMP (x, 0, outer_width);
|
|
|
|
clamped.y = CLAMP (y, 0, outer_height);
|
|
|
|
clamped.w = CLAMP (x2, 0, outer_width) - clamped.x;
|
|
|
|
clamped.h = CLAMP (y2, 0, outer_height) - clamped.y;
|
2015-08-26 10:10:16 +00:00
|
|
|
|
|
|
|
return clamped;
|
|
|
|
}
|
|
|
|
|
2019-05-26 15:29:23 +00:00
|
|
|
/* Call this with the lock taken */
|
|
|
|
static gboolean
|
2019-05-26 15:30:12 +00:00
|
|
|
_pad_obscures_rectangle (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad,
|
2020-07-17 01:56:47 +00:00
|
|
|
const GstVideoRectangle rect)
|
2019-05-26 15:29:23 +00:00
|
|
|
{
|
|
|
|
GstVideoRectangle pad_rect;
|
|
|
|
GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
|
|
|
|
|
|
|
|
/* No buffer to obscure the rectangle with */
|
2019-05-26 15:30:12 +00:00
|
|
|
if (!gst_video_aggregator_pad_has_current_buffer (pad))
|
2019-05-26 15:29:23 +00:00
|
|
|
return FALSE;
|
|
|
|
|
2020-07-17 01:56:47 +00:00
|
|
|
/* Can't obscure if we introduce alpha or if the format has an alpha
|
|
|
|
* component as we'd have to inspect every pixel to know if the frame is
|
|
|
|
* opaque, so assume it doesn't obscure
|
|
|
|
*/
|
|
|
|
if (cpad->alpha != 1.0 || GST_VIDEO_INFO_HAS_ALPHA (&pad->info))
|
2019-05-26 15:29:23 +00:00
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
pad_rect.x = cpad->xpos;
|
|
|
|
pad_rect.y = cpad->ypos;
|
|
|
|
/* Handle pixel and display aspect ratios to find the actual size */
|
2020-11-07 01:59:09 +00:00
|
|
|
_mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad,
|
|
|
|
GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info),
|
|
|
|
&(pad_rect.w), &(pad_rect.h));
|
2019-05-26 15:29:23 +00:00
|
|
|
|
|
|
|
if (!is_rectangle_contained (rect, pad_rect))
|
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
GST_DEBUG_OBJECT (pad, "Pad %s %ix%i@(%i,%i) obscures rect %ix%i@(%i,%i)",
|
|
|
|
GST_PAD_NAME (pad), pad_rect.w, pad_rect.h, pad_rect.x, pad_rect.y,
|
|
|
|
rect.w, rect.h, rect.x, rect.y);
|
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
|
2014-11-26 14:02:14 +00:00
|
|
|
static gboolean
|
|
|
|
gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
|
2018-05-05 13:49:17 +00:00
|
|
|
GstVideoAggregator * vagg, GstBuffer * buffer,
|
|
|
|
GstVideoFrame * prepared_frame)
|
2014-11-26 14:02:14 +00:00
|
|
|
{
|
|
|
|
GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
|
|
|
|
gint width, height;
|
2015-04-18 09:39:02 +00:00
|
|
|
gboolean frame_obscured = FALSE;
|
|
|
|
GList *l;
|
|
|
|
/* The rectangle representing this frame, clamped to the video's boundaries.
|
|
|
|
* Due to the clamping, this is different from the frame width/height above. */
|
|
|
|
GstVideoRectangle frame_rect;
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2015-04-18 09:39:02 +00:00
|
|
|
/* There's three types of width/height here:
|
|
|
|
* 1. GST_VIDEO_FRAME_WIDTH/HEIGHT:
|
2017-04-04 08:25:43 +00:00
|
|
|
* The frame width/height (same as pad->info.height/width;
|
2015-04-18 09:39:02 +00:00
|
|
|
* see gst_video_frame_map())
|
|
|
|
* 2. cpad->width/height:
|
|
|
|
* The optional pad property for scaling the frame (if zero, the video is
|
|
|
|
* left unscaled)
|
|
|
|
* 3. conversion_info.width/height:
|
2015-04-18 09:40:00 +00:00
|
|
|
* Equal to cpad->width/height if it's set, otherwise it's the pad
|
2015-04-18 09:39:02 +00:00
|
|
|
* width/height. See ->set_info()
|
|
|
|
* */
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2020-11-07 01:59:09 +00:00
|
|
|
_mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad,
|
|
|
|
GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info),
|
|
|
|
&width, &height);
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2015-08-26 10:10:16 +00:00
|
|
|
if (cpad->alpha == 0.0) {
|
2018-10-27 15:44:27 +00:00
|
|
|
GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame");
|
2015-08-26 10:10:16 +00:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height,
|
|
|
|
GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info));
|
|
|
|
|
|
|
|
if (frame_rect.w == 0 || frame_rect.h == 0) {
|
2018-10-27 15:44:27 +00:00
|
|
|
GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height "
|
2015-08-26 10:10:16 +00:00
|
|
|
"(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h);
|
|
|
|
goto done;
|
|
|
|
}
|
2015-04-18 09:39:02 +00:00
|
|
|
|
|
|
|
GST_OBJECT_LOCK (vagg);
|
|
|
|
/* Check if this frame is obscured by a higher-zorder frame
|
|
|
|
* TODO: Also skip a frame if it's obscured by a combination of
|
|
|
|
* higher-zorder frames */
|
2018-10-27 15:44:27 +00:00
|
|
|
l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next;
|
2017-07-06 18:26:21 +00:00
|
|
|
for (; l; l = l->next) {
|
2020-07-17 01:56:47 +00:00
|
|
|
if (_pad_obscures_rectangle (vagg, l->data, frame_rect)) {
|
2015-04-18 09:39:02 +00:00
|
|
|
frame_obscured = TRUE;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
GST_OBJECT_UNLOCK (vagg);
|
|
|
|
|
2018-05-05 13:49:17 +00:00
|
|
|
if (frame_obscured)
|
2015-04-18 09:39:02 +00:00
|
|
|
goto done;
|
|
|
|
|
2018-05-06 13:21:24 +00:00
|
|
|
return
|
|
|
|
GST_VIDEO_AGGREGATOR_PAD_CLASS
|
|
|
|
(gst_compositor_pad_parent_class)->prepare_frame (pad, vagg, buffer,
|
|
|
|
prepared_frame);
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2015-04-18 09:39:02 +00:00
|
|
|
done:
|
2014-11-26 14:02:14 +00:00
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2018-05-06 13:21:24 +00:00
|
|
|
gst_compositor_pad_create_conversion_info (GstVideoAggregatorConvertPad * pad,
|
|
|
|
GstVideoAggregator * vagg, GstVideoInfo * conversion_info)
|
2014-11-26 14:02:14 +00:00
|
|
|
{
|
|
|
|
GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad);
|
2018-05-06 13:21:24 +00:00
|
|
|
gint width, height;
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2018-05-06 13:21:24 +00:00
|
|
|
GST_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS
|
|
|
|
(gst_compositor_pad_parent_class)->create_conversion_info (pad, vagg,
|
|
|
|
conversion_info);
|
|
|
|
if (!conversion_info->finfo)
|
|
|
|
return;
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2020-11-07 01:59:09 +00:00
|
|
|
_mixer_pad_get_output_size (GST_COMPOSITOR (vagg), cpad,
|
|
|
|
GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info),
|
|
|
|
&width, &height);
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2018-05-06 13:21:24 +00:00
|
|
|
/* The only thing that can change here is the width
|
|
|
|
* and height, otherwise set_info would've been called */
|
|
|
|
if (GST_VIDEO_INFO_WIDTH (conversion_info) != width ||
|
|
|
|
GST_VIDEO_INFO_HEIGHT (conversion_info) != height) {
|
|
|
|
GstVideoInfo tmp_info;
|
2014-11-26 14:02:14 +00:00
|
|
|
|
2018-05-06 13:21:24 +00:00
|
|
|
/* Initialize with the wanted video format and our original width and
|
|
|
|
* height as we don't want to rescale. Then copy over the wanted
|
|
|
|
* colorimetry, and chroma-site and our current pixel-aspect-ratio
|
|
|
|
* and other relevant fields.
|
|
|
|
*/
|
|
|
|
gst_video_info_set_format (&tmp_info,
|
|
|
|
GST_VIDEO_INFO_FORMAT (conversion_info), width, height);
|
|
|
|
tmp_info.chroma_site = conversion_info->chroma_site;
|
|
|
|
tmp_info.colorimetry = conversion_info->colorimetry;
|
|
|
|
tmp_info.par_n = conversion_info->par_n;
|
|
|
|
tmp_info.par_d = conversion_info->par_d;
|
|
|
|
tmp_info.fps_n = conversion_info->fps_n;
|
|
|
|
tmp_info.fps_d = conversion_info->fps_d;
|
|
|
|
tmp_info.flags = conversion_info->flags;
|
|
|
|
tmp_info.interlace_mode = conversion_info->interlace_mode;
|
|
|
|
|
|
|
|
*conversion_info = tmp_info;
|
|
|
|
}
|
2014-11-26 14:02:14 +00:00
|
|
|
}
|
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
static void
|
|
|
|
gst_compositor_pad_class_init (GstCompositorPadClass * klass)
|
|
|
|
{
|
|
|
|
GObjectClass *gobject_class = (GObjectClass *) klass;
|
2014-11-26 14:02:14 +00:00
|
|
|
GstVideoAggregatorPadClass *vaggpadclass =
|
|
|
|
(GstVideoAggregatorPadClass *) klass;
|
2018-05-06 13:21:24 +00:00
|
|
|
GstVideoAggregatorConvertPadClass *vaggcpadclass =
|
|
|
|
(GstVideoAggregatorConvertPadClass *) klass;
|
2014-05-22 17:46:02 +00:00
|
|
|
|
|
|
|
gobject_class->set_property = gst_compositor_pad_set_property;
|
|
|
|
gobject_class->get_property = gst_compositor_pad_get_property;
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_XPOS,
|
|
|
|
g_param_spec_int ("xpos", "X Position", "X Position of the picture",
|
|
|
|
G_MININT, G_MAXINT, DEFAULT_PAD_XPOS,
|
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_YPOS,
|
|
|
|
g_param_spec_int ("ypos", "Y Position", "Y Position of the picture",
|
|
|
|
G_MININT, G_MAXINT, DEFAULT_PAD_YPOS,
|
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
2014-11-26 14:02:14 +00:00
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_WIDTH,
|
|
|
|
g_param_spec_int ("width", "Width", "Width of the picture",
|
|
|
|
G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH,
|
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT,
|
|
|
|
g_param_spec_int ("height", "Height", "Height of the picture",
|
|
|
|
G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT,
|
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
2014-05-22 17:46:02 +00:00
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_ALPHA,
|
|
|
|
g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
|
|
|
|
DEFAULT_PAD_ALPHA,
|
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
2018-10-27 15:44:27 +00:00
|
|
|
g_object_class_install_property (gobject_class, PROP_PAD_OPERATOR,
|
|
|
|
g_param_spec_enum ("operator", "Operator",
|
|
|
|
"Blending operator to use for blending this pad over the previous ones",
|
|
|
|
GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR,
|
2017-07-06 18:26:21 +00:00
|
|
|
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
|
2014-11-26 14:02:14 +00:00
|
|
|
|
|
|
|
vaggpadclass->prepare_frame =
|
|
|
|
GST_DEBUG_FUNCPTR (gst_compositor_pad_prepare_frame);
|
2018-05-06 13:21:24 +00:00
|
|
|
|
|
|
|
vaggcpadclass->create_conversion_info =
|
|
|
|
GST_DEBUG_FUNCPTR (gst_compositor_pad_create_conversion_info);
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_pad_init (GstCompositorPad * compo_pad)
|
|
|
|
{
|
|
|
|
compo_pad->xpos = DEFAULT_PAD_XPOS;
|
|
|
|
compo_pad->ypos = DEFAULT_PAD_YPOS;
|
|
|
|
compo_pad->alpha = DEFAULT_PAD_ALPHA;
|
2018-10-27 15:44:27 +00:00
|
|
|
compo_pad->op = DEFAULT_PAD_OPERATOR;
|
2020-11-07 01:59:09 +00:00
|
|
|
compo_pad->width = DEFAULT_PAD_WIDTH;
|
|
|
|
compo_pad->height = DEFAULT_PAD_HEIGHT;
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
/* GstCompositor */
|
|
|
|
#define DEFAULT_BACKGROUND COMPOSITOR_BACKGROUND_CHECKER
|
2020-11-07 01:59:09 +00:00
|
|
|
#define DEFAULT_ZERO_SIZE_IS_UNSCALED TRUE
|
2014-05-22 17:46:02 +00:00
|
|
|
enum
|
|
|
|
{
|
|
|
|
PROP_0,
|
2017-07-06 18:26:21 +00:00
|
|
|
PROP_BACKGROUND,
|
2020-11-07 01:59:09 +00:00
|
|
|
PROP_ZERO_SIZE_IS_UNSCALED,
|
2014-05-22 17:46:02 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_get_property (GObject * object,
|
|
|
|
guint prop_id, GValue * value, GParamSpec * pspec)
|
|
|
|
{
|
|
|
|
GstCompositor *self = GST_COMPOSITOR (object);
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
case PROP_BACKGROUND:
|
|
|
|
g_value_set_enum (value, self->background);
|
|
|
|
break;
|
2020-11-07 01:59:09 +00:00
|
|
|
case PROP_ZERO_SIZE_IS_UNSCALED:
|
|
|
|
g_value_set_boolean (value, self->zero_size_is_unscaled);
|
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
default:
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_set_property (GObject * object,
|
|
|
|
guint prop_id, const GValue * value, GParamSpec * pspec)
|
|
|
|
{
|
|
|
|
GstCompositor *self = GST_COMPOSITOR (object);
|
|
|
|
|
|
|
|
switch (prop_id) {
|
|
|
|
case PROP_BACKGROUND:
|
|
|
|
self->background = g_value_get_enum (value);
|
|
|
|
break;
|
2020-11-07 01:59:09 +00:00
|
|
|
case PROP_ZERO_SIZE_IS_UNSCALED:
|
|
|
|
self->zero_size_is_unscaled = g_value_get_boolean (value);
|
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
default:
|
|
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#define gst_compositor_parent_class parent_class
|
2018-05-04 14:13:16 +00:00
|
|
|
G_DEFINE_TYPE_WITH_CODE (GstCompositor, gst_compositor,
|
|
|
|
GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY,
|
|
|
|
gst_compositor_child_proxy_init));
|
2014-05-22 17:46:02 +00:00
|
|
|
|
|
|
|
static gboolean
|
|
|
|
set_functions (GstCompositor * self, GstVideoInfo * info)
|
|
|
|
{
|
|
|
|
gboolean ret = FALSE;
|
|
|
|
|
|
|
|
self->blend = NULL;
|
|
|
|
self->overlay = NULL;
|
|
|
|
self->fill_checker = NULL;
|
|
|
|
self->fill_color = NULL;
|
|
|
|
|
|
|
|
switch (GST_VIDEO_INFO_FORMAT (info)) {
|
|
|
|
case GST_VIDEO_FORMAT_AYUV:
|
|
|
|
self->blend = gst_compositor_blend_ayuv;
|
|
|
|
self->overlay = gst_compositor_overlay_ayuv;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_ayuv;
|
|
|
|
self->fill_color = gst_compositor_fill_color_ayuv;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
2019-08-20 14:12:22 +00:00
|
|
|
case GST_VIDEO_FORMAT_VUYA:
|
|
|
|
self->blend = gst_compositor_blend_vuya;
|
|
|
|
self->overlay = gst_compositor_overlay_vuya;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_vuya;
|
|
|
|
self->fill_color = gst_compositor_fill_color_vuya;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
2014-05-22 17:46:02 +00:00
|
|
|
case GST_VIDEO_FORMAT_ARGB:
|
|
|
|
self->blend = gst_compositor_blend_argb;
|
|
|
|
self->overlay = gst_compositor_overlay_argb;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_argb;
|
|
|
|
self->fill_color = gst_compositor_fill_color_argb;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_BGRA:
|
|
|
|
self->blend = gst_compositor_blend_bgra;
|
|
|
|
self->overlay = gst_compositor_overlay_bgra;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_bgra;
|
|
|
|
self->fill_color = gst_compositor_fill_color_bgra;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_ABGR:
|
|
|
|
self->blend = gst_compositor_blend_abgr;
|
|
|
|
self->overlay = gst_compositor_overlay_abgr;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_abgr;
|
|
|
|
self->fill_color = gst_compositor_fill_color_abgr;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_RGBA:
|
|
|
|
self->blend = gst_compositor_blend_rgba;
|
|
|
|
self->overlay = gst_compositor_overlay_rgba;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_rgba;
|
|
|
|
self->fill_color = gst_compositor_fill_color_rgba;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_Y444:
|
|
|
|
self->blend = gst_compositor_blend_y444;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_y444;
|
|
|
|
self->fill_color = gst_compositor_fill_color_y444;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_Y42B:
|
|
|
|
self->blend = gst_compositor_blend_y42b;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_y42b;
|
|
|
|
self->fill_color = gst_compositor_fill_color_y42b;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_YUY2:
|
|
|
|
self->blend = gst_compositor_blend_yuy2;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_yuy2;
|
|
|
|
self->fill_color = gst_compositor_fill_color_yuy2;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_UYVY:
|
|
|
|
self->blend = gst_compositor_blend_uyvy;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_uyvy;
|
|
|
|
self->fill_color = gst_compositor_fill_color_uyvy;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_YVYU:
|
|
|
|
self->blend = gst_compositor_blend_yvyu;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_yvyu;
|
|
|
|
self->fill_color = gst_compositor_fill_color_yvyu;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_I420:
|
|
|
|
self->blend = gst_compositor_blend_i420;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_i420;
|
|
|
|
self->fill_color = gst_compositor_fill_color_i420;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_YV12:
|
|
|
|
self->blend = gst_compositor_blend_yv12;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_yv12;
|
|
|
|
self->fill_color = gst_compositor_fill_color_yv12;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_NV12:
|
|
|
|
self->blend = gst_compositor_blend_nv12;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_nv12;
|
|
|
|
self->fill_color = gst_compositor_fill_color_nv12;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_NV21:
|
|
|
|
self->blend = gst_compositor_blend_nv21;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_nv21;
|
|
|
|
self->fill_color = gst_compositor_fill_color_nv21;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_Y41B:
|
|
|
|
self->blend = gst_compositor_blend_y41b;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_y41b;
|
|
|
|
self->fill_color = gst_compositor_fill_color_y41b;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_RGB:
|
|
|
|
self->blend = gst_compositor_blend_rgb;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_rgb;
|
|
|
|
self->fill_color = gst_compositor_fill_color_rgb;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_BGR:
|
|
|
|
self->blend = gst_compositor_blend_bgr;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_bgr;
|
|
|
|
self->fill_color = gst_compositor_fill_color_bgr;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_xRGB:
|
|
|
|
self->blend = gst_compositor_blend_xrgb;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_xrgb;
|
|
|
|
self->fill_color = gst_compositor_fill_color_xrgb;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_xBGR:
|
|
|
|
self->blend = gst_compositor_blend_xbgr;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_xbgr;
|
|
|
|
self->fill_color = gst_compositor_fill_color_xbgr;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_RGBx:
|
|
|
|
self->blend = gst_compositor_blend_rgbx;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_rgbx;
|
|
|
|
self->fill_color = gst_compositor_fill_color_rgbx;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
case GST_VIDEO_FORMAT_BGRx:
|
|
|
|
self->blend = gst_compositor_blend_bgrx;
|
|
|
|
self->overlay = self->blend;
|
|
|
|
self->fill_checker = gst_compositor_fill_checker_bgrx;
|
|
|
|
self->fill_color = gst_compositor_fill_color_bgrx;
|
|
|
|
ret = TRUE;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-10-19 23:34:27 +00:00
|
|
|
static GstCaps *
|
2017-05-20 12:24:57 +00:00
|
|
|
_fixate_caps (GstAggregator * agg, GstCaps * caps)
|
2014-05-22 17:46:02 +00:00
|
|
|
{
|
2017-05-20 12:24:57 +00:00
|
|
|
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg);
|
2014-05-22 17:46:02 +00:00
|
|
|
GList *l;
|
|
|
|
gint best_width = -1, best_height = -1;
|
2015-10-14 10:13:57 +00:00
|
|
|
gint best_fps_n = -1, best_fps_d = -1;
|
|
|
|
gint par_n, par_d;
|
|
|
|
gdouble best_fps = 0.;
|
2014-10-19 23:34:27 +00:00
|
|
|
GstCaps *ret = NULL;
|
2015-10-14 10:13:57 +00:00
|
|
|
GstStructure *s;
|
2014-10-19 23:34:27 +00:00
|
|
|
|
2015-10-14 10:13:57 +00:00
|
|
|
ret = gst_caps_make_writable (caps);
|
2014-05-22 17:46:02 +00:00
|
|
|
|
2015-10-14 10:13:57 +00:00
|
|
|
/* we need this to calculate how large to make the output frame */
|
|
|
|
s = gst_caps_get_structure (ret, 0);
|
|
|
|
if (gst_structure_has_field (s, "pixel-aspect-ratio")) {
|
|
|
|
gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1);
|
|
|
|
gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d);
|
|
|
|
} else {
|
|
|
|
par_n = par_d = 1;
|
|
|
|
}
|
2015-05-14 03:04:21 +00:00
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
GST_OBJECT_LOCK (vagg);
|
|
|
|
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
|
|
|
|
GstVideoAggregatorPad *vaggpad = l->data;
|
|
|
|
GstCompositorPad *compositor_pad = GST_COMPOSITOR_PAD (vaggpad);
|
|
|
|
gint this_width, this_height;
|
|
|
|
gint width, height;
|
2015-10-14 10:13:57 +00:00
|
|
|
gint fps_n, fps_d;
|
|
|
|
gdouble cur_fps;
|
2014-05-22 17:46:02 +00:00
|
|
|
|
2015-10-14 10:13:57 +00:00
|
|
|
fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info);
|
|
|
|
fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info);
|
2020-11-07 01:59:09 +00:00
|
|
|
_mixer_pad_get_output_size (GST_COMPOSITOR (vagg), compositor_pad, par_n,
|
|
|
|
par_d, &width, &height);
|
2014-05-22 17:46:02 +00:00
|
|
|
|
|
|
|
if (width == 0 || height == 0)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
this_width = width + MAX (compositor_pad->xpos, 0);
|
|
|
|
this_height = height + MAX (compositor_pad->ypos, 0);
|
|
|
|
|
|
|
|
if (best_width < this_width)
|
|
|
|
best_width = this_width;
|
|
|
|
if (best_height < this_height)
|
|
|
|
best_height = this_height;
|
2015-10-14 10:13:57 +00:00
|
|
|
|
|
|
|
if (fps_d == 0)
|
|
|
|
cur_fps = 0.0;
|
|
|
|
else
|
|
|
|
gst_util_fraction_to_double (fps_n, fps_d, &cur_fps);
|
|
|
|
|
|
|
|
if (best_fps < cur_fps) {
|
|
|
|
best_fps = cur_fps;
|
|
|
|
best_fps_n = fps_n;
|
|
|
|
best_fps_d = fps_d;
|
|
|
|
}
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
GST_OBJECT_UNLOCK (vagg);
|
|
|
|
|
2015-10-14 10:13:57 +00:00
|
|
|
if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) {
|
|
|
|
best_fps_n = 25;
|
|
|
|
best_fps_d = 1;
|
|
|
|
best_fps = 25.0;
|
|
|
|
}
|
|
|
|
|
|
|
|
gst_structure_fixate_field_nearest_int (s, "width", best_width);
|
|
|
|
gst_structure_fixate_field_nearest_int (s, "height", best_height);
|
|
|
|
gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n,
|
|
|
|
best_fps_d);
|
|
|
|
ret = gst_caps_fixate (ret);
|
|
|
|
|
2016-05-03 08:11:24 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2015-05-14 03:04:21 +00:00
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
static gpointer
|
|
|
|
gst_parallelized_task_thread_func (gpointer data)
|
|
|
|
{
|
|
|
|
GstParallelizedTaskThread *self = data;
|
|
|
|
|
|
|
|
g_mutex_lock (&self->runner->lock);
|
|
|
|
self->runner->n_done++;
|
|
|
|
if (self->runner->n_done == self->runner->n_threads - 1)
|
|
|
|
g_cond_signal (&self->runner->cond_done);
|
|
|
|
|
|
|
|
do {
|
|
|
|
gint idx;
|
|
|
|
|
|
|
|
while (self->runner->n_todo == -1 && !self->runner->quit)
|
|
|
|
g_cond_wait (&self->runner->cond_todo, &self->runner->lock);
|
|
|
|
|
|
|
|
if (self->runner->quit)
|
|
|
|
break;
|
|
|
|
|
|
|
|
idx = self->runner->n_todo--;
|
|
|
|
g_assert (self->runner->n_todo >= -1);
|
|
|
|
g_mutex_unlock (&self->runner->lock);
|
|
|
|
|
|
|
|
g_assert (self->runner->func != NULL);
|
|
|
|
|
|
|
|
self->runner->func (self->runner->task_data[idx]);
|
|
|
|
|
|
|
|
g_mutex_lock (&self->runner->lock);
|
|
|
|
self->runner->n_done++;
|
|
|
|
if (self->runner->n_done == self->runner->n_threads - 1)
|
|
|
|
g_cond_signal (&self->runner->cond_done);
|
|
|
|
} while (TRUE);
|
|
|
|
|
|
|
|
g_mutex_unlock (&self->runner->lock);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_parallelized_task_runner_free (GstParallelizedTaskRunner * self)
|
|
|
|
{
|
|
|
|
guint i;
|
|
|
|
|
|
|
|
g_mutex_lock (&self->lock);
|
|
|
|
self->quit = TRUE;
|
|
|
|
g_cond_broadcast (&self->cond_todo);
|
|
|
|
g_mutex_unlock (&self->lock);
|
|
|
|
|
|
|
|
for (i = 1; i < self->n_threads; i++) {
|
|
|
|
if (!self->threads[i].thread)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
g_thread_join (self->threads[i].thread);
|
|
|
|
}
|
|
|
|
|
|
|
|
g_mutex_clear (&self->lock);
|
|
|
|
g_cond_clear (&self->cond_todo);
|
|
|
|
g_cond_clear (&self->cond_done);
|
|
|
|
g_free (self->threads);
|
|
|
|
g_free (self);
|
|
|
|
}
|
|
|
|
|
|
|
|
static GstParallelizedTaskRunner *
|
|
|
|
gst_parallelized_task_runner_new (guint n_threads)
|
|
|
|
{
|
|
|
|
GstParallelizedTaskRunner *self;
|
|
|
|
guint i;
|
|
|
|
GError *err = NULL;
|
|
|
|
|
|
|
|
if (n_threads == 0)
|
|
|
|
n_threads = g_get_num_processors ();
|
|
|
|
|
|
|
|
self = g_new0 (GstParallelizedTaskRunner, 1);
|
|
|
|
self->n_threads = n_threads;
|
|
|
|
self->threads = g_new0 (GstParallelizedTaskThread, n_threads);
|
|
|
|
|
|
|
|
self->quit = FALSE;
|
|
|
|
self->n_todo = -1;
|
|
|
|
self->n_done = 0;
|
|
|
|
g_mutex_init (&self->lock);
|
|
|
|
g_cond_init (&self->cond_todo);
|
|
|
|
g_cond_init (&self->cond_done);
|
|
|
|
|
|
|
|
/* Set when scheduling a job */
|
|
|
|
self->func = NULL;
|
|
|
|
self->task_data = NULL;
|
|
|
|
|
|
|
|
for (i = 0; i < n_threads; i++) {
|
|
|
|
self->threads[i].runner = self;
|
|
|
|
self->threads[i].idx = i;
|
|
|
|
|
|
|
|
/* First thread is the one calling run() */
|
|
|
|
if (i > 0) {
|
|
|
|
self->threads[i].thread =
|
|
|
|
g_thread_try_new ("compositor-blend",
|
|
|
|
gst_parallelized_task_thread_func, &self->threads[i], &err);
|
|
|
|
if (!self->threads[i].thread)
|
|
|
|
goto error;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
g_mutex_lock (&self->lock);
|
|
|
|
while (self->n_done < self->n_threads - 1)
|
|
|
|
g_cond_wait (&self->cond_done, &self->lock);
|
|
|
|
self->n_done = 0;
|
|
|
|
g_mutex_unlock (&self->lock);
|
|
|
|
|
|
|
|
return self;
|
|
|
|
|
|
|
|
error:
|
|
|
|
{
|
|
|
|
GST_ERROR ("Failed to start thread %u: %s", i, err->message);
|
|
|
|
g_clear_error (&err);
|
|
|
|
|
|
|
|
gst_parallelized_task_runner_free (self);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_parallelized_task_runner_run (GstParallelizedTaskRunner * self,
|
|
|
|
GstParallelizedTaskFunc func, gpointer * task_data)
|
|
|
|
{
|
|
|
|
guint n_threads = self->n_threads;
|
|
|
|
|
|
|
|
self->func = func;
|
|
|
|
self->task_data = task_data;
|
|
|
|
|
|
|
|
if (n_threads > 1) {
|
|
|
|
g_mutex_lock (&self->lock);
|
|
|
|
self->n_todo = self->n_threads - 2;
|
|
|
|
self->n_done = 0;
|
|
|
|
g_cond_broadcast (&self->cond_todo);
|
|
|
|
g_mutex_unlock (&self->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
self->func (self->task_data[self->n_threads - 1]);
|
|
|
|
|
|
|
|
if (n_threads > 1) {
|
|
|
|
g_mutex_lock (&self->lock);
|
|
|
|
while (self->n_done < self->n_threads - 1)
|
|
|
|
g_cond_wait (&self->cond_done, &self->lock);
|
|
|
|
self->n_done = 0;
|
|
|
|
g_mutex_unlock (&self->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
self->func = NULL;
|
|
|
|
self->task_data = NULL;
|
|
|
|
}
|
|
|
|
|
2016-05-03 08:11:24 +00:00
|
|
|
static gboolean
|
2017-05-20 12:24:57 +00:00
|
|
|
_negotiated_caps (GstAggregator * agg, GstCaps * caps)
|
2016-05-03 08:11:24 +00:00
|
|
|
{
|
2020-07-15 05:46:56 +00:00
|
|
|
GstCompositor *compositor = GST_COMPOSITOR (agg);
|
2016-05-03 08:11:24 +00:00
|
|
|
GstVideoInfo v_info;
|
2020-07-15 05:46:56 +00:00
|
|
|
guint n_threads;
|
2016-05-03 08:11:24 +00:00
|
|
|
|
2017-05-20 12:24:57 +00:00
|
|
|
GST_DEBUG_OBJECT (agg, "Negotiated caps %" GST_PTR_FORMAT, caps);
|
2016-05-03 08:11:24 +00:00
|
|
|
|
|
|
|
if (!gst_video_info_from_caps (&v_info, caps))
|
|
|
|
return FALSE;
|
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
if (!set_functions (compositor, &v_info)) {
|
2017-05-20 12:24:57 +00:00
|
|
|
GST_ERROR_OBJECT (agg, "Failed to setup vfuncs");
|
2016-05-03 08:11:24 +00:00
|
|
|
return FALSE;
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
n_threads = g_get_num_processors ();
|
|
|
|
/* Magic number of 200 lines */
|
|
|
|
if (GST_VIDEO_INFO_HEIGHT (&v_info) / n_threads < 200)
|
|
|
|
n_threads = (GST_VIDEO_INFO_HEIGHT (&v_info) + 199) / 200;
|
|
|
|
if (n_threads < 1)
|
|
|
|
n_threads = 1;
|
|
|
|
|
|
|
|
/* XXX: implement better thread count change */
|
|
|
|
if (compositor->blend_runner
|
|
|
|
&& compositor->blend_runner->n_threads != n_threads) {
|
|
|
|
gst_parallelized_task_runner_free (compositor->blend_runner);
|
|
|
|
compositor->blend_runner = NULL;
|
|
|
|
}
|
|
|
|
if (!compositor->blend_runner)
|
|
|
|
compositor->blend_runner = gst_parallelized_task_runner_new (n_threads);
|
|
|
|
|
2017-05-20 12:24:57 +00:00
|
|
|
return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps);
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
2019-05-26 15:30:12 +00:00
|
|
|
static gboolean
|
2020-07-17 01:56:47 +00:00
|
|
|
_should_draw_background (GstVideoAggregator * vagg)
|
2014-05-22 17:46:02 +00:00
|
|
|
{
|
2019-05-26 15:30:12 +00:00
|
|
|
GstVideoRectangle bg_rect;
|
|
|
|
gboolean draw = TRUE;
|
2019-06-24 09:44:29 +00:00
|
|
|
GList *l;
|
|
|
|
|
2019-05-26 15:30:12 +00:00
|
|
|
bg_rect.x = bg_rect.y = 0;
|
2014-05-22 17:46:02 +00:00
|
|
|
|
2019-05-26 15:30:12 +00:00
|
|
|
GST_OBJECT_LOCK (vagg);
|
|
|
|
bg_rect.w = GST_VIDEO_INFO_WIDTH (&vagg->info);
|
|
|
|
bg_rect.h = GST_VIDEO_INFO_HEIGHT (&vagg->info);
|
|
|
|
/* Check if the background is completely obscured by a pad
|
|
|
|
* TODO: Also skip if it's obscured by a combination of pads */
|
2019-06-24 09:44:29 +00:00
|
|
|
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
|
2020-07-17 01:56:47 +00:00
|
|
|
if (_pad_obscures_rectangle (vagg, l->data, bg_rect)) {
|
2019-05-26 15:30:12 +00:00
|
|
|
draw = FALSE;
|
|
|
|
break;
|
|
|
|
}
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
2019-05-26 15:30:12 +00:00
|
|
|
GST_OBJECT_UNLOCK (vagg);
|
|
|
|
return draw;
|
|
|
|
}
|
2014-05-22 17:46:02 +00:00
|
|
|
|
2019-06-13 21:02:50 +00:00
|
|
|
static gboolean
|
2020-07-15 05:46:56 +00:00
|
|
|
frames_can_copy (const GstVideoFrame * frame1, const GstVideoFrame * frame2)
|
2019-05-26 15:30:12 +00:00
|
|
|
{
|
2020-07-15 05:46:56 +00:00
|
|
|
if (GST_VIDEO_FRAME_FORMAT (frame1) != GST_VIDEO_FRAME_FORMAT (frame2))
|
|
|
|
return FALSE;
|
|
|
|
if (GST_VIDEO_FRAME_HEIGHT (frame1) != GST_VIDEO_FRAME_HEIGHT (frame2))
|
|
|
|
return FALSE;
|
|
|
|
if (GST_VIDEO_FRAME_WIDTH (frame1) != GST_VIDEO_FRAME_WIDTH (frame2))
|
|
|
|
return FALSE;
|
|
|
|
return TRUE;
|
|
|
|
}
|
2019-05-26 15:30:12 +00:00
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
struct CompositePadInfo
|
|
|
|
{
|
|
|
|
GstVideoFrame *prepared_frame;
|
|
|
|
GstCompositorPad *pad;
|
|
|
|
GstCompositorBlendMode blend_mode;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct CompositeTask
|
|
|
|
{
|
|
|
|
GstCompositor *compositor;
|
|
|
|
GstVideoFrame *out_frame;
|
|
|
|
guint dst_line_start;
|
|
|
|
guint dst_line_end;
|
|
|
|
gboolean draw_background;
|
|
|
|
guint n_pads;
|
|
|
|
struct CompositePadInfo *pads_info;
|
|
|
|
};
|
|
|
|
|
|
|
|
static void
|
|
|
|
_draw_background (GstCompositor * comp, GstVideoFrame * outframe,
|
|
|
|
guint y_start, guint y_end, BlendFunction * composite)
|
|
|
|
{
|
2019-06-13 21:02:50 +00:00
|
|
|
*composite = comp->blend;
|
2019-05-26 15:30:12 +00:00
|
|
|
|
|
|
|
switch (comp->background) {
|
2014-05-22 17:46:02 +00:00
|
|
|
case COMPOSITOR_BACKGROUND_CHECKER:
|
2020-07-15 05:46:56 +00:00
|
|
|
comp->fill_checker (outframe, y_start, y_end);
|
2014-05-22 17:46:02 +00:00
|
|
|
break;
|
|
|
|
case COMPOSITOR_BACKGROUND_BLACK:
|
2020-07-15 05:46:56 +00:00
|
|
|
comp->fill_color (outframe, y_start, y_end, 16, 128, 128);
|
2014-05-22 17:46:02 +00:00
|
|
|
break;
|
|
|
|
case COMPOSITOR_BACKGROUND_WHITE:
|
2020-07-15 05:46:56 +00:00
|
|
|
comp->fill_color (outframe, y_start, y_end, 240, 128, 128);
|
2014-05-22 17:46:02 +00:00
|
|
|
break;
|
|
|
|
case COMPOSITOR_BACKGROUND_TRANSPARENT:
|
2018-10-27 15:44:27 +00:00
|
|
|
{
|
|
|
|
guint i, plane, num_planes, height;
|
|
|
|
|
|
|
|
num_planes = GST_VIDEO_FRAME_N_PLANES (outframe);
|
|
|
|
for (plane = 0; plane < num_planes; ++plane) {
|
2020-07-15 05:46:56 +00:00
|
|
|
const GstVideoFormatInfo *info;
|
2018-10-27 15:44:27 +00:00
|
|
|
guint8 *pdata;
|
|
|
|
gsize rowsize, plane_stride;
|
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
info = outframe->info.finfo;
|
2018-10-27 15:44:27 +00:00
|
|
|
pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane);
|
|
|
|
plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane);
|
|
|
|
rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane)
|
|
|
|
* GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane);
|
2020-07-15 05:46:56 +00:00
|
|
|
height =
|
|
|
|
GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, plane, y_end - y_start);
|
|
|
|
pdata += y_start * plane_stride;
|
2018-10-27 15:44:27 +00:00
|
|
|
for (i = 0; i < height; ++i) {
|
|
|
|
memset (pdata, 0, rowsize);
|
|
|
|
pdata += plane_stride;
|
|
|
|
}
|
|
|
|
}
|
2014-05-22 17:46:02 +00:00
|
|
|
/* use overlay to keep background transparent */
|
2019-06-13 21:02:50 +00:00
|
|
|
*composite = comp->overlay;
|
2014-05-22 17:46:02 +00:00
|
|
|
break;
|
2018-10-27 15:44:27 +00:00
|
|
|
}
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
2019-06-13 21:02:50 +00:00
|
|
|
}
|
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
static void
|
|
|
|
blend_pads (struct CompositeTask *comp)
|
2019-06-13 21:02:50 +00:00
|
|
|
{
|
2020-07-15 05:46:56 +00:00
|
|
|
BlendFunction composite;
|
|
|
|
guint i;
|
|
|
|
|
|
|
|
composite = comp->compositor->blend;
|
|
|
|
|
|
|
|
if (comp->draw_background) {
|
|
|
|
_draw_background (comp->compositor, comp->out_frame, comp->dst_line_start,
|
|
|
|
comp->dst_line_end, &composite);
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i < comp->n_pads; i++) {
|
|
|
|
composite (comp->pads_info[i].prepared_frame,
|
|
|
|
comp->pads_info[i].pad->xpos, comp->pads_info[i].pad->ypos,
|
|
|
|
comp->pads_info[i].pad->alpha, comp->out_frame, comp->dst_line_start,
|
|
|
|
comp->dst_line_end, comp->pads_info[i].blend_mode);
|
|
|
|
}
|
2019-05-26 15:30:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static GstFlowReturn
|
|
|
|
gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
|
|
|
|
{
|
2020-07-15 05:46:56 +00:00
|
|
|
GstCompositor *compositor = GST_COMPOSITOR (vagg);
|
2019-05-26 15:30:12 +00:00
|
|
|
GList *l;
|
|
|
|
GstVideoFrame out_frame, *outframe;
|
2020-07-15 05:46:56 +00:00
|
|
|
gboolean draw_background;
|
|
|
|
guint drawn_a_pad = FALSE;
|
|
|
|
struct CompositePadInfo *pads_info;
|
|
|
|
guint i, n_pads = 0;
|
2019-05-26 15:30:12 +00:00
|
|
|
|
|
|
|
if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, GST_MAP_WRITE)) {
|
|
|
|
GST_WARNING_OBJECT (vagg, "Could not map output buffer");
|
|
|
|
return GST_FLOW_ERROR;
|
|
|
|
}
|
|
|
|
|
|
|
|
outframe = &out_frame;
|
2020-07-15 05:46:56 +00:00
|
|
|
|
|
|
|
/* If one of the frames to be composited completely obscures the background,
|
|
|
|
* don't bother drawing the background at all. We can also always use the
|
|
|
|
* 'blend' BlendFunction in that case because it only changes if we have to
|
|
|
|
* overlay on top of a transparent background. */
|
|
|
|
draw_background = _should_draw_background (vagg);
|
2019-05-26 15:30:12 +00:00
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
GST_OBJECT_LOCK (vagg);
|
2020-07-15 05:46:56 +00:00
|
|
|
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
|
|
|
|
GstVideoAggregatorPad *pad = l->data;
|
|
|
|
GstVideoFrame *prepared_frame =
|
|
|
|
gst_video_aggregator_pad_get_prepared_frame (pad);
|
|
|
|
|
|
|
|
if (prepared_frame)
|
|
|
|
n_pads++;
|
|
|
|
}
|
|
|
|
|
|
|
|
pads_info = g_newa (struct CompositePadInfo, n_pads);
|
|
|
|
n_pads = 0;
|
|
|
|
|
2018-10-27 15:44:27 +00:00
|
|
|
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
|
|
|
|
GstVideoAggregatorPad *pad = l->data;
|
|
|
|
GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad);
|
|
|
|
GstVideoFrame *prepared_frame =
|
|
|
|
gst_video_aggregator_pad_get_prepared_frame (pad);
|
|
|
|
GstCompositorBlendMode blend_mode = COMPOSITOR_BLEND_MODE_OVER;
|
|
|
|
|
|
|
|
switch (compo_pad->op) {
|
|
|
|
case COMPOSITOR_OPERATOR_SOURCE:
|
2018-10-31 18:20:20 +00:00
|
|
|
blend_mode = COMPOSITOR_BLEND_MODE_SOURCE;
|
2018-10-27 15:44:27 +00:00
|
|
|
break;
|
|
|
|
case COMPOSITOR_OPERATOR_OVER:
|
2018-10-31 18:20:20 +00:00
|
|
|
blend_mode = COMPOSITOR_BLEND_MODE_OVER;
|
2018-10-27 15:44:27 +00:00
|
|
|
break;
|
|
|
|
case COMPOSITOR_OPERATOR_ADD:
|
2018-10-31 18:20:20 +00:00
|
|
|
blend_mode = COMPOSITOR_BLEND_MODE_ADD;
|
2018-10-27 15:44:27 +00:00
|
|
|
break;
|
|
|
|
default:
|
|
|
|
g_assert_not_reached ();
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (prepared_frame != NULL) {
|
2019-06-13 21:02:50 +00:00
|
|
|
/* If this is the first pad we're drawing, and we didn't draw the
|
|
|
|
* background, and @prepared_frame has the same format, height, and width
|
|
|
|
* as @outframe, then we can just copy it as-is. Subsequent pads (if any)
|
|
|
|
* will be composited on top of it. */
|
2020-07-15 05:46:56 +00:00
|
|
|
if (!drawn_a_pad && !draw_background &&
|
|
|
|
frames_can_copy (prepared_frame, outframe)) {
|
2019-06-13 21:02:50 +00:00
|
|
|
gst_video_frame_copy (outframe, prepared_frame);
|
2020-07-15 05:46:56 +00:00
|
|
|
} else {
|
|
|
|
pads_info[n_pads].pad = compo_pad;
|
|
|
|
pads_info[n_pads].prepared_frame = prepared_frame;
|
|
|
|
pads_info[n_pads].blend_mode = blend_mode;
|
|
|
|
n_pads++;
|
|
|
|
}
|
|
|
|
drawn_a_pad = TRUE;
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
}
|
2020-07-15 05:46:56 +00:00
|
|
|
|
|
|
|
{
|
|
|
|
guint n_threads, lines_per_thread;
|
|
|
|
guint out_height;
|
|
|
|
struct CompositeTask *tasks;
|
|
|
|
struct CompositeTask **tasks_p;
|
|
|
|
|
|
|
|
n_threads = compositor->blend_runner->n_threads;
|
|
|
|
|
|
|
|
tasks = g_newa (struct CompositeTask, n_threads);
|
|
|
|
tasks_p = g_newa (struct CompositeTask *, n_threads);
|
|
|
|
|
|
|
|
out_height = GST_VIDEO_FRAME_HEIGHT (outframe);
|
|
|
|
lines_per_thread = (out_height + n_threads - 1) / n_threads;
|
|
|
|
|
|
|
|
for (i = 0; i < n_threads; i++) {
|
|
|
|
tasks[i].compositor = compositor;
|
|
|
|
tasks[i].n_pads = n_pads;
|
|
|
|
tasks[i].pads_info = pads_info;
|
|
|
|
tasks[i].out_frame = outframe;
|
|
|
|
tasks[i].draw_background = draw_background;
|
|
|
|
/* This is a dumb split of the work by number of output lines.
|
|
|
|
* If there is a section of the output that reads from a lot of source
|
|
|
|
* pads, then that thread will consume more time. Maybe tracking and
|
|
|
|
* splitting on the source fill rate would produce better results. */
|
|
|
|
tasks[i].dst_line_start = i * lines_per_thread;
|
|
|
|
tasks[i].dst_line_end = MIN ((i + 1) * lines_per_thread, out_height);
|
|
|
|
|
|
|
|
tasks_p[i] = &tasks[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
gst_parallelized_task_runner_run (compositor->blend_runner,
|
|
|
|
(GstParallelizedTaskFunc) blend_pads, (gpointer *) tasks_p);
|
|
|
|
}
|
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
GST_OBJECT_UNLOCK (vagg);
|
|
|
|
|
|
|
|
gst_video_frame_unmap (outframe);
|
|
|
|
|
|
|
|
return GST_FLOW_OK;
|
|
|
|
}
|
|
|
|
|
2018-05-04 14:13:16 +00:00
|
|
|
static GstPad *
|
|
|
|
gst_compositor_request_new_pad (GstElement * element, GstPadTemplate * templ,
|
|
|
|
const gchar * req_name, const GstCaps * caps)
|
|
|
|
{
|
|
|
|
GstPad *newpad;
|
|
|
|
|
|
|
|
newpad = (GstPad *)
|
|
|
|
GST_ELEMENT_CLASS (parent_class)->request_new_pad (element,
|
|
|
|
templ, req_name, caps);
|
|
|
|
|
|
|
|
if (newpad == NULL)
|
|
|
|
goto could_not_create;
|
|
|
|
|
|
|
|
gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad),
|
|
|
|
GST_OBJECT_NAME (newpad));
|
|
|
|
|
|
|
|
return newpad;
|
|
|
|
|
|
|
|
could_not_create:
|
|
|
|
{
|
|
|
|
GST_DEBUG_OBJECT (element, "could not create/add pad");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_release_pad (GstElement * element, GstPad * pad)
|
|
|
|
{
|
|
|
|
GstCompositor *compositor;
|
|
|
|
|
|
|
|
compositor = GST_COMPOSITOR (element);
|
|
|
|
|
|
|
|
GST_DEBUG_OBJECT (compositor, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad));
|
|
|
|
|
|
|
|
gst_child_proxy_child_removed (GST_CHILD_PROXY (compositor), G_OBJECT (pad),
|
|
|
|
GST_OBJECT_NAME (pad));
|
|
|
|
|
|
|
|
GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad);
|
|
|
|
}
|
|
|
|
|
2014-11-25 17:53:55 +00:00
|
|
|
static gboolean
|
|
|
|
_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query)
|
|
|
|
{
|
|
|
|
switch (GST_QUERY_TYPE (query)) {
|
|
|
|
case GST_QUERY_ALLOCATION:{
|
|
|
|
GstCaps *caps;
|
|
|
|
GstVideoInfo info;
|
|
|
|
GstBufferPool *pool;
|
|
|
|
guint size;
|
|
|
|
GstStructure *structure;
|
|
|
|
|
|
|
|
gst_query_parse_allocation (query, &caps, NULL);
|
|
|
|
|
|
|
|
if (caps == NULL)
|
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
if (!gst_video_info_from_caps (&info, caps))
|
|
|
|
return FALSE;
|
|
|
|
|
|
|
|
size = GST_VIDEO_INFO_SIZE (&info);
|
|
|
|
|
|
|
|
pool = gst_video_buffer_pool_new ();
|
|
|
|
|
|
|
|
structure = gst_buffer_pool_get_config (pool);
|
|
|
|
gst_buffer_pool_config_set_params (structure, caps, size, 0, 0);
|
|
|
|
|
|
|
|
if (!gst_buffer_pool_set_config (pool, structure)) {
|
|
|
|
gst_object_unref (pool);
|
|
|
|
return FALSE;
|
|
|
|
}
|
|
|
|
|
|
|
|
gst_query_add_allocation_pool (query, pool, size, 0, 0);
|
|
|
|
gst_object_unref (pool);
|
|
|
|
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL);
|
|
|
|
|
|
|
|
return TRUE;
|
|
|
|
}
|
|
|
|
default:
|
|
|
|
return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-15 05:46:56 +00:00
|
|
|
static void
|
|
|
|
gst_compositor_finalize (GObject * object)
|
|
|
|
{
|
|
|
|
GstCompositor *compositor = GST_COMPOSITOR (object);
|
|
|
|
|
|
|
|
if (compositor->blend_runner)
|
|
|
|
gst_parallelized_task_runner_free (compositor->blend_runner);
|
|
|
|
compositor->blend_runner = NULL;
|
|
|
|
|
|
|
|
G_OBJECT_CLASS (parent_class)->finalize (object);
|
|
|
|
}
|
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
/* GObject boilerplate */
|
|
|
|
static void
|
|
|
|
gst_compositor_class_init (GstCompositorClass * klass)
|
|
|
|
{
|
|
|
|
GObjectClass *gobject_class = (GObjectClass *) klass;
|
|
|
|
GstElementClass *gstelement_class = (GstElementClass *) klass;
|
|
|
|
GstVideoAggregatorClass *videoaggregator_class =
|
|
|
|
(GstVideoAggregatorClass *) klass;
|
|
|
|
GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;
|
|
|
|
|
|
|
|
gobject_class->get_property = gst_compositor_get_property;
|
|
|
|
gobject_class->set_property = gst_compositor_set_property;
|
2020-07-15 05:46:56 +00:00
|
|
|
gobject_class->finalize = gst_compositor_finalize;
|
2014-05-22 17:46:02 +00:00
|
|
|
|
2018-05-04 14:13:16 +00:00
|
|
|
gstelement_class->request_new_pad =
|
|
|
|
GST_DEBUG_FUNCPTR (gst_compositor_request_new_pad);
|
|
|
|
gstelement_class->release_pad =
|
|
|
|
GST_DEBUG_FUNCPTR (gst_compositor_release_pad);
|
2014-11-25 17:53:55 +00:00
|
|
|
agg_class->sink_query = _sink_query;
|
2017-05-20 12:24:57 +00:00
|
|
|
agg_class->fixate_src_caps = _fixate_caps;
|
|
|
|
agg_class->negotiated_src_caps = _negotiated_caps;
|
2014-05-22 17:46:02 +00:00
|
|
|
videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames;
|
|
|
|
|
|
|
|
g_object_class_install_property (gobject_class, PROP_BACKGROUND,
|
|
|
|
g_param_spec_enum ("background", "Background", "Background type",
|
|
|
|
GST_TYPE_COMPOSITOR_BACKGROUND,
|
|
|
|
DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
|
2020-11-07 01:59:09 +00:00
|
|
|
/**
|
|
|
|
* compositor:zero-size-is-unscaled:
|
|
|
|
*
|
|
|
|
* Whether a pad with height or width 0 should be left unscaled
|
|
|
|
* in that dimension, or simply not composited in. Setting it to
|
|
|
|
* %FALSE might be useful when animating those properties.
|
|
|
|
*
|
|
|
|
* Since: 1.20
|
|
|
|
*/
|
|
|
|
g_object_class_install_property (gobject_class, PROP_ZERO_SIZE_IS_UNSCALED,
|
|
|
|
g_param_spec_boolean ("zero-size-is-unscaled", "Zero size is unscaled",
|
|
|
|
"If TRUE, then input video is unscaled in that dimension "
|
|
|
|
"if width or height is 0 (for backwards compatibility)",
|
|
|
|
DEFAULT_ZERO_SIZE_IS_UNSCALED,
|
|
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
|
2018-02-28 23:34:40 +00:00
|
|
|
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
|
|
|
|
&src_factory, GST_TYPE_AGGREGATOR_PAD);
|
2017-11-06 20:07:51 +00:00
|
|
|
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
|
|
|
|
&sink_factory, GST_TYPE_COMPOSITOR_PAD);
|
2014-05-22 17:46:02 +00:00
|
|
|
|
|
|
|
gst_element_class_set_static_metadata (gstelement_class, "Compositor",
|
|
|
|
"Filter/Editor/Video/Compositor",
|
|
|
|
"Composite multiple video streams", "Wim Taymans <wim@fluendo.com>, "
|
|
|
|
"Sebastian Dröge <sebastian.droege@collabora.co.uk>");
|
2020-05-29 00:43:59 +00:00
|
|
|
|
2020-06-05 22:41:17 +00:00
|
|
|
gst_type_mark_as_plugin_api (GST_TYPE_COMPOSITOR_PAD, 0);
|
|
|
|
gst_type_mark_as_plugin_api (GST_TYPE_COMPOSITOR_OPERATOR, 0);
|
|
|
|
gst_type_mark_as_plugin_api (GST_TYPE_COMPOSITOR_BACKGROUND, 0);
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_init (GstCompositor * self)
|
|
|
|
{
|
|
|
|
/* initialize variables */
|
2017-07-06 18:26:21 +00:00
|
|
|
self->background = DEFAULT_BACKGROUND;
|
2020-11-07 01:59:09 +00:00
|
|
|
self->zero_size_is_unscaled = DEFAULT_ZERO_SIZE_IS_UNSCALED;
|
2014-05-22 17:46:02 +00:00
|
|
|
}
|
|
|
|
|
2018-05-04 14:13:16 +00:00
|
|
|
/* GstChildProxy implementation */
|
|
|
|
static GObject *
|
|
|
|
gst_compositor_child_proxy_get_child_by_index (GstChildProxy * child_proxy,
|
|
|
|
guint index)
|
|
|
|
{
|
|
|
|
GstCompositor *compositor = GST_COMPOSITOR (child_proxy);
|
|
|
|
GObject *obj = NULL;
|
|
|
|
|
|
|
|
GST_OBJECT_LOCK (compositor);
|
|
|
|
obj = g_list_nth_data (GST_ELEMENT_CAST (compositor)->sinkpads, index);
|
|
|
|
if (obj)
|
|
|
|
gst_object_ref (obj);
|
|
|
|
GST_OBJECT_UNLOCK (compositor);
|
|
|
|
|
|
|
|
return obj;
|
|
|
|
}
|
|
|
|
|
|
|
|
static guint
|
|
|
|
gst_compositor_child_proxy_get_children_count (GstChildProxy * child_proxy)
|
|
|
|
{
|
|
|
|
guint count = 0;
|
|
|
|
GstCompositor *compositor = GST_COMPOSITOR (child_proxy);
|
|
|
|
|
|
|
|
GST_OBJECT_LOCK (compositor);
|
|
|
|
count = GST_ELEMENT_CAST (compositor)->numsinkpads;
|
|
|
|
GST_OBJECT_UNLOCK (compositor);
|
|
|
|
GST_INFO_OBJECT (compositor, "Children Count: %d", count);
|
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
gst_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data)
|
|
|
|
{
|
|
|
|
GstChildProxyInterface *iface = g_iface;
|
|
|
|
|
|
|
|
iface->get_child_by_index = gst_compositor_child_proxy_get_child_by_index;
|
|
|
|
iface->get_children_count = gst_compositor_child_proxy_get_children_count;
|
|
|
|
}
|
|
|
|
|
2014-05-22 17:46:02 +00:00
|
|
|
/* Element registration */
|
|
|
|
static gboolean
|
|
|
|
plugin_init (GstPlugin * plugin)
|
|
|
|
{
|
|
|
|
GST_DEBUG_CATEGORY_INIT (gst_compositor_debug, "compositor", 0, "compositor");
|
|
|
|
|
|
|
|
gst_compositor_init_blend ();
|
|
|
|
|
|
|
|
return gst_element_register (plugin, "compositor", GST_RANK_PRIMARY + 1,
|
|
|
|
GST_TYPE_COMPOSITOR);
|
|
|
|
}
|
|
|
|
|
|
|
|
GST_PLUGIN_DEFINE (GST_VERSION_MAJOR,
|
|
|
|
GST_VERSION_MINOR,
|
|
|
|
compositor,
|
|
|
|
"Compositor", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME,
|
|
|
|
GST_PACKAGE_ORIGIN)
|