mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-01-02 13:38:48 +00:00
Code Issues 1 Projects Releases Wiki Activity

compositor: Implement different operators via per-pad property

Browse source 
This removes the crossfade-ratio property and replaces it with an
operator property. Currently this implements the following operators:

- SOURCE: Copy over the source and don't look at the destination
- OVER: Default blending of the source over the destination
- ADD: Like OVER but simply adding the alpha instead

See the example for how to implement crossfading with this.

https://bugzilla.gnome.org/show_bug.cgi?id=797169
This commit is contained in:
Sebastian Dröge 2018-10-27 16:44:27 +01:00
parent 55134df54c
commit aae25e0032
6 changed files with 236 additions and 217 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
gst/compositor/blend.c
View file

@ -100,32 +100,46 @@ _overlay_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \
{ \ { \
s_alpha = MIN (255, s_alpha); \ s_alpha = MIN (255, s_alpha); \
switch (mode) { \ switch (mode) { \
case COMPOSITOR_BLEND_MODE_NORMAL:\ case COMPOSITOR_BLEND_MODE_SOURCE:\
compositor_orc_source_##name (dest, dest_stride, src, src_stride, \
s_alpha, src_width, src_height); \
break;\
case COMPOSITOR_BLEND_MODE_OVER:\
compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \ compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \
s_alpha, src_width, src_height); \ s_alpha, src_width, src_height); \
break;\ break;\
case COMPOSITOR_BLEND_MODE_ADDITIVE:\ case COMPOSITOR_BLEND_MODE_ADD:\
compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \ compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \
s_alpha, src_width, src_height); \ s_alpha, src_width, src_height); \
break;\ break;\
}\ }\
} }
#define BLEND_A32_LOOP_WITH_MODE(name) \ #define BLEND_A32_LOOP(name) \
static inline void \ static inline void \
_blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ _blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \
gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \
GstCompositorBlendMode mode) \ GstCompositorBlendMode mode) \
{ \ { \
s_alpha = MIN (255, s_alpha); \ s_alpha = MIN (255, s_alpha); \
compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ switch (mode) { \
s_alpha, src_width, src_height); \ case COMPOSITOR_BLEND_MODE_SOURCE:\
compositor_orc_source_##name (dest, dest_stride, src, src_stride, \
s_alpha, src_width, src_height); \
break;\
case COMPOSITOR_BLEND_MODE_OVER:\
case COMPOSITOR_BLEND_MODE_ADD:\
/* both modes are the same for opaque background */ \
compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \
s_alpha, src_width, src_height); \
break;\
}\
} }
OVERLAY_A32_LOOP (argb); OVERLAY_A32_LOOP (argb);
OVERLAY_A32_LOOP (bgra); OVERLAY_A32_LOOP (bgra);
BLEND_A32_LOOP_WITH_MODE (argb); BLEND_A32_LOOP (argb);
BLEND_A32_LOOP_WITH_MODE (bgra); BLEND_A32_LOOP (bgra);
#if G_BYTE_ORDER == G_LITTLE_ENDIAN #if G_BYTE_ORDER == G_LITTLE_ENDIAN
BLEND_A32 (argb, blend, _blend_loop_argb); BLEND_A32 (argb, blend, _blend_loop_argb);
@ -223,11 +237,16 @@ A32_COLOR (ayuv, FALSE, 24, 16, 8, 0);
inline static void \ inline static void \
_blend_##format_name (const guint8 * src, guint8 * dest, \ _blend_##format_name (const guint8 * src, guint8 * dest, \
gint src_stride, gint dest_stride, gint src_width, gint src_height, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \
gdouble src_alpha) \ gdouble src_alpha, GstCompositorBlendMode mode) \
{ \ { \
gint i; \ gint i; \
gint b_alpha; \ gint b_alpha; \
\ \
/* in source mode we just have to copy over things */ \
if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \
src_alpha = 1.0; \
} \
\
/* If it's completely transparent... we just return */ \ /* If it's completely transparent... we just return */ \
if (G_UNLIKELY (src_alpha == 0.0)) { \ if (G_UNLIKELY (src_alpha == 0.0)) { \
GST_INFO ("Fast copy (alpha == 0.0)"); \ GST_INFO ("Fast copy (alpha == 0.0)"); \
@ -324,7 +343,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
src_comp_rowstride, \ src_comp_rowstride, \
dest_comp_rowstride, src_comp_width, src_comp_height, \ dest_comp_rowstride, src_comp_width, src_comp_height, \
src_alpha); \ src_alpha, mode); \
\ \
b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 1); \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 1); \
b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 1); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 1); \
@ -340,7 +359,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
src_comp_rowstride, \ src_comp_rowstride, \
dest_comp_rowstride, src_comp_width, src_comp_height, \ dest_comp_rowstride, src_comp_width, src_comp_height, \
src_alpha); \ src_alpha, mode); \
\ \
b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 2); \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 2); \
b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 2); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 2); \
@ -356,7 +375,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
src_comp_rowstride, \ src_comp_rowstride, \
dest_comp_rowstride, src_comp_width, src_comp_height, \ dest_comp_rowstride, src_comp_width, src_comp_height, \
src_alpha); \ src_alpha, mode); \
} }
#define PLANAR_YUV_FILL_CHECKER(format_name, format_enum, MEMSET) \ #define PLANAR_YUV_FILL_CHECKER(format_name, format_enum, MEMSET) \
@ -468,11 +487,16 @@ PLANAR_YUV_FILL_COLOR (y41b, GST_VIDEO_FORMAT_Y41B, memset);
inline static void \ inline static void \
_blend_##format_name (const guint8 * src, guint8 * dest, \ _blend_##format_name (const guint8 * src, guint8 * dest, \
gint src_stride, gint dest_stride, gint src_width, gint src_height, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \
gdouble src_alpha) \ gdouble src_alpha, GstCompositorBlendMode mode) \
{ \ { \
gint i; \ gint i; \
gint b_alpha; \ gint b_alpha; \
\ \
/* in source mode we just have to copy over things */ \
if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \
src_alpha = 1.0; \
} \
\
/* If it's completely transparent... we just return */ \ /* If it's completely transparent... we just return */ \
if (G_UNLIKELY (src_alpha == 0.0)) { \ if (G_UNLIKELY (src_alpha == 0.0)) { \
GST_INFO ("Fast copy (alpha == 0.0)"); \ GST_INFO ("Fast copy (alpha == 0.0)"); \
@ -569,7 +593,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \
src_comp_rowstride, \ src_comp_rowstride, \
dest_comp_rowstride, src_comp_width, src_comp_height, \ dest_comp_rowstride, src_comp_width, src_comp_height, \
src_alpha); \ src_alpha, mode); \
\ \
b_src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 1); \ b_src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 1); \
b_dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 1); \ b_dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 1); \
@ -585,7 +609,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
b_dest + comp_xpos * 2 + comp_ypos * dest_comp_rowstride, \ b_dest + comp_xpos * 2 + comp_ypos * dest_comp_rowstride, \
src_comp_rowstride, \ src_comp_rowstride, \
dest_comp_rowstride, 2 * src_comp_width, src_comp_height, \ dest_comp_rowstride, 2 * src_comp_width, src_comp_height, \
src_alpha); \ src_alpha, mode); \
} }
#define NV_YUV_FILL_CHECKER(format_name, MEMSET) \ #define NV_YUV_FILL_CHECKER(format_name, MEMSET) \
@ -711,6 +735,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
} \ } \
\ \
dest = dest + bpp * xpos + (ypos * dest_stride); \ dest = dest + bpp * xpos + (ypos * dest_stride); \
\
/* in source mode we just have to copy over things */ \
if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \
src_alpha = 1.0; \
} \
\
/* If it's completely transparent... we just return */ \ /* If it's completely transparent... we just return */ \
if (G_UNLIKELY (src_alpha == 0.0)) { \ if (G_UNLIKELY (src_alpha == 0.0)) { \
GST_INFO ("Fast copy (alpha == 0.0)"); \ GST_INFO ("Fast copy (alpha == 0.0)"); \
@ -880,6 +910,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \
} \ } \
\ \
dest = dest + 2 * xpos + (ypos * dest_stride); \ dest = dest + 2 * xpos + (ypos * dest_stride); \
\
/* in source mode we just have to copy over things */ \
if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \
src_alpha = 1.0; \
} \
\
/* If it's completely transparent... we just return */ \ /* If it's completely transparent... we just return */ \
if (G_UNLIKELY (src_alpha == 0.0)) { \ if (G_UNLIKELY (src_alpha == 0.0)) { \
GST_INFO ("Fast copy (alpha == 0.0)"); \ GST_INFO ("Fast copy (alpha == 0.0)"); \

15
gst/compositor/blend.h
View file

@ -25,15 +25,17 @@
/** /**
* GstCompositorBlendMode: * GstCompositorBlendMode:
* @COMPOSITOR_BLEND_MODE_NORMAL: Normal blending * @COMPOSITOR_BLEND_MODE_SOURCE: Copy source
* @COMPOSITOR_BLEND_MODE_ADDITIVE: Alphas are simply added, * @COMPOSITOR_BLEND_MODE_OVER: Normal blending
* @COMPOSITOR_BLEND_MODE_ADD: Alphas are simply added,
* *
* The different modes compositor can use for blending. * The different modes compositor can use for blending.
*/ */
typedef enum typedef enum
{ {
COMPOSITOR_BLEND_MODE_NORMAL, COMPOSITOR_BLEND_MODE_SOURCE,
COMPOSITOR_BLEND_MODE_ADDITIVE, COMPOSITOR_BLEND_MODE_OVER,
COMPOSITOR_BLEND_MODE_ADD,
} GstCompositorBlendMode; } GstCompositorBlendMode;
typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe, typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe,
@ -46,16 +48,11 @@ extern BlendFunction gst_compositor_blend_bgra;
#define gst_compositor_blend_ayuv gst_compositor_blend_argb #define gst_compositor_blend_ayuv gst_compositor_blend_argb
#define gst_compositor_blend_abgr gst_compositor_blend_argb #define gst_compositor_blend_abgr gst_compositor_blend_argb
#define gst_compositor_blend_rgba gst_compositor_blend_bgra #define gst_compositor_blend_rgba gst_compositor_blend_bgra
#define gst_compositor_blend_ayuv_addition gst_compositor_blend_argb_addition
#define gst_compositor_blend_abgr_addition gst_compositor_blend_argb_addition
#define gst_compositor_blend_rgba_addition gst_compositor_blend_bgra_addition
extern BlendFunction gst_compositor_overlay_argb; extern BlendFunction gst_compositor_overlay_argb;
extern BlendFunction gst_compositor_overlay_bgra; extern BlendFunction gst_compositor_overlay_bgra;
#define gst_compositor_overlay_ayuv gst_compositor_overlay_argb #define gst_compositor_overlay_ayuv gst_compositor_overlay_argb
#define gst_compositor_overlay_abgr gst_compositor_overlay_argb #define gst_compositor_overlay_abgr gst_compositor_overlay_argb
#define gst_compositor_overlay_abgr gst_compositor_overlay_argb
#define gst_compositor_overlay_rgba gst_compositor_overlay_bgra #define gst_compositor_overlay_rgba gst_compositor_overlay_bgra
extern BlendFunction gst_compositor_blend_i420; extern BlendFunction gst_compositor_blend_i420;
#define gst_compositor_blend_yv12 gst_compositor_blend_i420 #define gst_compositor_blend_yv12 gst_compositor_blend_i420

249
gst/compositor/compositor.c
View file

@ -122,12 +122,32 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
static void gst_compositor_child_proxy_init (gpointer g_iface, static void gst_compositor_child_proxy_init (gpointer g_iface,
gpointer iface_data); gpointer iface_data);
#define GST_TYPE_COMPOSITOR_OPERATOR (gst_compositor_operator_get_type())
static GType
gst_compositor_operator_get_type (void)
{
static GType compositor_operator_type = 0;
static const GEnumValue compositor_operator[] = {
{COMPOSITOR_OPERATOR_SOURCE, "Source", "source"},
{COMPOSITOR_OPERATOR_OVER, "Over", "over"},
{COMPOSITOR_OPERATOR_ADD, "Add", "add"},
{0, NULL, NULL},
};
if (!compositor_operator_type) {
compositor_operator_type =
g_enum_register_static ("GstCompositorOperator", compositor_operator);
}
return compositor_operator_type;
}
#define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_XPOS 0
#define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_YPOS 0
#define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_WIDTH 0
#define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_HEIGHT 0
#define DEFAULT_PAD_ALPHA 1.0 #define DEFAULT_PAD_ALPHA 1.0
#define DEFAULT_PAD_CROSSFADE_RATIO 0.0 #define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER
enum enum
{ {
PROP_PAD_0, PROP_PAD_0,
@ -136,7 +156,7 @@ enum
PROP_PAD_WIDTH, PROP_PAD_WIDTH,
PROP_PAD_HEIGHT, PROP_PAD_HEIGHT,
PROP_PAD_ALPHA, PROP_PAD_ALPHA,
PROP_PAD_CROSSFADE_RATIO, PROP_PAD_OPERATOR,
}; };
G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad,
@ -164,8 +184,8 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id,
case PROP_PAD_ALPHA: case PROP_PAD_ALPHA:
g_value_set_double (value, pad->alpha); g_value_set_double (value, pad->alpha);
break; break;
case PROP_PAD_CROSSFADE_RATIO: case PROP_PAD_OPERATOR:
g_value_set_double (value, pad->crossfade); g_value_set_enum (value, pad->op);
break; break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@ -199,10 +219,10 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id,
case PROP_PAD_ALPHA: case PROP_PAD_ALPHA:
pad->alpha = g_value_get_double (value); pad->alpha = g_value_get_double (value);
break; break;
case PROP_PAD_CROSSFADE_RATIO: case PROP_PAD_OPERATOR:
pad->crossfade = g_value_get_double (value); pad->op = g_value_get_enum (value);
gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad),
pad->crossfade > 0.0); pad->op == COMPOSITOR_OPERATOR_ADD);
break; break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
@ -319,7 +339,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height);
if (cpad->alpha == 0.0) { if (cpad->alpha == 0.0) {
GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame");
goto done; goto done;
} }
@ -327,25 +347,16 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad,
GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info)); GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info));
if (frame_rect.w == 0 || frame_rect.h == 0) { if (frame_rect.w == 0 || frame_rect.h == 0) {
GST_DEBUG_OBJECT (vagg, "Resulting frame is zero-width or zero-height " GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height "
"(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h);
goto done; goto done;
} }
GST_OBJECT_LOCK (vagg); GST_OBJECT_LOCK (vagg);
/* Check if we are crossfading the pad one way or another */
l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad);
if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade > 0.0) ||
(GST_COMPOSITOR_PAD (pad)->crossfade > 0.0)) {
GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad");
l = NULL;
} else {
l = l->next;
}
/* Check if this frame is obscured by a higher-zorder frame /* Check if this frame is obscured by a higher-zorder frame
* TODO: Also skip a frame if it's obscured by a combination of * TODO: Also skip a frame if it's obscured by a combination of
* higher-zorder frames */ * higher-zorder frames */
l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next;
for (; l; l = l->next) { for (; l; l = l->next) {
GstVideoRectangle frame2_rect; GstVideoRectangle frame2_rect;
GstVideoAggregatorPad *pad2 = l->data; GstVideoAggregatorPad *pad2 = l->data;
@ -468,10 +479,10 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass)
g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0,
DEFAULT_PAD_ALPHA, DEFAULT_PAD_ALPHA,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, g_object_class_install_property (gobject_class, PROP_PAD_OPERATOR,
g_param_spec_double ("crossfade-ratio", "Crossfade ratio", g_param_spec_enum ("operator", "Operator",
"The crossfade ratio to use while crossfading with the following pad", "Blending operator to use for blending this pad over the previous ones",
0.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR,
G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS));
vaggpadclass->prepare_frame = vaggpadclass->prepare_frame =
@ -487,7 +498,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad)
compo_pad->xpos = DEFAULT_PAD_XPOS; compo_pad->xpos = DEFAULT_PAD_XPOS;
compo_pad->ypos = DEFAULT_PAD_YPOS; compo_pad->ypos = DEFAULT_PAD_YPOS;
compo_pad->alpha = DEFAULT_PAD_ALPHA; compo_pad->alpha = DEFAULT_PAD_ALPHA;
compo_pad->crossfade = DEFAULT_PAD_CROSSFADE_RATIO; compo_pad->op = DEFAULT_PAD_OPERATOR;
} }
@ -818,137 +829,6 @@ _negotiated_caps (GstAggregator * agg, GstCaps * caps)
return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps);
} }
/* Fills frame with transparent pixels if @nframe is NULL otherwise copy @frame
* properties and fill @nframes with transparent pixels */
static GstFlowReturn
gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame,
GstVideoFrame * nframe)
{
guint plane, num_planes, height, i;
if (nframe) {
GstBuffer *cbuffer = gst_buffer_copy_deep (frame->buffer);
if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) {
GST_WARNING_OBJECT (self, "Could not map output buffer");
gst_buffer_unref (cbuffer);
return GST_FLOW_ERROR;
}
/* the last reference is owned by the frame and released once the frame
* is unmapped. We leak it if we don't unref here */
gst_buffer_unref (cbuffer);
} else {
nframe = frame;
}
num_planes = GST_VIDEO_FRAME_N_PLANES (nframe);
for (plane = 0; plane < num_planes; ++plane) {
guint8 *pdata;
gsize rowsize, plane_stride;
pdata = GST_VIDEO_FRAME_PLANE_DATA (nframe, plane);
plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (nframe, plane);
rowsize = GST_VIDEO_FRAME_COMP_WIDTH (nframe, plane)
* GST_VIDEO_FRAME_COMP_PSTRIDE (nframe, plane);
height = GST_VIDEO_FRAME_COMP_HEIGHT (nframe, plane);
for (i = 0; i < height; ++i) {
memset (pdata, 0, rowsize);
pdata += plane_stride;
}
}
return GST_FLOW_OK;
}
/* WITH GST_OBJECT_LOCK !!
* Returns: %TRUE if outframe is allready ready to be used as we are using
* a transparent background and all pads have already been crossfaded
* %FALSE otherwise
*/
static gboolean
gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe)
{
GList *l;
gboolean all_crossfading = FALSE;
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (self);
if (self->background == COMPOSITOR_BACKGROUND_TRANSPARENT) {
all_crossfading = TRUE;
for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) {
GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data);
if (compo_pad->crossfade == 0.0 && l->next &&
GST_COMPOSITOR_PAD (l->next->data)->crossfade == 0.0) {
all_crossfading = FALSE;
break;
}
}
}
for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) {
GstVideoAggregatorPad *pad = l->data;
GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad);
GstVideoAggregatorPadClass *pad_class =
GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (compo_pad);
GstVideoFrame *prepared_frame =
gst_video_aggregator_pad_get_prepared_frame (pad);
if (compo_pad->crossfade > 0.0 && prepared_frame) {
gfloat alpha = compo_pad->crossfade * compo_pad->alpha;
GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL;
GstVideoFrame *next_prepared_frame;
GstVideoFrame nframe;
next_prepared_frame =
npad ? gst_video_aggregator_pad_get_prepared_frame (npad) : NULL;
if (!all_crossfading) {
gst_compositor_fill_transparent (self, outframe, &nframe);
} else {
nframe = *outframe;
}
self->overlay (prepared_frame,
compo_pad->crossfaded ? 0 : compo_pad->xpos,
compo_pad->crossfaded ? 0 : compo_pad->ypos,
alpha, &nframe, COMPOSITOR_BLEND_MODE_ADDITIVE);
if (npad && next_prepared_frame) {
GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad);
alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha;
self->overlay (next_prepared_frame, next_compo_pad->xpos,
next_compo_pad->ypos, alpha, &nframe,
COMPOSITOR_BLEND_MODE_ADDITIVE);
/* Replace frame with current frame */
pad_class->clean_frame (npad, vagg, next_prepared_frame);
if (!all_crossfading)
*next_prepared_frame = nframe;
next_compo_pad->crossfaded = TRUE;
/* Frame is now consumed, clean it up */
pad_class->clean_frame (pad, vagg, prepared_frame);
} else {
GST_LOG_OBJECT (self, "Simply fading out as no following pad found");
pad_class->clean_frame (pad, vagg, prepared_frame);
if (!all_crossfading)
*prepared_frame = nframe;
compo_pad->crossfaded = TRUE;
}
}
}
if (all_crossfading)
for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next)
GST_COMPOSITOR_PAD (l->data)->crossfaded = FALSE;
return all_crossfading;
}
static GstFlowReturn static GstFlowReturn
gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
{ {
@ -978,28 +858,57 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
self->fill_color (outframe, 240, 128, 128); self->fill_color (outframe, 240, 128, 128);
break; break;
case COMPOSITOR_BACKGROUND_TRANSPARENT: case COMPOSITOR_BACKGROUND_TRANSPARENT:
gst_compositor_fill_transparent (self, outframe, NULL); {
guint i, plane, num_planes, height;
num_planes = GST_VIDEO_FRAME_N_PLANES (outframe);
for (plane = 0; plane < num_planes; ++plane) {
guint8 *pdata;
gsize rowsize, plane_stride;
pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane);
plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane);
rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane)
* GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane);
height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane);
for (i = 0; i < height; ++i) {
memset (pdata, 0, rowsize);
pdata += plane_stride;
}
}
/* use overlay to keep background transparent */ /* use overlay to keep background transparent */
composite = self->overlay; composite = self->overlay;
break; break;
}
} }
GST_OBJECT_LOCK (vagg); GST_OBJECT_LOCK (vagg);
/* First mix the crossfade frames as required */ for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) {
if (!gst_compositor_crossfade_frames (self, outframe)) { GstVideoAggregatorPad *pad = l->data;
for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad);
GstVideoAggregatorPad *pad = l->data; GstVideoFrame *prepared_frame =
GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); gst_video_aggregator_pad_get_prepared_frame (pad);
GstVideoFrame *prepared_frame = GstCompositorBlendMode blend_mode = COMPOSITOR_BLEND_MODE_OVER;
gst_video_aggregator_pad_get_prepared_frame (pad);
if (prepared_frame != NULL) { switch (compo_pad->op) {
composite (prepared_frame, case COMPOSITOR_OPERATOR_SOURCE:
compo_pad->crossfaded ? 0 : compo_pad->xpos, blend_mode = COMPOSITOR_OPERATOR_SOURCE;
compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, break;
outframe, COMPOSITOR_BLEND_MODE_NORMAL); case COMPOSITOR_OPERATOR_OVER:
compo_pad->crossfaded = FALSE; blend_mode = COMPOSITOR_OPERATOR_OVER;
} break;
case COMPOSITOR_OPERATOR_ADD:
blend_mode = COMPOSITOR_OPERATOR_ADD;
break;
default:
g_assert_not_reached ();
break;
}
if (prepared_frame != NULL) {
composite (prepared_frame,
compo_pad->xpos,
compo_pad->ypos, compo_pad->alpha, outframe, blend_mode);
} }
} }
GST_OBJECT_UNLOCK (vagg); GST_OBJECT_UNLOCK (vagg);

27
gst/compositor/compositor.h
View file

@ -56,7 +56,7 @@ typedef struct _GstCompositorPad GstCompositorPad;
typedef struct _GstCompositorPadClass GstCompositorPadClass; typedef struct _GstCompositorPadClass GstCompositorPadClass;
/** /**
* GstcompositorBackground: * GstCompositorBackground:
* @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background * @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background
* @COMPOSITOR_BACKGROUND_BLACK: solid color black background * @COMPOSITOR_BACKGROUND_BLACK: solid color black background
* @COMPOSITOR_BACKGROUND_WHITE: solid color white background * @COMPOSITOR_BACKGROUND_WHITE: solid color white background
@ -72,6 +72,28 @@ typedef enum
COMPOSITOR_BACKGROUND_TRANSPARENT, COMPOSITOR_BACKGROUND_TRANSPARENT,
} GstCompositorBackground; } GstCompositorBackground;
/**
* GstCompositorOperator:
* @COMPOSITOR_OPERATOR_SOURCE: Copy the source over the destination,
* without the destination pixels.
* @COMPOSITOR_OPERATOR_OVER: Blend the source over the destination.
* @COMPOSITOR_OPERATOR_ADD: Similar to over but add the source and
* destination alpha. Requires output with alpha
* channel.
*
* The different blending operators that can be used by compositor.
*
* See https://www.cairographics.org/operators/ for some explanation and
* visualizations.
*
*/
typedef enum
{
COMPOSITOR_OPERATOR_SOURCE,
COMPOSITOR_OPERATOR_OVER,
COMPOSITOR_OPERATOR_ADD,
} GstCompositorOperator;
/** /**
* GstCompositor: * GstCompositor:
* *
@ -105,9 +127,8 @@ struct _GstCompositorPad
gint xpos, ypos; gint xpos, ypos;
gint width, height; gint width, height;
gdouble alpha; gdouble alpha;
gdouble crossfade;
gboolean crossfaded; GstCompositorOperator op;
}; };
struct _GstCompositorPadClass struct _GstCompositorPadClass

60
gst/compositor/compositororc.orc
View file

@ -62,6 +62,36 @@ x4 convwb t, d_wide
orl t, t, a_alpha orl t, t, a_alpha
storel d, t storel d, t
.function compositor_orc_source_argb
.flags 2d
.dest 4 d guint8
.source 4 s guint8
.param 2 alpha
.temp 4 t
.temp 4 t2
.temp 2 tw
.temp 1 tb
.temp 4 a
.temp 8 a_wide
.const 4 a_alpha 0x000000ff
.const 4 a_not_alpha 0xffffff00
loadl t, s
shrul t2, t, 24
convlw tw, t2
convwb tb, tw
splatbl a, tb
x4 convubw a_wide, a
x4 mullw a_wide, a_wide, alpha
x4 div255w a_wide, a_wide
andl t, t, a_not_alpha
x4 convwb t2, a_wide
andl t2, t2, a_alpha
orl t, t, t2
storel d, t
.function compositor_orc_blend_bgra .function compositor_orc_blend_bgra
.flags 2d .flags 2d
.dest 4 d guint8 .dest 4 d guint8
@ -98,6 +128,36 @@ x4 convwb t, d_wide
orl t, t, a_alpha orl t, t, a_alpha
storel d, t storel d, t
.function compositor_orc_source_bgra
.flags 2d
.dest 4 d guint8
.source 4 s guint8
.param 2 alpha
.temp 4 t
.temp 4 t2
.temp 2 tw
.temp 1 tb
.temp 4 a
.temp 8 a_wide
.const 4 a_alpha 0xff000000
.const 4 a_not_alpha 0x00ffffff
loadl t, s
shrul t2, t, 24
convlw tw, t2
convwb tb, tw
splatbl a, tb
x4 convubw a_wide, a
x4 mullw a_wide, a_wide, alpha
x4 div255w a_wide, a_wide
andl t, t, a_not_alpha
x4 convwb t2, a_wide
andl t2, t2, a_alpha
orl t, t, t2
storel d, t
.function compositor_orc_overlay_argb .function compositor_orc_overlay_argb
.flags 2d .flags 2d
.dest 4 d guint8 .dest 4 d guint8

38
tests/examples/compositor/crossfade.c
View file

@ -21,8 +21,7 @@
/** /**
* Simple crossfade example using the compositor element. * Simple crossfade example using the compositor element.
* *
* Takes a list of uri/path to video files and crossfade them * Takes two video files and crossfades them for 10 seconds and returns.
* for 10 seconds and returns.
*/ */
#include <stdlib.h> #include <stdlib.h>
@ -34,7 +33,6 @@ typedef struct
{ {
GstElement *compositor; GstElement *compositor;
guint z_order; guint z_order;
gboolean is_last;
} VideoInfo; } VideoInfo;
static gchar * static gchar *
@ -51,24 +49,23 @@ _pad_added_cb (GstElement * decodebin, GstPad * pad, VideoInfo * info)
{ {
GstPad *sinkpad = GstPad *sinkpad =
gst_element_get_request_pad (GST_ELEMENT (info->compositor), "sink_%u"); gst_element_get_request_pad (GST_ELEMENT (info->compositor), "sink_%u");
GstControlSource *control_source;
gboolean is_last = info->z_order == 1;
if (!info->is_last) { control_source = gst_interpolation_control_source_new ();
GstControlSource *control_source;
control_source = gst_interpolation_control_source_new (); gst_util_set_object_arg (G_OBJECT (sinkpad), "operator",
info->z_order == 0 ? "source" : "add");
gst_object_add_control_binding (GST_OBJECT (sinkpad),
gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), "alpha",
control_source));
g_object_set (sinkpad, "crossfade-ratio", 1.0, NULL); g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL);
gst_object_add_control_binding (GST_OBJECT (sinkpad),
gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad),
"crossfade-ratio", control_source));
g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE
(control_source), 0, is_last ? 0.0 : 1.0);
gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE
(control_source), 0, 1.0); (control_source), 10 * GST_SECOND, is_last ? 1.0 : 0.0);
gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE
(control_source), 10 * GST_SECOND, 0.0);
}
g_object_set (sinkpad, "zorder", info->z_order, NULL); g_object_set (sinkpad, "zorder", info->z_order, NULL);
gst_pad_link (pad, sinkpad); gst_pad_link (pad, sinkpad);
@ -84,8 +81,8 @@ main (int argc, char *argv[])
GstElement *compositor, *sink, *pipeline; GstElement *compositor, *sink, *pipeline;
GstBus *bus; GstBus *bus;
if (argc < 2) { if (argc != 3) {
g_error ("At least 1 valid video file paths/urls need to " "be provided"); g_error ("Need to provide 2 input videos");
return -1; return -1;
} }
@ -101,7 +98,7 @@ main (int argc, char *argv[])
gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL); gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL);
g_assert (gst_element_link (compositor, sink)); g_assert (gst_element_link (compositor, sink));
for (i = 1; i < argc; i++) { for (i = 1; i < 3; i++) {
gchar *uri = ensure_uri (argv[i]); gchar *uri = ensure_uri (argv[i]);
VideoInfo *info = g_malloc0 (sizeof (VideoInfo)); VideoInfo *info = g_malloc0 (sizeof (VideoInfo));
GstElement *uridecodebin = gst_element_factory_make ("uridecodebin", NULL); GstElement *uridecodebin = gst_element_factory_make ("uridecodebin", NULL);
@ -111,7 +108,6 @@ main (int argc, char *argv[])
info->compositor = compositor; info->compositor = compositor;
info->z_order = i - 1; info->z_order = i - 1;
info->is_last = (i == (argc - 1)) && (argc > 2);
g_signal_connect (uridecodebin, "pad-added", (GCallback) _pad_added_cb, g_signal_connect (uridecodebin, "pad-added", (GCallback) _pad_added_cb,
info); info);