gstreamer/subprojects/gst-plugins-bad/gst/codecalpha/gstalphacombine.c
Nicolas Dufresne 2886eab3c4 alphacombine: Fix for early allocation queries
When using playbin3, it seems that the alpha decode is always first to
push caps and run an allocation query. As the format change from sink
and alpha were not synchronized, the allocation query could endup
being run before the caps are pushed. That may lead to failing query,
which makes the decoder thinks there is no GstVideoMeta downstream and
most likely CPU copy the frame.

This patch implements a format cookie to track and synchronize the
format changes on both pads fixing the racy performance issue.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1439>
2021-12-10 21:37:14 +00:00

673 lines
19 KiB
C

/* GStreamer
* Copyright (C) <2021> Collabora Ltd.
* Author: Nicolas Dufresne <nicolas.dufresne@collabora.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-alphacombine
* @title: Alpha Combiner
*
* This element can combine a Luma plane from one stream as being the alpha
* plane of another stream. This element can only work with planar formats
* that have an equivalent format with an alpha plane. This is notably used to
* combine VP8/VP9 alpha streams from WebM container.
*
* ## Example launch line
* |[
* gst-launch-1.0 -v videotestsrc ! .c videotestsrc pattern=ball ! .c
* alphacombine name=c ! compositor ! autovideosink
* ]| This pipeline uses luma of a ball test pattern as alpha, combined with
* default test pattern and renders the resulting moving ball on a checker
* board.
*
* Since: 1.20
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <gst/video/video.h>
#include "gstalphacombine.h"
#define SUPPORTED_SINK_FORMATS "{ I420, NV12 }"
#define SUPPORTED_ALPHA_FORMATS "{ GRAY8, I420, NV12 }"
#define SUPPORTED_SRC_FORMATS "{ A420, AV12 }"
/* *INDENT-OFF* */
struct {
GstVideoFormat sink;
GstVideoFormat alpha;
GstVideoFormat src;
} format_map[] = {
{
.sink = GST_VIDEO_FORMAT_I420,
.alpha = GST_VIDEO_FORMAT_I420,
.src = GST_VIDEO_FORMAT_A420
},{
.sink = GST_VIDEO_FORMAT_I420,
.alpha = GST_VIDEO_FORMAT_GRAY8,
.src = GST_VIDEO_FORMAT_A420
},{
.sink = GST_VIDEO_FORMAT_I420,
.alpha = GST_VIDEO_FORMAT_NV12,
.src = GST_VIDEO_FORMAT_A420
}, {
.sink = GST_VIDEO_FORMAT_NV12,
.alpha = GST_VIDEO_FORMAT_NV12,
.src = GST_VIDEO_FORMAT_AV12,
}, {
.sink = GST_VIDEO_FORMAT_NV12,
.alpha = GST_VIDEO_FORMAT_GRAY8,
.src = GST_VIDEO_FORMAT_AV12
},{
.sink = GST_VIDEO_FORMAT_NV12,
.alpha = GST_VIDEO_FORMAT_I420,
.src = GST_VIDEO_FORMAT_AV12
},
};
/* *INDENT-ON* */
GST_DEBUG_CATEGORY_STATIC (alphacombine_debug);
#define GST_CAT_DEFAULT (alphacombine_debug)
struct _GstAlphaCombine
{
GstElement parent;
GstPad *sink_pad;
GstPad *alpha_pad;
GstPad *src_pad;
/* protected by sink_pad stream lock */
GstBuffer *last_alpha_buffer;
GstFlowReturn last_flow_ret;
GMutex buffer_lock;
GCond buffer_cond;
GstBuffer *alpha_buffer;
/* Ref-counted flushing state */
guint flushing;
GstVideoInfo sink_vinfo;
GstVideoInfo alpha_vinfo;
GstVideoFormat src_format;
guint sink_format_cookie;
guint alpha_format_cookie;
};
#define gst_alpha_combine_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstAlphaCombine, gst_alpha_combine,
GST_TYPE_ELEMENT,
GST_DEBUG_CATEGORY_INIT (alphacombine_debug, "alphacombine", 0,
"Alpha Combiner"));
GST_ELEMENT_REGISTER_DEFINE (alpha_combine, "alphacombine",
GST_RANK_NONE, GST_TYPE_ALPHA_COMBINE);
static GstStaticPadTemplate gst_alpha_combine_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SUPPORTED_SINK_FORMATS))
);
static GstStaticPadTemplate gst_alpha_combine_alpha_template =
GST_STATIC_PAD_TEMPLATE ("alpha",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SUPPORTED_ALPHA_FORMATS))
);
static GstStaticPadTemplate gst_alpha_combine_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (SUPPORTED_SRC_FORMATS))
);
static void
gst_alpha_combine_unlock (GstAlphaCombine * self)
{
g_mutex_lock (&self->buffer_lock);
self->flushing++;
g_cond_broadcast (&self->buffer_cond);
g_mutex_unlock (&self->buffer_lock);
}
static void
gst_alpha_combine_unlock_stop (GstAlphaCombine * self)
{
g_mutex_lock (&self->buffer_lock);
g_assert (self->flushing);
self->flushing--;
/* Reset the format cookies to ensure they are equal */
if (!self->flushing) {
self->sink_format_cookie = 0;
self->alpha_format_cookie = 0;
}
g_mutex_unlock (&self->buffer_lock);
}
static void
gst_alpha_combine_reset (GstAlphaCombine * self)
{
gst_buffer_replace (&self->alpha_buffer, NULL);
gst_buffer_replace (&self->last_alpha_buffer, NULL);
self->last_flow_ret = GST_FLOW_OK;
}
/*
* gst_alpha_combine_negotiate:
* @self: #GstAlphaCombine pointer
*
* Verify that the stream and alpha stream format are compatible and fail
* otherwise. There is no effort in helping upstream to dynamically negotiate
* a valid combination to keep the complexity low, and because this would be a
* very atypical usage.
*/
static gboolean
gst_alpha_combine_negotiate (GstAlphaCombine * self)
{
gint i;
GstVideoFormat src_format = GST_VIDEO_FORMAT_UNKNOWN;
GstVideoFormat sink_format = GST_VIDEO_INFO_FORMAT (&self->sink_vinfo);
GstVideoFormat alpha_format = GST_VIDEO_INFO_FORMAT (&self->alpha_vinfo);
if (self->src_format != GST_VIDEO_FORMAT_UNKNOWN)
return TRUE;
for (i = 0; i < G_N_ELEMENTS (format_map); i++) {
if (format_map[i].sink == sink_format
&& format_map[i].alpha == alpha_format) {
src_format = format_map[i].src;
break;
}
}
if (src_format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_ELEMENT_ERROR (self, STREAM, FORMAT, ("Unsupported formats."),
("Cannot combined '%s' and '%s' into any supported transparent format",
gst_video_format_to_string (sink_format),
gst_video_format_to_string (alpha_format)));
return FALSE;
}
if (GST_VIDEO_INFO_COLORIMETRY (&self->sink_vinfo).range !=
GST_VIDEO_INFO_COLORIMETRY (&self->alpha_vinfo).range) {
GST_ELEMENT_ERROR (self, STREAM, FORMAT, ("Color range miss-match"),
("We can only combine buffers if they have the same color range."));
return FALSE;
}
self->src_format = src_format;
return TRUE;
}
static GstFlowReturn
gst_alpha_combine_peek_alpha_buffer (GstAlphaCombine * self,
GstBuffer ** alpha_buffer)
{
g_mutex_lock (&self->buffer_lock);
while (!self->alpha_buffer && !self->flushing)
g_cond_wait (&self->buffer_cond, &self->buffer_lock);
if (self->flushing) {
g_mutex_unlock (&self->buffer_lock);
return GST_FLOW_FLUSHING;
}
/* Now is a good time to validate the formats, as the alpha_vinfo won't be
* updated until we signal this alpha_buffer_as being consumed */
if (!gst_alpha_combine_negotiate (self)) {
g_mutex_unlock (&self->buffer_lock);
return GST_FLOW_NOT_NEGOTIATED;
}
*alpha_buffer = gst_buffer_ref (self->alpha_buffer);
g_mutex_unlock (&self->buffer_lock);
if (GST_BUFFER_FLAG_IS_SET (*alpha_buffer, GST_BUFFER_FLAG_GAP)) {
if (!self->last_alpha_buffer) {
GST_ELEMENT_ERROR (self, STREAM, WRONG_TYPE,
("Cannot handle streams without an initial alpha buffer."), (NULL));
gst_clear_buffer (alpha_buffer);
return GST_FLOW_ERROR;
}
/* Re-use the last alpha buffer if one is gone missing */
gst_buffer_replace (alpha_buffer, self->last_alpha_buffer);
}
return GST_FLOW_OK;
}
static void
gst_alpha_combine_pop_alpha_buffer (GstAlphaCombine * self,
GstFlowReturn flow_ret)
{
g_mutex_lock (&self->buffer_lock);
self->last_flow_ret = flow_ret;
gst_clear_buffer (&self->alpha_buffer);
g_cond_broadcast (&self->buffer_cond);
g_mutex_unlock (&self->buffer_lock);
}
static GstFlowReturn
gst_alpha_combine_push_alpha_buffer (GstAlphaCombine * self, GstBuffer * buffer)
{
GstFlowReturn ret;
g_mutex_lock (&self->buffer_lock);
/* We wait for the alpha_buffer to be consumed and store the buffer for the
* sink_chain to pick it up */
while (self->alpha_buffer && !self->flushing)
g_cond_wait (&self->buffer_cond, &self->buffer_lock);
if (self->flushing) {
gst_buffer_unref (buffer);
g_mutex_unlock (&self->buffer_lock);
return GST_FLOW_FLUSHING;
}
self->alpha_buffer = buffer;
GST_DEBUG_OBJECT (self, "Stored pending alpha buffer %p", buffer);
g_cond_signal (&self->buffer_cond);
ret = self->last_flow_ret;
g_mutex_unlock (&self->buffer_lock);
return ret;
}
static GstFlowReturn
gst_alpha_combine_sink_chain (GstPad * pad, GstObject * object,
GstBuffer * src_buffer)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
GstFlowReturn ret;
GstVideoMeta *vmeta;
GstBuffer *alpha_buffer;
GstMemory *alpha_mem = NULL;
gsize alpha_skip = 0;
gint alpha_stride;
GstBuffer *buffer;
guint alpha_plane_idx;
ret = gst_alpha_combine_peek_alpha_buffer (self, &alpha_buffer);
if (ret != GST_FLOW_OK)
return ret;
GST_DEBUG_OBJECT (self, "Combining buffer %p with alpha buffer %p",
src_buffer, alpha_buffer);
vmeta = gst_buffer_get_video_meta (alpha_buffer);
if (vmeta) {
guint idx, length;
if (gst_buffer_find_memory (alpha_buffer, vmeta->offset[GST_VIDEO_COMP_Y],
1, &idx, &length, &alpha_skip)) {
alpha_mem = gst_buffer_get_memory (alpha_buffer, idx);
}
alpha_stride = vmeta->stride[GST_VIDEO_COMP_Y];
} else {
alpha_mem = gst_buffer_get_memory (alpha_buffer, 0);
alpha_stride = self->alpha_vinfo.stride[GST_VIDEO_COMP_Y];
}
if (!alpha_mem) {
gst_buffer_unref (alpha_buffer);
gst_buffer_unref (src_buffer);
GST_ELEMENT_ERROR (self, STREAM, WRONG_TYPE,
("Invalid alpha video frame."), ("Could not find the plane"));
return GST_FLOW_ERROR;
}
/* FIXME use some GstBuffer cache to reduce run-time allocation */
buffer = gst_buffer_copy (src_buffer);
vmeta = gst_buffer_get_video_meta (buffer);
if (!vmeta)
vmeta = gst_buffer_add_video_meta (buffer, 0,
GST_VIDEO_INFO_FORMAT (&self->sink_vinfo),
GST_VIDEO_INFO_WIDTH (&self->sink_vinfo),
GST_VIDEO_INFO_HEIGHT (&self->sink_vinfo));
alpha_skip += gst_buffer_get_size (buffer);
gst_buffer_append_memory (buffer, alpha_mem);
alpha_plane_idx = GST_VIDEO_INFO_N_PLANES (&self->sink_vinfo);
vmeta->offset[alpha_plane_idx] = alpha_skip;
vmeta->stride[alpha_plane_idx] = alpha_stride;
vmeta->format = self->src_format;
vmeta->n_planes = alpha_plane_idx + 1;
/* Keep the origina GstBuffer alive to make this buffer pool friendly */
gst_buffer_add_parent_buffer_meta (buffer, src_buffer);
gst_buffer_add_parent_buffer_meta (buffer, alpha_buffer);
gst_buffer_replace (&self->last_alpha_buffer, alpha_buffer);
gst_buffer_unref (src_buffer);
gst_buffer_unref (alpha_buffer);
ret = gst_pad_push (self->src_pad, buffer);
gst_alpha_combine_pop_alpha_buffer (self, ret);
return ret;
}
static GstFlowReturn
gst_alpha_combine_alpha_chain (GstPad * pad, GstObject * object,
GstBuffer * buffer)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
return gst_alpha_combine_push_alpha_buffer (self, buffer);
}
static gboolean
gst_alpha_combine_set_sink_format (GstAlphaCombine * self, GstCaps * caps)
{
GstVideoFormat sink_format, src_format = GST_VIDEO_FORMAT_UNKNOWN;
GstEvent *event;
gint i;
gboolean ret;
if (!gst_video_info_from_caps (&self->sink_vinfo, caps)) {
GST_ELEMENT_ERROR (self, STREAM, FORMAT, ("Invalid video format"), (NULL));
return FALSE;
}
sink_format = GST_VIDEO_INFO_FORMAT (&self->sink_vinfo);
/* The sink format determin the src format, though we cannot fully validate
* the negotiation here, since we don't have the alpha format yet. */
for (i = 0; i < G_N_ELEMENTS (format_map); i++) {
if (format_map[i].sink == sink_format) {
src_format = format_map[i].src;
break;
}
}
if (src_format == GST_VIDEO_FORMAT_UNKNOWN) {
GST_ELEMENT_ERROR (self, STREAM, FORMAT, ("Unsupported formats."),
("Sink format '%s' not supported.",
gst_video_format_to_string (sink_format)));
return FALSE;
}
caps = gst_caps_copy (caps);
gst_caps_set_simple (caps, "format", G_TYPE_STRING,
gst_video_format_to_string (src_format), NULL);
event = gst_event_new_caps (caps);
gst_caps_unref (caps);
ret = gst_pad_push_event (self->src_pad, event);
/* signal the sink format change */
g_mutex_lock (&self->buffer_lock);
self->sink_format_cookie++;
g_cond_signal (&self->buffer_cond);
g_mutex_unlock (&self->buffer_lock);
return ret;
}
static gboolean
gst_alpha_combine_set_alpha_format (GstAlphaCombine * self, GstCaps * caps)
{
/* We wait for the alpha_buffer to be consumed, so that we don't pick the
* caps too soon */
g_mutex_lock (&self->buffer_lock);
/* We wait for the alpha_buffer to be consumed and store the buffer for the
* sink_chain to pick it up */
while (self->alpha_buffer && !self->flushing)
g_cond_wait (&self->buffer_cond, &self->buffer_lock);
if (self->flushing) {
g_mutex_unlock (&self->buffer_lock);
return GST_FLOW_FLUSHING;
}
if (!gst_video_info_from_caps (&self->alpha_vinfo, caps)) {
g_mutex_unlock (&self->buffer_lock);
GST_ELEMENT_ERROR (self, STREAM, FORMAT, ("Invalid video format"), (NULL));
return FALSE;
}
self->alpha_format_cookie++;
/* wait for the matching format change on the sink pad */
while (self->alpha_format_cookie != self->sink_format_cookie &&
!self->flushing)
g_cond_wait (&self->buffer_cond, &self->buffer_lock);
g_mutex_unlock (&self->buffer_lock);
return TRUE;
}
static gboolean
gst_alpha_combine_sink_event (GstPad * pad, GstObject * object,
GstEvent * event)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
switch (event->type) {
case GST_EVENT_FLUSH_START:
gst_alpha_combine_unlock (self);
break;
case GST_EVENT_FLUSH_STOP:
gst_alpha_combine_unlock_stop (self);
break;
case GST_EVENT_CAPS:
{
GstCaps *caps;
gboolean ret;
gst_event_parse_caps (event, &caps);
ret = gst_alpha_combine_set_sink_format (self, caps);
gst_event_unref (event);
return ret;
}
default:
break;
}
return gst_pad_event_default (pad, object, event);
}
static gboolean
gst_alpha_combine_alpha_event (GstPad * pad, GstObject * object,
GstEvent * event)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
switch (event->type) {
case GST_EVENT_FLUSH_START:
gst_alpha_combine_unlock (self);
break;
case GST_EVENT_FLUSH_STOP:
gst_alpha_combine_unlock_stop (self);
gst_alpha_combine_reset (self);
break;
case GST_EVENT_CAPS:
{
GstCaps *caps;
gst_event_parse_caps (event, &caps);
gst_alpha_combine_set_alpha_format (self, caps);
}
default:
break;
}
/* Events are being duplicated, over both branches, so let's just drop this
* secondary stream and use the one from the main stream. */
gst_event_unref (event);
return TRUE;
}
static gboolean
gst_alpha_combine_sink_query (GstPad * pad, GstObject * object,
GstQuery * query)
{
switch (query->type) {
case GST_QUERY_ALLOCATION:
{
int i;
if (!gst_pad_query_default (pad, object, query))
return FALSE;
/* Ensure NULL pool because it cannot be shared between the 2 decoders.
* Ideally, we should cache the downstream query and use it for both
* decoders, but it is hard to know when we should refresh it */
for (i = 0; i < gst_query_get_n_allocation_pools (query); i++) {
guint size = 0, min = 0, max = 0;
gst_query_parse_nth_allocation_pool (query, i, NULL, &size, &min, &max);
gst_query_set_nth_allocation_pool (query, i, NULL, size, min, max);
}
return TRUE;
break;
}
default:
break;
}
return gst_pad_query_default (pad, object, query);
}
static GstStateChangeReturn
gst_alpha_combine_change_state (GstElement * element, GstStateChange transition)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (element);
GstStateChangeReturn ret;
switch (transition) {
case GST_STATE_CHANGE_READY_TO_PAUSED:
gst_alpha_combine_unlock_stop (self);
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
gst_alpha_combine_unlock (self);
break;
default:
break;
}
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
switch (transition) {
case GST_STATE_CHANGE_PAUSED_TO_READY:
gst_alpha_combine_reset (self);
self->src_format = GST_VIDEO_FORMAT_UNKNOWN;
gst_video_info_init (&self->sink_vinfo);
gst_video_info_init (&self->alpha_vinfo);
self->sink_format_cookie = 0;
self->alpha_format_cookie = 0;
break;
default:
break;
}
return ret;
}
static void
gst_alpha_combine_dispose (GObject * object)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
g_clear_object (&self->sink_pad);
g_clear_object (&self->alpha_pad);
g_clear_object (&self->src_pad);
G_OBJECT_CLASS (parent_class)->dispose (object);
}
static void
gst_alpha_combine_finalize (GObject * object)
{
GstAlphaCombine *self = GST_ALPHA_COMBINE (object);
g_mutex_clear (&self->buffer_lock);
g_cond_clear (&self->buffer_cond);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_alpha_combine_class_init (GstAlphaCombineClass * klass)
{
GstElementClass *element_class = (GstElementClass *) klass;
GObjectClass *object_class = (GObjectClass *) klass;
gst_element_class_set_static_metadata (element_class,
"Alpha Combiner", "Codec/Demuxer",
"Use luma from an opaque stream as alpha plane on another",
"Nicolas Dufresne <nicolas.dufresne@collabora.com>");
gst_element_class_add_static_pad_template (element_class,
&gst_alpha_combine_sink_template);
gst_element_class_add_static_pad_template (element_class,
&gst_alpha_combine_alpha_template);
gst_element_class_add_static_pad_template (element_class,
&gst_alpha_combine_src_template);
element_class->change_state =
GST_DEBUG_FUNCPTR (gst_alpha_combine_change_state);
object_class->dispose = GST_DEBUG_FUNCPTR (gst_alpha_combine_dispose);
object_class->finalize = GST_DEBUG_FUNCPTR (gst_alpha_combine_finalize);
}
static void
gst_alpha_combine_init (GstAlphaCombine * self)
{
gst_element_create_all_pads (GST_ELEMENT (self));
self->sink_pad = gst_element_get_static_pad (GST_ELEMENT (self), "sink");
self->alpha_pad = gst_element_get_static_pad (GST_ELEMENT (self), "alpha");
self->src_pad = gst_element_get_static_pad (GST_ELEMENT (self), "src");
self->flushing = 1;
g_mutex_init (&self->buffer_lock);
g_cond_init (&self->buffer_cond);
GST_PAD_SET_PROXY_SCHEDULING (self->sink_pad);
GST_PAD_SET_PROXY_SCHEDULING (self->src_pad);
GST_PAD_SET_PROXY_ALLOCATION (self->sink_pad);
GST_PAD_SET_PROXY_ALLOCATION (self->alpha_pad);
gst_pad_set_chain_function (self->sink_pad, gst_alpha_combine_sink_chain);
gst_pad_set_chain_function (self->alpha_pad, gst_alpha_combine_alpha_chain);
gst_pad_set_event_function (self->sink_pad, gst_alpha_combine_sink_event);
gst_pad_set_event_function (self->alpha_pad, gst_alpha_combine_alpha_event);
gst_pad_set_query_function (self->sink_pad, gst_alpha_combine_sink_query);
gst_pad_set_query_function (self->alpha_pad, gst_alpha_combine_sink_query);
}