audioaggregator: refactor conversion API

For the rationale, see:

https://bugzilla.gnome.org/show_bug.cgi?id=793917

Also test audiomixer conversion of current output buffer
This commit is contained in:
Mathieu Duponchelle 2018-02-28 18:13:10 +01:00
parent c920d994ab
commit 10835e9919
5 changed files with 207 additions and 99 deletions

View file

@ -28,22 +28,24 @@
* @title: GstAudioAggregator
* @short_description: Base class that manages a set of audio input pads
* with the purpose of aggregating or mixing their raw audio input buffers
* @see_also: #GstAggregator
* @see_also: #GstAggregator, #GstAudioMixer
*
* #GstAudioAggregator will perform conversion on the data arriving
* on its sink pads, based on the format expected downstream.
* Subclasses must use (a subclass of) #GstAudioAggregatorPad for both
* their source and sink pads,
* gst_element_class_add_static_pad_template_with_gtype() is a convenient
* helper.
*
* Subclasses can opt out of the conversion behaviour by setting
* #GstAudioAggregatorClass.convert_buffer() to %NULL.
* #GstAudioAggregator can perform conversion on the data arriving
* on its sink pads, based on the format expected downstream: in order
* to enable that behaviour, the GType of the sink pads must either be
* a (subclass of) #GstAudioAggregatorConvertPad to use the default
* #GstAudioConverter implementation, or a subclass of #GstAudioAggregatorPad
* implementing #GstAudioAggregatorPad.convert_buffer.
*
* Subclasses that wish to use the default conversion implementation
* should use a (subclass of) #GstAudioAggregatorConvertPad as their
* #GstAggregatorClass.sinkpads_type, as it will cache the created
* #GstAudioConverter and install a property allowing to configure it,
* #GstAudioAggregatorPad:converter-config.
* To allow for the output caps to change, the mechanism is the same as
* above, with the GType of the source pad.
*
* Subclasses that wish to perform custom conversion should override
* #GstAudioAggregatorClass.convert_buffer().
* See #GstAudioMixer for an example.
*
* When conversion is enabled, #GstAudioAggregator will accept
* any type of raw audio caps and perform conversion
@ -54,10 +56,6 @@
* the first configured sink pad to finish fixating its source pad
* caps.
*
* Additionally, handling audio conversion directly in the element
* means that this base class supports safely reconfiguring its
* source pad.
*
* A notable exception for now is the sample rate, sink pads must
* have the same sample rate as either the downstream requirement,
* or the first configured pad, or a combination of both (when
@ -223,12 +221,22 @@ gst_audio_aggregator_convert_pad_update_converter (GstAudioAggregatorConvertPad
aaggcpad->priv->converter_config_changed = FALSE;
}
static void
gst_audio_aggregator_pad_update_conversion_info (GstAudioAggregatorPad *
aaggpad)
{
GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad)->priv->converter_config_changed =
TRUE;
}
static GstBuffer *
gst_audio_aggregator_convert_pad_convert_buffer (GstAudioAggregatorConvertPad *
aaggcpad, GstAudioInfo * in_info, GstAudioInfo * out_info,
gst_audio_aggregator_convert_pad_convert_buffer (GstAudioAggregatorPad *
aaggpad, GstAudioInfo * in_info, GstAudioInfo * out_info,
GstBuffer * input_buffer)
{
GstBuffer *res;
GstAudioAggregatorConvertPad *aaggcpad =
GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad);
gst_audio_aggregator_convert_pad_update_converter (aaggcpad, in_info,
out_info);
@ -327,6 +335,8 @@ gst_audio_aggregator_convert_pad_class_init (GstAudioAggregatorConvertPadClass *
klass)
{
GObjectClass *gobject_class = (GObjectClass *) klass;
GstAudioAggregatorPadClass *aaggpad_class =
(GstAudioAggregatorPadClass *) klass;
g_type_class_add_private (klass,
sizeof (GstAudioAggregatorConvertPadPrivate));
@ -339,6 +349,12 @@ gst_audio_aggregator_convert_pad_class_init (GstAudioAggregatorConvertPadClass *
"when converting this pad's audio buffers",
GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
aaggpad_class->convert_buffer =
gst_audio_aggregator_convert_pad_convert_buffer;
aaggpad_class->update_conversion_info =
gst_audio_aggregator_pad_update_conversion_info;
gobject_class->finalize = gst_audio_aggregator_convert_pad_finalize;
}
@ -449,64 +465,16 @@ gst_audio_aggregator_get_next_time (GstAggregator * agg)
return next_time;
}
static GstBuffer *
gst_audio_aggregator_convert_once (GstAudioAggregator * aagg, GstPad * pad,
GstAudioInfo * in_info, GstAudioInfo * out_info, GstBuffer * buffer)
{
GstAudioConverter *converter =
gst_audio_converter_new (GST_AUDIO_CONVERTER_FLAG_NONE,
in_info, out_info, NULL);
gint insize = gst_buffer_get_size (buffer);
gsize insamples = insize / in_info->bpf;
gsize outsamples = gst_audio_converter_get_out_frames (converter,
insamples);
gint outsize = outsamples * out_info->bpf;
GstMapInfo inmap, outmap;
GstBuffer *converted = gst_buffer_new_allocate (NULL, outsize, NULL);
gst_buffer_copy_into (converted, buffer,
GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS |
GST_BUFFER_COPY_META, 0, -1);
gst_buffer_map (buffer, &inmap, GST_MAP_READ);
gst_buffer_map (converted, &outmap, GST_MAP_WRITE);
gst_audio_converter_samples (converter,
GST_AUDIO_CONVERTER_FLAG_NONE,
(gpointer *) & inmap.data, insamples,
(gpointer *) & outmap.data, outsamples);
gst_buffer_unmap (buffer, &inmap);
gst_buffer_unmap (converted, &outmap);
gst_audio_converter_free (converter);
return converted;
}
static GstBuffer *
gst_audio_aggregator_default_convert_buffer (GstAudioAggregator * aagg,
GstPad * pad, GstAudioInfo * in_info, GstAudioInfo * out_info,
GstBuffer * buffer)
{
if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (pad))
return
gst_audio_aggregator_convert_pad_convert_buffer
(GST_AUDIO_AGGREGATOR_CONVERT_PAD (pad),
&GST_AUDIO_AGGREGATOR_PAD (pad)->info, out_info, buffer);
else
return gst_audio_aggregator_convert_once (aagg, pad, in_info, out_info,
buffer);
}
static GstBuffer *
gst_audio_aggregator_convert_buffer (GstAudioAggregator * aagg, GstPad * pad,
GstAudioInfo * in_info, GstAudioInfo * out_info, GstBuffer * buffer)
{
GstAudioAggregatorClass *klass = GST_AUDIO_AGGREGATOR_GET_CLASS (aagg);
GstAudioAggregatorPadClass *klass = GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (pad);
GstAudioAggregatorPad *aaggpad = GST_AUDIO_AGGREGATOR_PAD (pad);
g_assert (klass->convert_buffer);
return klass->convert_buffer (aagg, pad, in_info, out_info, buffer);
return klass->convert_buffer (aaggpad, in_info, out_info, buffer);
}
static void
@ -542,7 +510,6 @@ gst_audio_aggregator_class_init (GstAudioAggregatorClass * klass)
gst_audio_aggregator_negotiated_src_caps;
klass->create_output_buffer = gst_audio_aggregator_create_output_buffer;
klass->convert_buffer = gst_audio_aggregator_default_convert_buffer;
GST_DEBUG_CATEGORY_INIT (audio_aggregator_debug, "audioaggregator",
GST_DEBUG_FG_MAGENTA, "GstAudioAggregator");
@ -746,11 +713,12 @@ gst_audio_aggregator_sink_setcaps (GstAudioAggregatorPad * aaggpad,
gst_pad_push_event (GST_PAD (aaggpad), gst_event_new_reconfigure ());
ret = FALSE;
} else {
GstAudioAggregatorPadClass *klass =
GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (aaggpad);
GST_OBJECT_LOCK (aaggpad);
gst_audio_info_from_caps (&aaggpad->info, caps);
if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad))
GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad)->
priv->converter_config_changed = TRUE;
if (klass->update_conversion_info)
klass->update_conversion_info (aaggpad);
GST_OBJECT_UNLOCK (aaggpad);
}
@ -792,10 +760,9 @@ gst_audio_aggregator_update_src_caps (GstAggregator * agg,
static GstCaps *
gst_audio_aggregator_fixate_src_caps (GstAggregator * agg, GstCaps * caps)
{
GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (agg);
GstAudioAggregatorPad *first_configured_pad;
if (!aaggclass->convert_buffer)
if (!GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (agg->srcpad)->convert_buffer)
return
GST_AGGREGATOR_CLASS
(gst_audio_aggregator_parent_class)->fixate_src_caps (agg, caps);
@ -844,10 +811,11 @@ gst_audio_aggregator_update_converters (GstAudioAggregator * aagg,
for (l = GST_ELEMENT (aagg)->sinkpads; l; l = l->next) {
GstAudioAggregatorPad *aaggpad = l->data;
GstAudioAggregatorPadClass *klass =
GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (aaggpad);
if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad))
GST_AUDIO_AGGREGATOR_CONVERT_PAD (aaggpad)->
priv->converter_config_changed = TRUE;
if (klass->update_conversion_info)
klass->update_conversion_info (aaggpad);
/* If we currently were mixing a buffer, we need to convert it to the new
* format */
@ -865,7 +833,6 @@ static gboolean
gst_audio_aggregator_negotiated_src_caps (GstAggregator * agg, GstCaps * caps)
{
GstAudioAggregator *aagg = GST_AUDIO_AGGREGATOR (agg);
GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (agg);
GstAudioInfo info;
GST_INFO_OBJECT (agg, "src caps negotiated %" GST_PTR_FORMAT, caps);
@ -878,12 +845,19 @@ gst_audio_aggregator_negotiated_src_caps (GstAggregator * agg, GstCaps * caps)
GST_AUDIO_AGGREGATOR_LOCK (aagg);
GST_OBJECT_LOCK (aagg);
if (aaggclass->convert_buffer) {
if (GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (agg->srcpad)->convert_buffer) {
gst_audio_aggregator_update_converters (aagg, &info);
if (aagg->priv->current_buffer
&& !gst_audio_info_is_equal (&aagg->info, &info)) {
GstBuffer *converted =
GstBuffer *converted;
GstAudioAggregatorPadClass *klass =
GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (agg->srcpad);
if (klass->update_conversion_info)
klass->update_conversion_info (GST_AUDIO_AGGREGATOR_PAD (agg->srcpad));
converted =
gst_audio_aggregator_convert_buffer (aagg, agg->srcpad, &aagg->info,
&info, aagg->priv->current_buffer);
gst_buffer_unref (aagg->priv->current_buffer);
@ -1324,7 +1298,6 @@ static gboolean
gst_audio_aggregator_fill_buffer (GstAudioAggregator * aagg,
GstAudioAggregatorPad * pad)
{
GstAudioAggregatorClass *aaggclass = GST_AUDIO_AGGREGATOR_GET_CLASS (aagg);
GstClockTime start_time, end_time;
gboolean discont = FALSE;
guint64 start_offset, end_offset;
@ -1333,7 +1306,7 @@ gst_audio_aggregator_fill_buffer (GstAudioAggregator * aagg,
GstAggregator *agg = GST_AGGREGATOR (aagg);
GstAggregatorPad *aggpad = GST_AGGREGATOR_PAD (pad);
if (aaggclass->convert_buffer) {
if (GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (pad)->convert_buffer) {
rate = GST_AUDIO_INFO_RATE (&aagg->info);
bpf = GST_AUDIO_INFO_BPF (&aagg->info);
} else {
@ -1802,7 +1775,7 @@ gst_audio_aggregator_aggregate (GstAggregator * agg, gboolean timeout)
/* New buffer? */
if (!pad->priv->buffer) {
if (GST_IS_AUDIO_AGGREGATOR_CONVERT_PAD (pad))
if (GST_AUDIO_AGGREGATOR_PAD_GET_CLASS (pad)->convert_buffer)
pad->priv->buffer =
gst_audio_aggregator_convert_buffer
(aagg, GST_PAD (pad), &pad->info, &aagg->info,

View file

@ -79,12 +79,21 @@ struct _GstAudioAggregatorPad
/**
* GstAudioAggregatorPadClass:
*
* @convert_buffer: Convert a buffer from one format to another.
* @update_conversion_info: Called when either the input or output
* formats have changed.
*/
struct _GstAudioAggregatorPadClass
{
GstAggregatorPadClass parent_class;
GstBuffer * (* convert_buffer) (GstAudioAggregatorPad * pad,
GstAudioInfo *in_info,
GstAudioInfo *out_info,
GstBuffer * buffer);
void (* update_conversion_info) (GstAudioAggregatorPad *pad);
/*< private >*/
gpointer _gst_reserved[GST_PADDING_LARGE];
};
@ -181,10 +190,6 @@ struct _GstAudioAggregator
* buffer. The in_offset and out_offset are in "frames", which is
* the size of a sample times the number of channels. Returns TRUE if
* any non-silence was added to the buffer
* @convert_buffer: Convert a buffer from one format to another. The pad
* is either a sinkpad, when converting an input buffer, or the source pad,
* when converting the output buffer after a downstream format change is
* requested.
*/
struct _GstAudioAggregatorClass {
GstAggregatorClass parent_class;
@ -194,11 +199,6 @@ struct _GstAudioAggregatorClass {
gboolean (* aggregate_one_buffer) (GstAudioAggregator * aagg,
GstAudioAggregatorPad * pad, GstBuffer * inbuf, guint in_offset,
GstBuffer * outbuf, guint out_offset, guint num_frames);
GstBuffer * (* convert_buffer) (GstAudioAggregator *aagg,
GstPad * pad,
GstAudioInfo *in_info,
GstAudioInfo *out_info,
GstBuffer * buffer);
/*< private >*/
gpointer _gst_reserved[GST_PADDING_LARGE];

View file

@ -560,8 +560,8 @@ gst_audio_interleave_class_init (GstAudioInterleaveClass * klass)
gobject_class->get_property = gst_audio_interleave_get_property;
gobject_class->finalize = gst_audio_interleave_finalize;
gst_element_class_add_static_pad_template (gstelement_class,
&gst_audio_interleave_src_template);
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
&gst_audio_interleave_src_template, GST_TYPE_AUDIO_AGGREGATOR_PAD);
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
&gst_audio_interleave_sink_template, GST_TYPE_AUDIO_INTERLEAVE_PAD);
gst_element_class_set_static_metadata (gstelement_class, "AudioInterleave",
@ -580,7 +580,6 @@ gst_audio_interleave_class_init (GstAudioInterleaveClass * klass)
agg_class->negotiated_src_caps = gst_audio_interleave_negotiated_src_caps;
aagg_class->aggregate_one_buffer = gst_audio_interleave_aggregate_one_buffer;
aagg_class->convert_buffer = NULL;
/**
* GstInterleave:channel-positions

View file

@ -224,8 +224,8 @@ gst_audiomixer_class_init (GstAudioMixerClass * klass)
GstElementClass *gstelement_class = (GstElementClass *) klass;
GstAudioAggregatorClass *aagg_class = (GstAudioAggregatorClass *) klass;
gst_element_class_add_static_pad_template (gstelement_class,
&gst_audiomixer_src_template);
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
&gst_audiomixer_src_template, GST_TYPE_AUDIO_AGGREGATOR_CONVERT_PAD);
gst_element_class_add_static_pad_template_with_gtype (gstelement_class,
&gst_audiomixer_sink_template, GST_TYPE_AUDIO_MIXER_PAD);
gst_element_class_set_static_metadata (gstelement_class, "AudioMixer",

View file

@ -1849,6 +1849,141 @@ GST_START_TEST (test_change_output_caps)
GST_END_TEST;
/* In this test, we create two input buffers with a duration of 1 second,
* and require the audiomixer to output 1.5 second long buffers.
*
* After we have input two buffers, we change the output format
* from S8 to S32, then push a last buffer.
*
* This makes audioaggregator convert its "half-mixed" current_buffer,
* we can then ensure that the second output buffer is as expected.
*/
GST_START_TEST (test_change_output_caps_mid_output_buffer)
{
GstSegment segment;
GstElement *bin, *audiomixer, *capsfilter, *sink;
GstBus *bus;
GstPad *sinkpad;
gboolean res;
GstStateChangeReturn state_res;
GstFlowReturn ret;
GstEvent *event;
GstBuffer *buffer;
GstCaps *caps;
GstQuery *drain;
GstMapInfo inmap;
GstMapInfo outmap;
guint i;
bin = gst_pipeline_new ("pipeline");
bus = gst_element_get_bus (bin);
gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH);
g_signal_connect (bus, "message::error", (GCallback) message_received, bin);
g_signal_connect (bus, "message::warning", (GCallback) message_received, bin);
g_signal_connect (bus, "message::eos", (GCallback) message_received, bin);
audiomixer = gst_element_factory_make ("audiomixer", "audiomixer");
g_object_set (audiomixer, "output-buffer-duration", 1500 * GST_MSECOND, NULL);
capsfilter = gst_element_factory_make ("capsfilter", NULL);
sink = gst_element_factory_make ("fakesink", "sink");
gst_bin_add_many (GST_BIN (bin), audiomixer, capsfilter, sink, NULL);
res = gst_element_link_many (audiomixer, capsfilter, sink, NULL);
fail_unless (res == TRUE, NULL);
state_res = gst_element_set_state (bin, GST_STATE_PLAYING);
ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE);
sinkpad = gst_element_get_request_pad (audiomixer, "sink_%u");
fail_if (sinkpad == NULL, NULL);
gst_pad_send_event (sinkpad, gst_event_new_stream_start ("test"));
caps = gst_caps_new_simple ("audio/x-raw",
"format", G_TYPE_STRING, "S8",
"layout", G_TYPE_STRING, "interleaved",
"rate", G_TYPE_INT, 10, "channels", G_TYPE_INT, 1, NULL);
gst_pad_set_caps (sinkpad, caps);
g_object_set (capsfilter, "caps", caps, NULL);
gst_caps_unref (caps);
gst_segment_init (&segment, GST_FORMAT_TIME);
segment.start = 0;
segment.stop = 3 * GST_SECOND;
segment.time = 0;
event = gst_event_new_segment (&segment);
gst_pad_send_event (sinkpad, event);
buffer = new_buffer (10, 0, 0, 1 * GST_SECOND, 0);
ret = gst_pad_chain (sinkpad, buffer);
ck_assert_int_eq (ret, GST_FLOW_OK);
buffer = new_buffer (10, 0, 1 * GST_SECOND, 1 * GST_SECOND, 0);
gst_buffer_map (buffer, &inmap, GST_MAP_WRITE);
memset (inmap.data, 1, 10);
gst_buffer_unmap (buffer, &inmap);
ret = gst_pad_chain (sinkpad, buffer);
ck_assert_int_eq (ret, GST_FLOW_OK);
drain = gst_query_new_drain ();
gst_pad_query (sinkpad, drain);
gst_query_unref (drain);
caps = gst_caps_new_simple ("audio/x-raw",
"format", G_TYPE_STRING, GST_AUDIO_NE (S32),
"layout", G_TYPE_STRING, "interleaved",
"rate", G_TYPE_INT, 10, "channels", G_TYPE_INT, 1, NULL);
g_object_set (capsfilter, "caps", caps, NULL);
gst_caps_unref (caps);
gst_buffer_replace (&handoff_buffer, NULL);
g_object_set (sink, "signal-handoffs", TRUE, NULL);
g_signal_connect (sink, "handoff", (GCallback) handoff_buffer_cb, NULL);
buffer = new_buffer (10, 0, 2 * GST_SECOND, 1 * GST_SECOND, 0);
gst_buffer_map (buffer, &inmap, GST_MAP_WRITE);
memset (inmap.data, 0, 10);
gst_buffer_unmap (buffer, &inmap);
ret = gst_pad_chain (sinkpad, buffer);
ck_assert_int_eq (ret, GST_FLOW_OK);
drain = gst_query_new_drain ();
gst_pad_query (sinkpad, drain);
gst_query_unref (drain);
fail_unless (handoff_buffer);
fail_unless_equals_int (gst_buffer_get_size (handoff_buffer), 60);
gst_buffer_map (handoff_buffer, &outmap, GST_MAP_READ);
for (i = 0; i < 15; i++) {
guint32 sample;
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
sample = GUINT32_FROM_LE (((guint32 *) outmap.data)[i]);
#else
sample = GUINT32_FROM_BE (((guint32 *) outmap.data)[i]);
#endif
if (i < 5) {
fail_unless_equals_int (sample, 1 << 24);
} else {
fail_unless_equals_int (sample, 0);
}
}
gst_buffer_unmap (handoff_buffer, &outmap);
gst_element_release_request_pad (audiomixer, sinkpad);
gst_object_unref (sinkpad);
gst_element_set_state (bin, GST_STATE_NULL);
gst_bus_remove_signal_watch (bus);
gst_object_unref (bus);
gst_object_unref (bin);
}
GST_END_TEST;
static Suite *
audiomixer_suite (void)
{
@ -1876,6 +2011,7 @@ audiomixer_suite (void)
tcase_add_test (tc_chain, test_sinkpad_property_controller);
tcase_add_checked_fixture (tc_chain, test_setup, test_teardown);
tcase_add_test (tc_chain, test_change_output_caps);
tcase_add_test (tc_chain, test_change_output_caps_mid_output_buffer);
/* Use a longer timeout */
#ifdef HAVE_VALGRIND