From 4b6c3c9a1baf8aea98bc87f30507684494568e3b Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Tue, 16 Jun 2020 12:01:30 +0200 Subject: [PATCH] level: add GstRTPAudioLevelMeta on buffers This meta can be used by a RTP payloader to send the level information to the peer. Part of https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/issues/446 Part-of: --- docs/gst_plugins_cache.json | 12 +++++++++ gst/level/gstlevel.c | 47 ++++++++++++++++++++++++++++++++++- gst/level/gstlevel.h | 1 + tests/check/elements/level.c | 48 ++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 1 deletion(-) diff --git a/docs/gst_plugins_cache.json b/docs/gst_plugins_cache.json index ab669167ee..13a57df5ed 100644 --- a/docs/gst_plugins_cache.json +++ b/docs/gst_plugins_cache.json @@ -8868,6 +8868,18 @@ } }, "properties": { + "audio-level-meta": { + "blurb": "Set GstAudioLevelMeta on buffers", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + }, "interval": { "blurb": "Interval of time between message posts (in nanoseconds)", "conditionally-available": false, diff --git a/gst/level/gstlevel.c b/gst/level/gstlevel.c index a5125799b9..67e109df0e 100644 --- a/gst/level/gstlevel.c +++ b/gst/level/gstlevel.c @@ -98,7 +98,8 @@ enum PROP_MESSAGE, PROP_INTERVAL, PROP_PEAK_TTL, - PROP_PEAK_FALLOFF + PROP_PEAK_FALLOFF, + PROP_AUDIO_LEVEL_META, }; #define gst_level_parent_class parent_class @@ -171,6 +172,17 @@ gst_level_class_init (GstLevelClass * klass) g_param_spec_double ("peak-falloff", "Peak Falloff", "Decay rate of decay peak after TTL (in dB/sec)", 0.0, G_MAXDOUBLE, 10.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** + * GstLevel:audio-level-meta: + * + * If %TRUE, generate or update GstAudioLevelMeta on output buffers. + * + * Since: 1.20 + */ + g_object_class_install_property (gobject_class, PROP_AUDIO_LEVEL_META, + g_param_spec_boolean ("audio-level-meta", "Audio Level Meta", + "Set GstAudioLevelMeta on buffers", FALSE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); GST_DEBUG_CATEGORY_INIT (level_debug, "level", 0, "Level calculation"); @@ -262,6 +274,9 @@ gst_level_set_property (GObject * object, guint prop_id, case PROP_PEAK_FALLOFF: filter->decay_peak_falloff = g_value_get_double (value); break; + case PROP_AUDIO_LEVEL_META: + filter->audio_level_meta = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -289,6 +304,9 @@ gst_level_get_property (GObject * object, guint prop_id, case PROP_PEAK_FALLOFF: g_value_set_double (value, filter->decay_peak_falloff); break; + case PROP_AUDIO_LEVEL_META: + g_value_set_boolean (value, filter->audio_level_meta); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -546,6 +564,24 @@ gst_level_message_append_channel (GstMessage * m, gdouble rms, gdouble peak, g_value_unset (&v); } +static void +gst_level_rtp_audio_level_meta (GstLevel * self, GstBuffer * buffer, + guint8 level) +{ + GstAudioLevelMeta *meta; + + /* Update the existing meta, if any, so we can have an upstream element + * filling the voice activity part of the meta. */ + meta = gst_buffer_get_audio_level_meta (buffer); + if (meta) { + meta->level = level; + } else { + /* Assume audio does not contain voice, it can be detected by another + * downstream element. */ + gst_buffer_add_audio_level_meta (buffer, level, FALSE); + } +} + static GstFlowReturn gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) { @@ -562,6 +598,7 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) * intervals */ GstClockTimeDiff falloff_time; gint channels, rate, bps; + gdouble CS_tot = 0; /* Total Cumulative Square on all samples */ filter = GST_LEVEL (trans); @@ -598,6 +635,7 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) if (!GST_BUFFER_FLAG_IS_SET (in, GST_BUFFER_FLAG_GAP)) { filter->process (in_data + (bps * i), block_int_size, channels, &CS, &filter->peak[i]); + CS_tot += CS; GST_LOG_OBJECT (filter, "[%d]: cumulative squares %lf, over %d samples/%d channels", i, CS, block_int_size, channels); @@ -664,6 +702,13 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in) gst_buffer_unmap (in, &map); + if (filter->audio_level_meta) { + gdouble RMS = sqrt (CS_tot / num_int_samples); + gdouble RMSdB = 20 * log10 (RMS + EPSILON); + + gst_level_rtp_audio_level_meta (filter, in, -RMSdB); + } + return GST_FLOW_OK; } diff --git a/gst/level/gstlevel.h b/gst/level/gstlevel.h index e6d927ae6d..3fe4c72bc1 100644 --- a/gst/level/gstlevel.h +++ b/gst/level/gstlevel.h @@ -67,6 +67,7 @@ struct _GstLevel { * since last emit */ gint interval_frames; /* after how many frame to sent a message */ GstClockTime message_ts; /* starttime for next message */ + gboolean audio_level_meta; /* whether or not generate GstAudioLevelMeta */ /* per-channel arrays for intermediate values */ gdouble *CS; /* normalized Cumulative Square */ diff --git a/tests/check/elements/level.c b/tests/check/elements/level.c index d816c1a5fa..ac79fa41ce 100644 --- a/tests/check/elements/level.c +++ b/tests/check/elements/level.c @@ -588,6 +588,53 @@ GST_START_TEST (test_message_timestamps) GST_END_TEST; +GST_START_TEST (test_rtp_audio_level_meta) +{ + GstElement *level; + GstBuffer *inbuffer, *outbuffer; + GstAudioLevelMeta *meta; + + level = setup_level (LEVEL_S16_CAPS_STRING); + g_object_set (level, "post-messages", FALSE, "audio-level-meta", TRUE, NULL); + gst_element_set_state (level, GST_STATE_PLAYING); + + /* create a fake 0.1 sec buffer with a half-amplitude block signal */ + inbuffer = create_s16_buffer (16536, 16536); + + fail_unless (gst_pad_push (mysrcpad, inbuffer) == GST_FLOW_OK); + fail_unless_equals_int (g_list_length (buffers), 1); + fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL); + fail_unless (inbuffer == outbuffer); + + /* level added the meta */ + meta = gst_buffer_get_audio_level_meta (outbuffer); + fail_unless (meta); + fail_unless_equals_int (meta->voice_activity, 0); + fail_unless_equals_int (meta->level, 5); + + /* same but with input buffer already having the meta so level will update it */ + inbuffer = create_s16_buffer (16536, 16536); + gst_buffer_add_audio_level_meta (inbuffer, 0, TRUE); + + fail_unless (gst_pad_push (mysrcpad, inbuffer) == GST_FLOW_OK); + fail_unless_equals_int (g_list_length (buffers), 2); + fail_if ((outbuffer = (GstBuffer *) buffers->next->data) == NULL); + fail_unless (inbuffer == outbuffer); + + /* level updated the meta */ + meta = gst_buffer_get_audio_level_meta (outbuffer); + fail_unless (meta); + fail_unless_equals_int (meta->voice_activity, 1); + fail_unless_equals_int (meta->level, 5); + + /* clean up */ + /* flush current messages,and future state change messages */ + gst_element_set_state (level, GST_STATE_NULL); + cleanup_level (level); +} + +GST_END_TEST; + static Suite * level_suite (void) { @@ -603,6 +650,7 @@ level_suite (void) tcase_add_test (tc_chain, test_message_on_eos); tcase_add_test (tc_chain, test_message_count); tcase_add_test (tc_chain, test_message_timestamps); + tcase_add_test (tc_chain, test_rtp_audio_level_meta); return s; }