level: add GstRTPAudioLevelMeta on buffers

This meta can be used by a RTP payloader to send the level information
to the peer.

Part of https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/issues/446

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-good/-/merge_requests/630>

docs/gst_plugins_cache.json

@@ -8868,6 +8868,18 @@
                     }
                 },
                 "properties": {
+                    "audio-level-meta": {
+                        "blurb": "Set GstAudioLevelMeta on buffers",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "false",
+                        "mutable": "null",
+                        "readable": true,
+                        "type": "gboolean",
+                        "writable": true
+                    },
                     "interval": {
                         "blurb": "Interval of time between message posts (in nanoseconds)",
                         "conditionally-available": false,

gst/level/gstlevel.c

@@ -98,7 +98,8 @@ enum
   PROP_MESSAGE,
   PROP_INTERVAL,
   PROP_PEAK_TTL,
-  PROP_PEAK_FALLOFF
+  PROP_PEAK_FALLOFF,
+  PROP_AUDIO_LEVEL_META,
 };
 
 #define gst_level_parent_class parent_class
@@ -171,6 +172,17 @@ gst_level_class_init (GstLevelClass * klass)
       g_param_spec_double ("peak-falloff", "Peak Falloff",
           "Decay rate of decay peak after TTL (in dB/sec)",
           0.0, G_MAXDOUBLE, 10.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+  /**
+   * GstLevel:audio-level-meta:
+   *
+   * If %TRUE, generate or update GstAudioLevelMeta on output buffers.
+   *
+   * Since: 1.20
+   */
+  g_object_class_install_property (gobject_class, PROP_AUDIO_LEVEL_META,
+      g_param_spec_boolean ("audio-level-meta", "Audio Level Meta",
+          "Set GstAudioLevelMeta on buffers", FALSE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
   GST_DEBUG_CATEGORY_INIT (level_debug, "level", 0, "Level calculation");
 
@@ -262,6 +274,9 @@ gst_level_set_property (GObject * object, guint prop_id,
     case PROP_PEAK_FALLOFF:
       filter->decay_peak_falloff = g_value_get_double (value);
       break;
+    case PROP_AUDIO_LEVEL_META:
+      filter->audio_level_meta = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -289,6 +304,9 @@ gst_level_get_property (GObject * object, guint prop_id,
     case PROP_PEAK_FALLOFF:
       g_value_set_double (value, filter->decay_peak_falloff);
       break;
+    case PROP_AUDIO_LEVEL_META:
+      g_value_set_boolean (value, filter->audio_level_meta);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -546,6 +564,24 @@ gst_level_message_append_channel (GstMessage * m, gdouble rms, gdouble peak,
   g_value_unset (&v);
 }
 
+static void
+gst_level_rtp_audio_level_meta (GstLevel * self, GstBuffer * buffer,
+    guint8 level)
+{
+  GstAudioLevelMeta *meta;
+
+  /* Update the existing meta, if any, so we can have an upstream element
+   * filling the voice activity part of the meta. */
+  meta = gst_buffer_get_audio_level_meta (buffer);
+  if (meta) {
+    meta->level = level;
+  } else {
+    /* Assume audio does not contain voice, it can be detected by another
+     * downstream element. */
+    gst_buffer_add_audio_level_meta (buffer, level, FALSE);
+  }
+}
+
 static GstFlowReturn
 gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in)
 {
@@ -562,6 +598,7 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in)
                                          * intervals */
   GstClockTimeDiff falloff_time;
   gint channels, rate, bps;
+  gdouble CS_tot = 0;           /* Total Cumulative Square on all samples */
 
   filter = GST_LEVEL (trans);
 
@@ -598,6 +635,7 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in)
       if (!GST_BUFFER_FLAG_IS_SET (in, GST_BUFFER_FLAG_GAP)) {
         filter->process (in_data + (bps * i), block_int_size, channels, &CS,
             &filter->peak[i]);
+        CS_tot += CS;
         GST_LOG_OBJECT (filter,
             "[%d]: cumulative squares %lf, over %d samples/%d channels",
             i, CS, block_int_size, channels);
@@ -664,6 +702,13 @@ gst_level_transform_ip (GstBaseTransform * trans, GstBuffer * in)
 
   gst_buffer_unmap (in, &map);
 
+  if (filter->audio_level_meta) {
+    gdouble RMS = sqrt (CS_tot / num_int_samples);
+    gdouble RMSdB = 20 * log10 (RMS + EPSILON);
+
+    gst_level_rtp_audio_level_meta (filter, in, -RMSdB);
+  }
+
   return GST_FLOW_OK;
 }
 

gst/level/gstlevel.h

@@ -67,6 +67,7 @@ struct _GstLevel {
                                  * since last emit */
   gint interval_frames;         /* after how many frame to sent a message */
   GstClockTime message_ts;      /* starttime for next message */
+  gboolean audio_level_meta; /* whether or not generate GstAudioLevelMeta */
 
   /* per-channel arrays for intermediate values */
   gdouble *CS;                  /* normalized Cumulative Square */

tests/check/elements/level.c

@@ -588,6 +588,53 @@ GST_START_TEST (test_message_timestamps)
 
 GST_END_TEST;
 
+GST_START_TEST (test_rtp_audio_level_meta)
+{
+  GstElement *level;
+  GstBuffer *inbuffer, *outbuffer;
+  GstAudioLevelMeta *meta;
+
+  level = setup_level (LEVEL_S16_CAPS_STRING);
+  g_object_set (level, "post-messages", FALSE, "audio-level-meta", TRUE, NULL);
+  gst_element_set_state (level, GST_STATE_PLAYING);
+
+  /* create a fake 0.1 sec buffer with a half-amplitude block signal */
+  inbuffer = create_s16_buffer (16536, 16536);
+
+  fail_unless (gst_pad_push (mysrcpad, inbuffer) == GST_FLOW_OK);
+  fail_unless_equals_int (g_list_length (buffers), 1);
+  fail_if ((outbuffer = (GstBuffer *) buffers->data) == NULL);
+  fail_unless (inbuffer == outbuffer);
+
+  /* level added the meta */
+  meta = gst_buffer_get_audio_level_meta (outbuffer);
+  fail_unless (meta);
+  fail_unless_equals_int (meta->voice_activity, 0);
+  fail_unless_equals_int (meta->level, 5);
+
+  /* same but with input buffer already having the meta so level will update it */
+  inbuffer = create_s16_buffer (16536, 16536);
+  gst_buffer_add_audio_level_meta (inbuffer, 0, TRUE);
+
+  fail_unless (gst_pad_push (mysrcpad, inbuffer) == GST_FLOW_OK);
+  fail_unless_equals_int (g_list_length (buffers), 2);
+  fail_if ((outbuffer = (GstBuffer *) buffers->next->data) == NULL);
+  fail_unless (inbuffer == outbuffer);
+
+  /* level updated the meta */
+  meta = gst_buffer_get_audio_level_meta (outbuffer);
+  fail_unless (meta);
+  fail_unless_equals_int (meta->voice_activity, 1);
+  fail_unless_equals_int (meta->level, 5);
+
+  /* clean up */
+  /* flush current messages,and future state change messages */
+  gst_element_set_state (level, GST_STATE_NULL);
+  cleanup_level (level);
+}
+
+GST_END_TEST;
+
 static Suite *
 level_suite (void)
 {
@@ -603,6 +650,7 @@ level_suite (void)
   tcase_add_test (tc_chain, test_message_on_eos);
   tcase_add_test (tc_chain, test_message_count);
   tcase_add_test (tc_chain, test_message_timestamps);
+  tcase_add_test (tc_chain, test_rtp_audio_level_meta);
 
   return s;
 }