interaudio: Make buffer size and latency handling more explicit and add properties for them

This now makes audio work more reliable without disconts.

gst/inter/gstinteraudiosink.c

@@ -47,9 +47,6 @@
 #include "gstinteraudiosink.h"
 #include <string.h>
 
-#define PERIOD    1600
-#define N_PERIODS 10
-
 GST_DEBUG_CATEGORY_STATIC (gst_inter_audio_sink_debug_category);
 #define GST_CAT_DEFAULT gst_inter_audio_sink_debug_category
 
@@ -68,6 +65,8 @@ static gboolean gst_inter_audio_sink_set_caps (GstBaseSink * sink,
     GstCaps * caps);
 static GstFlowReturn gst_inter_audio_sink_render (GstBaseSink * sink,
     GstBuffer * buffer);
+static gboolean gst_inter_audio_sink_query (GstBaseSink * sink,
+    GstQuery * query);
 
 enum
 {
@@ -83,7 +82,6 @@ GST_STATIC_PAD_TEMPLATE ("sink",
     GST_STATIC_CAPS (GST_AUDIO_CAPS_MAKE (GST_AUDIO_FORMATS_ALL))
     );
 
-
 /* class initialization */
 G_DEFINE_TYPE (GstInterAudioSink, gst_inter_audio_sink, GST_TYPE_BASE_SINK);
 
@@ -114,6 +112,7 @@ gst_inter_audio_sink_class_init (GstInterAudioSinkClass * klass)
   base_sink_class->stop = GST_DEBUG_FUNCPTR (gst_inter_audio_sink_stop);
   base_sink_class->set_caps = GST_DEBUG_FUNCPTR (gst_inter_audio_sink_set_caps);
   base_sink_class->render = GST_DEBUG_FUNCPTR (gst_inter_audio_sink_render);
+  base_sink_class->query = GST_DEBUG_FUNCPTR (gst_inter_audio_sink_query);
 
   g_object_class_install_property (gobject_class, PROP_CHANNEL,
       g_param_spec_string ("channel", "Channel",
@@ -201,6 +200,11 @@ gst_inter_audio_sink_start (GstBaseSink * sink)
   interaudiosink->surface = gst_inter_surface_get (interaudiosink->channel);
   g_mutex_lock (&interaudiosink->surface->mutex);
   memset (&interaudiosink->surface->audio_info, 0, sizeof (GstAudioInfo));
+
+  /* We want to write latency-time before syncing has happened */
+  /* FIXME: The other side can change this value when it starts */
+  gst_base_sink_set_render_delay (sink,
+      interaudiosink->surface->audio_latency_time);
   g_mutex_unlock (&interaudiosink->surface->mutex);
 
   return TRUE;
@@ -249,19 +253,32 @@ static GstFlowReturn
 gst_inter_audio_sink_render (GstBaseSink * sink, GstBuffer * buffer)
 {
   GstInterAudioSink *interaudiosink = GST_INTER_AUDIO_SINK (sink);
-  int n, bpf;
+  guint n, bpf;
+  guint64 period_time, buffer_time;
+  guint64 period_samples, buffer_samples;
 
   GST_DEBUG_OBJECT (interaudiosink, "render %" G_GSIZE_FORMAT,
       gst_buffer_get_size (buffer));
   bpf = interaudiosink->info.bpf;
 
   g_mutex_lock (&interaudiosink->surface->mutex);
+
+  buffer_time = interaudiosink->surface->audio_buffer_time;
+  period_time = interaudiosink->surface->audio_period_time;
+  buffer_samples =
+      gst_util_uint64_scale (buffer_time, interaudiosink->info.rate,
+      GST_SECOND);
+  period_samples =
+      gst_util_uint64_scale (period_time, interaudiosink->info.rate,
+      GST_SECOND);
+
   n = gst_adapter_available (interaudiosink->surface->audio_adapter) / bpf;
-  while (n > PERIOD * N_PERIODS) {
+  while (n > buffer_samples) {
-    GST_WARNING_OBJECT (interaudiosink, "flushing %d samples", PERIOD / 2);
+    GST_WARNING_OBJECT (interaudiosink, "flushing %" GST_TIME_FORMAT,
+        GST_TIME_ARGS (period_time));
     gst_adapter_flush (interaudiosink->surface->audio_adapter,
-        (PERIOD / 2) * bpf);
+        period_samples * bpf);
-    n -= (PERIOD / 2);
+    n -= period_samples;
   }
   gst_adapter_push (interaudiosink->surface->audio_adapter,
       gst_buffer_ref (buffer));
@@ -269,3 +286,62 @@ gst_inter_audio_sink_render (GstBaseSink * sink, GstBuffer * buffer)
 
   return GST_FLOW_OK;
 }
+
+static gboolean
+gst_inter_audio_sink_query (GstBaseSink * sink, GstQuery * query)
+{
+  GstInterAudioSink *interaudiosink = GST_INTER_AUDIO_SINK (sink);
+  gboolean ret;
+
+  GST_DEBUG_OBJECT (sink, "query");
+
+  switch (GST_QUERY_TYPE (query)) {
+    case GST_QUERY_LATENCY:{
+      gboolean live, us_live;
+      GstClockTime min_l, max_l;
+
+      GST_DEBUG_OBJECT (sink, "latency query");
+
+      if ((ret =
+              gst_base_sink_query_latency (GST_BASE_SINK_CAST (sink), &live,
+                  &us_live, &min_l, &max_l))) {
+        GstClockTime base_latency, min_latency, max_latency;
+
+        /* we and upstream are both live, adjust the min_latency */
+        if (live && us_live) {
+          /* FIXME: The other side can change this value when it starts */
+          base_latency = interaudiosink->surface->audio_latency_time;
+
+          /* we cannot go lower than the buffer size and the min peer latency */
+          min_latency = base_latency + min_l;
+          /* the max latency is the max of the peer, we can delay an infinite
+           * amount of time. */
+          max_latency = (max_l == -1) ? -1 : (base_latency + max_l);
+
+          GST_DEBUG_OBJECT (sink,
+              "peer min %" GST_TIME_FORMAT ", our min latency: %"
+              GST_TIME_FORMAT, GST_TIME_ARGS (min_l),
+              GST_TIME_ARGS (min_latency));
+          GST_DEBUG_OBJECT (sink,
+              "peer max %" GST_TIME_FORMAT ", our max latency: %"
+              GST_TIME_FORMAT, GST_TIME_ARGS (max_l),
+              GST_TIME_ARGS (max_latency));
+        } else {
+          GST_DEBUG_OBJECT (sink,
+              "peer or we are not live, don't care about latency");
+          min_latency = min_l;
+          max_latency = max_l;
+        }
+        gst_query_set_latency (query, live, min_latency, max_latency);
+      }
+      break;
+    }
+    default:
+      ret =
+          GST_BASE_SINK_CLASS (gst_inter_audio_sink_parent_class)->query (sink,
+          query);
+      break;
+  }
+
+  return ret;
+}

gst/inter/gstinteraudiosrc.c

@@ -50,9 +50,6 @@
 GST_DEBUG_CATEGORY_STATIC (gst_inter_audio_src_debug_category);
 #define GST_CAT_DEFAULT gst_inter_audio_src_debug_category
 
-#define PERIOD    1600
-#define N_PERIODS 10
-
 /* prototypes */
 static void gst_inter_audio_src_set_property (GObject * object,
     guint property_id, const GValue * value, GParamSpec * pspec);
@@ -77,7 +74,10 @@ static GstCaps *gst_inter_audio_src_fixate (GstBaseSrc * src, GstCaps * caps);
 enum
 {
   PROP_0,
-  PROP_CHANNEL
+  PROP_CHANNEL,
+  PROP_BUFFER_TIME,
+  PROP_LATENCY_TIME,
+  PROP_PERIOD_TIME
 };
 
 /* pad templates */
@@ -128,6 +128,23 @@ gst_inter_audio_src_class_init (GstInterAudioSrcClass * klass)
       g_param_spec_string ("channel", "Channel",
           "Channel name to match inter src and sink elements",
           "default", G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_BUFFER_TIME,
+      g_param_spec_uint64 ("buffer-time", "Buffer Time",
+          "Size of audio buffer", 1, G_MAXUINT64, DEFAULT_AUDIO_BUFFER_TIME,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_LATENCY_TIME,
+      g_param_spec_uint64 ("latency-time", "Latency Time",
+          "Latency as reported by the source",
+          1, G_MAXUINT64, DEFAULT_AUDIO_LATENCY_TIME,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_PERIOD_TIME,
+      g_param_spec_uint64 ("period-time", "Period Time",
+          "The minimum amount of data to read in each iteration",
+          1, G_MAXUINT64, DEFAULT_AUDIO_PERIOD_TIME,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 }
 
 static void
@@ -138,6 +155,9 @@ gst_inter_audio_src_init (GstInterAudioSrc * interaudiosrc)
   gst_base_src_set_blocksize (GST_BASE_SRC (interaudiosrc), -1);
 
   interaudiosrc->channel = g_strdup ("default");
+  interaudiosrc->buffer_time = DEFAULT_AUDIO_BUFFER_TIME;
+  interaudiosrc->latency_time = DEFAULT_AUDIO_LATENCY_TIME;
+  interaudiosrc->period_time = DEFAULT_AUDIO_PERIOD_TIME;
 }
 
 void
@@ -151,6 +171,15 @@ gst_inter_audio_src_set_property (GObject * object, guint property_id,
       g_free (interaudiosrc->channel);
       interaudiosrc->channel = g_value_dup_string (value);
       break;
+    case PROP_BUFFER_TIME:
+      interaudiosrc->buffer_time = g_value_get_uint64 (value);
+      break;
+    case PROP_LATENCY_TIME:
+      interaudiosrc->latency_time = g_value_get_uint64 (value);
+      break;
+    case PROP_PERIOD_TIME:
+      interaudiosrc->period_time = g_value_get_uint64 (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
       break;
@@ -167,6 +196,15 @@ gst_inter_audio_src_get_property (GObject * object, guint property_id,
     case PROP_CHANNEL:
       g_value_set_string (value, interaudiosrc->channel);
       break;
+    case PROP_BUFFER_TIME:
+      g_value_set_uint64 (value, interaudiosrc->buffer_time);
+      break;
+    case PROP_LATENCY_TIME:
+      g_value_set_uint64 (value, interaudiosrc->latency_time);
+      break;
+    case PROP_PERIOD_TIME:
+      g_value_set_uint64 (value, interaudiosrc->period_time);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
       break;
@@ -242,6 +280,12 @@ gst_inter_audio_src_start (GstBaseSrc * src)
   interaudiosrc->timestamp_offset = 0;
   interaudiosrc->n_samples = 0;
 
+  g_mutex_lock (&interaudiosrc->surface->mutex);
+  interaudiosrc->surface->audio_buffer_time = interaudiosrc->buffer_time;
+  interaudiosrc->surface->audio_latency_time = interaudiosrc->latency_time;
+  interaudiosrc->surface->audio_period_time = interaudiosrc->period_time;
+  g_mutex_unlock (&interaudiosrc->surface->mutex);
+
   return TRUE;
 }
 
@@ -262,24 +306,24 @@ static void
 gst_inter_audio_src_get_times (GstBaseSrc * src, GstBuffer * buffer,
     GstClockTime * start, GstClockTime * end)
 {
+  GstInterAudioSrc *interaudiosrc = GST_INTER_AUDIO_SRC (src);
+
   GST_DEBUG_OBJECT (src, "get_times");
 
   /* for live sources, sync on the timestamp of the buffer */
   if (gst_base_src_is_live (src)) {
-    GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer);
+    if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) {
-
+      *start = GST_BUFFER_TIMESTAMP (buffer);
-    if (GST_CLOCK_TIME_IS_VALID (timestamp)) {
+      if (GST_BUFFER_DURATION_IS_VALID (buffer)) {
-      /* get duration to calculate end time */
+        *end = *start + GST_BUFFER_DURATION (buffer);
-      GstClockTime duration = GST_BUFFER_DURATION (buffer);
+      } else {
-
+        if (interaudiosrc->info.rate > 0) {
-      if (GST_CLOCK_TIME_IS_VALID (duration)) {
+          *end = *start +
-        *end = timestamp + duration;
+              gst_util_uint64_scale_int (gst_buffer_get_size (buffer),
+              GST_SECOND, interaudiosrc->info.rate * interaudiosrc->info.bpf);
+        }
       }
-      *start = timestamp;
     }
-  } else {
-    *start = -1;
-    *end = -1;
   }
 }
 
@@ -290,7 +334,9 @@ gst_inter_audio_src_create (GstBaseSrc * src, guint64 offset, guint size,
   GstInterAudioSrc *interaudiosrc = GST_INTER_AUDIO_SRC (src);
   GstCaps *caps;
   GstBuffer *buffer;
-  int n, bpf;
+  guint n, bpf;
+  guint64 period_time, buffer_time;
+  guint64 period_samples, buffer_samples;
 
   GST_DEBUG_OBJECT (interaudiosrc, "create");
 
@@ -310,14 +356,20 @@ gst_inter_audio_src_create (GstBaseSrc * src, guint64 offset, guint size,
   }
 
   bpf = interaudiosrc->surface->audio_info.bpf;
+  buffer_time = interaudiosrc->surface->audio_buffer_time;
+  period_time = interaudiosrc->surface->audio_period_time;
+  buffer_samples =
+      gst_util_uint64_scale (buffer_time, interaudiosrc->info.rate, GST_SECOND);
+  period_samples =
+      gst_util_uint64_scale (period_time, interaudiosrc->info.rate, GST_SECOND);
 
   if (bpf > 0)
     n = gst_adapter_available (interaudiosrc->surface->audio_adapter) / bpf;
   else
     n = 0;
 
-  if (n > PERIOD)
+  if (n > period_samples)
-    n = PERIOD;
+    n = period_samples;
   if (n > 0) {
     buffer = gst_adapter_take_buffer (interaudiosrc->surface->audio_adapter,
         n * bpf);
@@ -338,13 +390,14 @@ gst_inter_audio_src_create (GstBaseSrc * src, guint64 offset, guint size,
   }
 
   bpf = interaudiosrc->info.bpf;
-  if (n < PERIOD) {
+  if (n < period_samples) {
     GstMapInfo map;
     GstMemory *mem;
 
-    GST_WARNING_OBJECT (interaudiosrc, "creating %d samples of silence",
+    GST_WARNING_OBJECT (interaudiosrc,
-        PERIOD - n);
+        "creating %" G_GUINT64_FORMAT " samples of silence",
-    mem = gst_allocator_alloc (NULL, (PERIOD - n) * bpf, NULL);
+        period_samples - n);
+    mem = gst_allocator_alloc (NULL, (period_samples - n) * bpf, NULL);
     if (gst_memory_map (mem, &map, GST_MAP_WRITE)) {
       gst_audio_format_fill_silence (interaudiosrc->info.finfo, map.data,
           map.size);
@@ -353,7 +406,7 @@ gst_inter_audio_src_create (GstBaseSrc * src, guint64 offset, guint size,
     buffer = gst_buffer_make_writable (buffer);
     gst_buffer_prepend_memory (buffer, mem);
   }
-  n = PERIOD;
+  n = period_samples;
 
   GST_BUFFER_OFFSET (buffer) = interaudiosrc->n_samples;
   GST_BUFFER_OFFSET_END (buffer) = interaudiosrc->n_samples + n;
@@ -388,25 +441,17 @@ gst_inter_audio_src_query (GstBaseSrc * src, GstQuery * query)
     case GST_QUERY_LATENCY:{
       GstClockTime min_latency, max_latency;
 
-      if (interaudiosrc->info.rate > 0) {
+      min_latency = interaudiosrc->latency_time;
-        /* 1.5 just as a good measure */
+      max_latency = min_latency;
-        min_latency =
-            1.5 * N_PERIODS * gst_util_uint64_scale_int (GST_SECOND, PERIOD,
-            interaudiosrc->info.rate);
 
-        max_latency = min_latency;
+      GST_DEBUG_OBJECT (src,
+          "report latency min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
+          GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
 
-        GST_DEBUG_OBJECT (src,
+      gst_query_set_latency (query,
-            "report latency min %" GST_TIME_FORMAT " max %" GST_TIME_FORMAT,
+          gst_base_src_is_live (src), min_latency, max_latency);
-            GST_TIME_ARGS (min_latency), GST_TIME_ARGS (max_latency));
 
-        gst_query_set_latency (query,
+      ret = TRUE;
-            gst_base_src_is_live (src), min_latency, max_latency);
-
-        ret = TRUE;
-      } else {
-        ret = FALSE;
-      }
       break;
     }
     default:

gst/inter/gstinteraudiosrc.h

@@ -45,6 +45,7 @@ struct _GstInterAudioSrc
   guint64 n_samples;
   GstClockTime timestamp_offset;
   GstAudioInfo info;
+  guint64 buffer_time, latency_time, period_time;
 };
 
 struct _GstInterAudioSrcClass

gst/inter/gstintersurface.c

@@ -49,6 +49,9 @@ gst_inter_surface_get (const char *name)
   surface->name = g_strdup (name);
   g_mutex_init (&surface->mutex);
   surface->audio_adapter = gst_adapter_new ();
+  surface->audio_buffer_time = DEFAULT_AUDIO_BUFFER_TIME;
+  surface->audio_latency_time = DEFAULT_AUDIO_LATENCY_TIME;
+  surface->audio_period_time = DEFAULT_AUDIO_PERIOD_TIME;
 
   list = g_list_append (list, surface);
   g_mutex_unlock (&mutex);

gst/inter/gstintersurface.h

@@ -41,12 +41,19 @@ struct _GstInterSurface
 
   /* audio */
   GstAudioInfo audio_info;
+  guint64 audio_buffer_time;
+  guint64 audio_latency_time;
+  guint64 audio_period_time;
 
   GstBuffer *video_buffer;
   GstBuffer *sub_buffer;
   GstAdapter *audio_adapter;
 };
 
+#define DEFAULT_AUDIO_BUFFER_TIME  (GST_SECOND)
+#define DEFAULT_AUDIO_LATENCY_TIME (100 * GST_MSECOND)
+#define DEFAULT_AUDIO_PERIOD_TIME  (25 * GST_MSECOND)
+
 
 GstInterSurface * gst_inter_surface_get (const char *name);
 void gst_inter_surface_unref (GstInterSurface *surface);