decklink: Detect gaps on incoming stream times, issue warnings

When we receive a video or audio buffer, we calculate the next stream
time based on the current stream time + buffer duration. If the next
buffer's stream time is after that, we issue a warning.

This happens because the stream time incoming from Decklink should be
really constant and without gaps. If there is a gap, it means that
something went wrong, e.g. the internal buffer pool is empty (too many
buffers queued up downstream).

https://bugzilla.gnome.org/show_bug.cgi?id=781776

sys/decklink/gstdecklinkaudiosrc.cpp

@@ -36,6 +36,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_decklink_audio_src_debug);
 #define DEFAULT_DISCONT_WAIT          (1 * GST_SECOND)
 #define DEFAULT_CHANNELS              (GST_DECKLINK_AUDIO_CHANNELS_2)
 
+#ifndef ABSDIFF
+#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )
+#endif
+
 enum
 {
   PROP_0,
@@ -701,6 +705,48 @@ retry:
         self->info.rate) - timestamp;
   }
 
+  // Detect gaps in stream time
+  self->processed += sample_count;
+
+  if (p.stream_timestamp != GST_CLOCK_TIME_NONE) {
+    GstClockTime start_stream_time, end_stream_time;
+
+    start_stream_time = p.stream_timestamp;
+
+    start_offset =
+        gst_util_uint64_scale (start_stream_time, self->info.rate, GST_SECOND);
+
+    end_offset = start_offset + sample_count;
+    end_stream_time = gst_util_uint64_scale_int (end_offset, GST_SECOND,
+        self->info.rate);
+
+    /* With drop-frame we have gaps of 1 sample every now and then (rounding
+     * errors because of the samples-per-frame pattern which is not 100%
+     * accurate), and due to rounding errors in the calculations these can be
+     * 2>x>1 */
+    if (self->expected_stream_time != GST_CLOCK_TIME_NONE &&
+        ABSDIFF (self->expected_stream_time, p.stream_timestamp) >
+        gst_util_uint64_scale (2, GST_SECOND, self->info.rate)) {
+      GstMessage *msg;
+      GstClockTime running_time;
+
+      self->dropped +=
+          gst_util_uint64_scale (ABSDIFF (self->expected_stream_time,
+              p.stream_timestamp), self->info.rate, GST_SECOND);
+      running_time =
+          gst_segment_to_running_time (&GST_BASE_SRC (self)->segment,
+          GST_FORMAT_TIME, timestamp);
+
+      msg =
+          gst_message_new_qos (GST_OBJECT (self), TRUE, running_time, p.stream_timestamp,
+          timestamp, duration);
+      gst_message_set_qos_stats (msg, GST_FORMAT_DEFAULT, self->processed,
+          self->dropped);
+      gst_element_post_message (GST_ELEMENT (self), msg);
+    }
+    self->expected_stream_time = end_stream_time;
+  }
+
   if (p.no_signal)
     GST_BUFFER_FLAG_SET (*buffer, GST_BUFFER_FLAG_GAP);
   GST_BUFFER_TIMESTAMP (*buffer) = timestamp;
@@ -907,6 +953,9 @@ gst_decklink_audio_src_change_state (GstElement * element,
 
   switch (transition) {
     case GST_STATE_CHANGE_NULL_TO_READY:
+      self->processed = 0;
+      self->dropped = 0;
+      self->expected_stream_time = GST_CLOCK_TIME_NONE;
       if (!gst_decklink_audio_src_open (self)) {
         ret = GST_STATE_CHANGE_FAILURE;
         goto out;

sys/decklink/gstdecklinkaudiosrc.h

@@ -71,6 +71,11 @@ struct _GstDecklinkAudioSrc
   /* counter to keep track of timestamps */
   guint64 next_offset;
 
+  /* detect gaps in stream time */
+  GstClockTime expected_stream_time;
+  guint64 processed;
+  guint64 dropped;
+
   /* Last time we noticed a discont */
   GstClockTime discont_time;
 

sys/decklink/gstdecklinkvideosrc.cpp

@@ -36,6 +36,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_decklink_video_src_debug);
 #define DEFAULT_SKIP_FIRST_TIME (0)
 #define DEFAULT_DROP_NO_SIGNAL_FRAMES (FALSE)
 
+#ifndef ABSDIFF
+#define ABSDIFF(x, y) ( (x) > (y) ? ((x) - (y)) : ((y) - (x)) )
+#endif
+
 enum
 {
   PROP_0,
@@ -825,6 +829,29 @@ gst_decklink_video_src_create (GstPushSrc * bsrc, GstBuffer ** buffer)
     }
   }
 
+  /* 1 ns error can be just a rounding error, so that's OK. The Decklink
+   * drivers give us a really steady stream time, so anything above 1 ns can't
+   * be a rounding error and is therefore something to worry about */
+  if (self->expected_stream_time != GST_CLOCK_TIME_NONE &&
+      ABSDIFF (self->expected_stream_time, f.stream_timestamp) > 1) {
+    GstMessage *msg;
+    GstClockTime running_time;
+
+    self->dropped += f.stream_timestamp - self->expected_stream_time;
+    running_time = gst_segment_to_running_time (&GST_BASE_SRC (self)->segment,
+        GST_FORMAT_TIME, f.timestamp);
+
+    msg = gst_message_new_qos (GST_OBJECT (self), TRUE, running_time, f.stream_timestamp,
+        f.timestamp, f.duration);
+    gst_message_set_qos_stats (msg, GST_FORMAT_TIME, self->processed,
+        self->dropped);
+    gst_element_post_message (GST_ELEMENT (self), msg);
+  }
+  if (self->first_stream_time == GST_CLOCK_TIME_NONE)
+    self->first_stream_time = f.stream_timestamp;
+  self->processed = f.stream_timestamp - self->dropped - self->first_stream_time;
+  self->expected_stream_time = f.stream_timestamp + f.stream_duration;
+
   g_mutex_unlock (&self->lock);
   if (caps_changed) {
     caps = gst_decklink_mode_get_caps (f.mode, f.format, TRUE);
@@ -1080,6 +1107,10 @@ gst_decklink_video_src_change_state (GstElement * element,
 
   switch (transition) {
     case GST_STATE_CHANGE_NULL_TO_READY:
+      self->processed = 0;
+      self->dropped = 0;
+      self->expected_stream_time = GST_CLOCK_TIME_NONE;
+      self->first_stream_time = GST_CLOCK_TIME_NONE;
       if (!gst_decklink_video_src_open (self)) {
         ret = GST_STATE_CHANGE_FAILURE;
         goto out;

sys/decklink/gstdecklinkvideosrc.h

@@ -58,6 +58,10 @@ struct _GstDecklinkVideoSrc
   gboolean output_stream_time;
   GstClockTime skip_first_time;
   gboolean drop_no_signal_frames;
+  GstClockTime expected_stream_time;
+  guint64 processed;
+  guint64 dropped;
+  guint64 first_stream_time;
 
   GstVideoInfo info;
   GstDecklinkVideoFormat video_format;