rtpbin: add option for increasing ts_offset gradually

Instant large changes to ts_offset may cause timestamps to move
backwards and also cause visible effects in media playback. The new
option max-ts-offset-adjustment lets the application control the rate to
apply changes to ts_offset.

https://bugzilla.gnome.org/show_bug.cgi?id=784002

gst/rtpmanager/gstrtpbin.c

@@ -309,6 +309,7 @@ enum
 #define DEFAULT_MAX_MISORDER_TIME    2000
 #define DEFAULT_RFC7273_SYNC         FALSE
 #define DEFAULT_MAX_STREAMS          G_MAXUINT
+#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0
 
 enum
 {
@@ -333,7 +334,8 @@ enum
   PROP_MAX_DROPOUT_TIME,
   PROP_MAX_MISORDER_TIME,
   PROP_RFC7273_SYNC,
-  PROP_MAX_STREAMS
+  PROP_MAX_STREAMS,
+  PROP_MAX_TS_OFFSET_ADJUSTMENT
 };
 
 #define GST_RTP_BIN_RTCP_SYNC_TYPE (gst_rtp_bin_rtcp_sync_get_type())
@@ -1759,6 +1761,8 @@ create_stream (GstRtpBinSession * session, guint32 ssrc)
   g_object_set (buffer, "max-dropout-time", rtpbin->max_dropout_time,
       "max-misorder-time", rtpbin->max_misorder_time, NULL);
   g_object_set (buffer, "rfc7273-sync", rtpbin->rfc7273_sync, NULL);
+  g_object_set (buffer, "max-ts-offset-adjustment",
+      rtpbin->max_ts_offset_adjustment, NULL);
 
   g_signal_emit (rtpbin, gst_rtp_bin_signals[SIGNAL_NEW_JITTERBUFFER], 0,
       buffer, session->id, ssrc);
@@ -2493,6 +2497,22 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass)
           0, G_MAXUINT, DEFAULT_MAX_STREAMS,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  /**
+   * GstRtpBin:max-ts-offset-adjustment:
+   *
+   * Syncing time stamps to NTP time adds a time offset. This parameter
+   * specifies the maximum number of nanoseconds per frame that this time offset
+   * may be adjusted with. This is used to avoid sudden large changes to time
+   * stamps.
+   */
+  g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT,
+      g_param_spec_uint64 ("max-ts-offset-adjustment",
+          "Max Timestamp Offset Adjustment",
+          "The maximum number of nanoseconds per frame that time stamp offsets "
+          "may be adjusted (0 = no limit).", 0, G_MAXUINT64,
+          DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE |
+          G_PARAM_STATIC_STRINGS));
+
   gstelement_class->change_state = GST_DEBUG_FUNCPTR (gst_rtp_bin_change_state);
   gstelement_class->request_new_pad =
       GST_DEBUG_FUNCPTR (gst_rtp_bin_request_new_pad);
@@ -2564,6 +2584,7 @@ gst_rtp_bin_init (GstRtpBin * rtpbin)
   rtpbin->max_misorder_time = DEFAULT_MAX_MISORDER_TIME;
   rtpbin->rfc7273_sync = DEFAULT_RFC7273_SYNC;
   rtpbin->max_streams = DEFAULT_MAX_STREAMS;
+  rtpbin->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT;
 
   /* some default SDES entries */
   cname = g_strdup_printf ("user%u@host-%x", g_random_int (), g_random_int ());
@@ -2788,6 +2809,11 @@ gst_rtp_bin_set_property (GObject * object, guint prop_id,
     case PROP_MAX_STREAMS:
       rtpbin->max_streams = g_value_get_uint (value);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      rtpbin->max_ts_offset_adjustment = g_value_get_uint64 (value);
+      gst_rtp_bin_propagate_property_to_jitterbuffer (rtpbin,
+          "max-ts-offset-adjustment", value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -2876,6 +2902,9 @@ gst_rtp_bin_get_property (GObject * object, guint prop_id,
     case PROP_MAX_STREAMS:
       g_value_set_uint (value, rtpbin->max_streams);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      g_value_set_uint64 (value, rtpbin->max_ts_offset_adjustment);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

gst/rtpmanager/gstrtpbin.h

@@ -75,6 +75,7 @@ struct _GstRtpBin {
   guint32         max_misorder_time;
   gboolean        rfc7273_sync;
   guint           max_streams;
+  guint64         max_ts_offset_adjustment;
 
   /* a list of session */
   GSList         *sessions;

gst/rtpmanager/gstrtpjitterbuffer.c

@@ -131,6 +131,7 @@ enum
 #define DEFAULT_LATENCY_MS          200
 #define DEFAULT_DROP_ON_LATENCY     FALSE
 #define DEFAULT_TS_OFFSET           0
+#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT 0
 #define DEFAULT_DO_LOST             FALSE
 #define DEFAULT_MODE                RTP_JITTER_BUFFER_MODE_SLAVE
 #define DEFAULT_PERCENT             0
@@ -160,6 +161,7 @@ enum
   PROP_LATENCY,
   PROP_DROP_ON_LATENCY,
   PROP_TS_OFFSET,
+  PROP_MAX_TS_OFFSET_ADJUSTMENT,
   PROP_DO_LOST,
   PROP_MODE,
   PROP_PERCENT,
@@ -281,6 +283,7 @@ struct _GstRtpJitterBufferPrivate
   guint64 latency_ns;
   gboolean drop_on_latency;
   gint64 ts_offset;
+  guint64 max_ts_offset_adjustment;
   gboolean do_lost;
   gboolean do_retransmission;
   gboolean rtx_next_seqnum;
@@ -337,7 +340,7 @@ struct _GstRtpJitterBufferPrivate
   gint last_pt;
   gint32 clock_rate;
   gint64 clock_base;
-  gint64 prev_ts_offset;
+  gint64 ts_offset_remainder;
 
   /* when we are shutting down */
   GstFlowReturn srcresult;
@@ -368,6 +371,7 @@ struct _GstRtpJitterBufferPrivate
 
   /* for the jitter */
   GstClockTime last_dts;
+  GstClockTime last_pts;
   guint64 last_rtptime;
   GstClockTime avg_jitter;
 };
@@ -549,6 +553,20 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass)
           G_MAXINT64, DEFAULT_TS_OFFSET,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  /**
+   * GstRtpJitterBuffer:max-ts-offset-adjustment:
+   *
+   * The maximum number of nanoseconds per frame that time offset may be
+   * adjusted with. This is used to avoid sudden large changes to time stamps.
+   */
+  g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT,
+      g_param_spec_uint64 ("max-ts-offset-adjustment",
+          "Max Timestamp Offset Adjustment",
+          "The maximum number of nanoseconds per frame that time stamp "
+          "offsets may be adjusted (0 = no limit).", 0, G_MAXUINT64,
+          DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE |
+          G_PARAM_STATIC_STRINGS));
+
   /**
    * GstRtpJitterBuffer:do-lost:
    *
@@ -980,6 +998,8 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer)
   priv->latency_ms = DEFAULT_LATENCY_MS;
   priv->latency_ns = priv->latency_ms * GST_MSECOND;
   priv->drop_on_latency = DEFAULT_DROP_ON_LATENCY;
+  priv->ts_offset = DEFAULT_TS_OFFSET;
+  priv->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT;
   priv->do_lost = DEFAULT_DO_LOST;
   priv->do_retransmission = DEFAULT_DO_RETRANSMISSION;
   priv->rtx_next_seqnum = DEFAULT_RTX_NEXT_SEQNUM;
@@ -997,7 +1017,9 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer)
   priv->max_misorder_time = DEFAULT_MAX_MISORDER_TIME;
   priv->faststart_min_packets = DEFAULT_FASTSTART_MIN_PACKETS;
 
+  priv->ts_offset_remainder = 0;
   priv->last_dts = -1;
+  priv->last_pts = -1;
   priv->last_rtptime = -1;
   priv->avg_jitter = 0;
   priv->timers = g_array_new (FALSE, TRUE, sizeof (TimerData));
@@ -1571,7 +1593,7 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer)
   priv->srcresult = GST_FLOW_OK;
   gst_segment_init (&priv->segment, GST_FORMAT_TIME);
   priv->last_popped_seqnum = -1;
-  priv->last_out_time = -1;
+  priv->last_out_time = GST_CLOCK_TIME_NONE;
   priv->next_seqnum = -1;
   priv->seqnum_base = -1;
   priv->ips_rtptime = -1;
@@ -1990,6 +2012,32 @@ check_buffering_percent (GstRtpJitterBuffer * jitterbuffer, gint percent)
   return message;
 }
 
+static void
+update_offset (GstRtpJitterBuffer * jitterbuffer)
+{
+  GstRtpJitterBufferPrivate *priv;
+
+  priv = jitterbuffer->priv;
+
+  if (priv->ts_offset_remainder != 0) {
+    GST_DEBUG ("adjustment %" G_GUINT64_FORMAT " remain %" G_GINT64_FORMAT
+        " off %" G_GINT64_FORMAT, priv->max_ts_offset_adjustment,
+        priv->ts_offset_remainder, priv->ts_offset);
+    if (ABS (priv->ts_offset_remainder) > priv->max_ts_offset_adjustment) {
+      if (priv->ts_offset_remainder > 0) {
+        priv->ts_offset += priv->max_ts_offset_adjustment;
+        priv->ts_offset_remainder -= priv->max_ts_offset_adjustment;
+      } else {
+        priv->ts_offset -= priv->max_ts_offset_adjustment;
+        priv->ts_offset_remainder += priv->max_ts_offset_adjustment;
+      }
+    } else {
+      priv->ts_offset += priv->ts_offset_remainder;
+      priv->ts_offset_remainder = 0;
+    }
+  }
+}
+
 static GstClockTime
 apply_offset (GstRtpJitterBuffer * jitterbuffer, GstClockTime timestamp)
 {
@@ -3388,6 +3436,11 @@ pop_and_push_next (GstRtpJitterBuffer * jitterbuffer, guint seqnum)
           gst_segment_position_from_running_time (&priv->segment,
           GST_FORMAT_TIME, item->pts);
 
+      /* if this is a new frame, check if ts_offset needs to be updated */
+      if (pts != priv->last_pts) {
+        update_offset (jitterbuffer);
+      }
+
       /* apply timestamp with offset to buffer now */
       GST_BUFFER_DTS (outbuf) = apply_offset (jitterbuffer, dts);
       GST_BUFFER_PTS (outbuf) = apply_offset (jitterbuffer, pts);
@@ -3395,6 +3448,20 @@ pop_and_push_next (GstRtpJitterBuffer * jitterbuffer, guint seqnum)
       /* update the elapsed time when we need to check against the npt stop time. */
       update_estimated_eos (jitterbuffer, item);
 
+      /* verify that an offset has not caused time stamps to go backwards, if so
+       * handle by reusing the previous timestamp */
+      if (priv->last_out_time != GST_CLOCK_TIME_NONE &&
+          GST_BUFFER_PTS (outbuf) < priv->last_out_time) {
+        GST_DEBUG_OBJECT (jitterbuffer, "buffer PTS %" GST_TIME_FORMAT
+            " older than preceding PTS %" GST_TIME_FORMAT
+            " adjusting to %" GST_TIME_FORMAT,
+            GST_TIME_ARGS (GST_BUFFER_PTS (outbuf)),
+            GST_TIME_ARGS (priv->last_out_time),
+            GST_TIME_ARGS (priv->last_out_time));
+        GST_BUFFER_PTS (outbuf) = priv->last_out_time;
+      }
+
+      priv->last_pts = pts;
       priv->last_out_time = GST_BUFFER_PTS (outbuf);
       break;
     case ITEM_TYPE_LOST:
@@ -4480,10 +4547,26 @@ gst_rtp_jitter_buffer_set_property (GObject * object,
       break;
     case PROP_TS_OFFSET:
       JBUF_LOCK (priv);
-      priv->ts_offset = g_value_get_int64 (value);
+      if (priv->max_ts_offset_adjustment != 0) {
+        gint64 new_offset = g_value_get_int64 (value);
+
+        if (new_offset > priv->ts_offset) {
+          priv->ts_offset_remainder = new_offset - priv->ts_offset;
+        } else {
+          priv->ts_offset_remainder = -(priv->ts_offset - new_offset);
+        }
+      } else {
+        priv->ts_offset = g_value_get_int64 (value);
+        priv->ts_offset_remainder = 0;
+      }
       priv->ts_discont = TRUE;
       JBUF_UNLOCK (priv);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      JBUF_LOCK (priv);
+      priv->max_ts_offset_adjustment = g_value_get_uint64 (value);
+      JBUF_UNLOCK (priv);
+      break;
     case PROP_DO_LOST:
       JBUF_LOCK (priv);
       priv->do_lost = g_value_get_boolean (value);
@@ -4607,6 +4690,11 @@ gst_rtp_jitter_buffer_get_property (GObject * object,
       g_value_set_int64 (value, priv->ts_offset);
       JBUF_UNLOCK (priv);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      JBUF_LOCK (priv);
+      g_value_set_uint64 (value, priv->max_ts_offset_adjustment);
+      JBUF_UNLOCK (priv);
+      break;
     case PROP_DO_LOST:
       JBUF_LOCK (priv);
       g_value_set_boolean (value, priv->do_lost);

gst/rtpmanager/rtpsource.c

@@ -1280,7 +1280,8 @@ rtp_source_send_rtp (RTPSource * src, RTPPacketInfo * pinfo)
     return GST_FLOW_OK;
 
   if (src->pt_set && src->pt != pinfo->pt) {
-    GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt, src->ssrc);
+    GST_WARNING ("Changing pt from %u to %u for SSRC %u", src->pt, pinfo->pt,
+        src->ssrc);
   }
 
   src->pt = pinfo->pt;

gst/rtsp/gstrtspsrc.c

@@ -228,6 +228,7 @@ gst_rtsp_src_ntp_time_source_get_type (void)
 #define DEFAULT_USER_AGENT       "GStreamer/" PACKAGE_VERSION
 #define DEFAULT_MAX_RTCP_RTP_TIME_DIFF 1000
 #define DEFAULT_RFC7273_SYNC         FALSE
+#define DEFAULT_MAX_TS_OFFSET_ADJUSTMENT   0
 
 enum
 {
@@ -267,7 +268,8 @@ enum
   PROP_NTP_TIME_SOURCE,
   PROP_USER_AGENT,
   PROP_MAX_RTCP_RTP_TIME_DIFF,
-  PROP_RFC7273_SYNC
+  PROP_RFC7273_SYNC,
+  PROP_MAX_TS_OFFSET_ADJUSTMENT
 };
 
 #define GST_TYPE_RTSP_NAT_METHOD (gst_rtsp_nat_method_get_type())
@@ -748,6 +750,22 @@ gst_rtspsrc_class_init (GstRTSPSrcClass * klass)
           "(requires clock and offset to be provided)", DEFAULT_RFC7273_SYNC,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  /**
+   * GstRTSPSrc:max-ts-offset-adjustment:
+   *
+   * Syncing time stamps to NTP time adds a time offset. This parameter
+   * specifies the maximum number of nanoseconds per frame that this time offset
+   * may be adjusted with. This is used to avoid sudden large changes to time
+   * stamps.
+   */
+  g_object_class_install_property (gobject_class, PROP_MAX_TS_OFFSET_ADJUSTMENT,
+      g_param_spec_uint64 ("max-ts-offset-adjustment",
+          "Max Timestamp Offset Adjustment",
+          "The maximum number of nanoseconds per frame that time stamp offsets "
+          "may be adjusted (0 = no limit).", 0, G_MAXUINT64,
+          DEFAULT_MAX_TS_OFFSET_ADJUSTMENT, G_PARAM_READWRITE |
+          G_PARAM_STATIC_STRINGS));
+
   /**
    * GstRTSPSrc::handle-request:
    * @rtspsrc: a #GstRTSPSrc
@@ -896,6 +914,7 @@ gst_rtspsrc_init (GstRTSPSrc * src)
   src->user_agent = g_strdup (DEFAULT_USER_AGENT);
   src->max_rtcp_rtp_time_diff = DEFAULT_MAX_RTCP_RTP_TIME_DIFF;
   src->rfc7273_sync = DEFAULT_RFC7273_SYNC;
+  src->max_ts_offset_adjustment = DEFAULT_MAX_TS_OFFSET_ADJUSTMENT;
 
   /* get a list of all extensions */
   src->extensions = gst_rtsp_ext_list_get ();
@@ -1186,6 +1205,9 @@ gst_rtspsrc_set_property (GObject * object, guint prop_id, const GValue * value,
     case PROP_RFC7273_SYNC:
       rtspsrc->rfc7273_sync = g_value_get_boolean (value);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      rtspsrc->max_ts_offset_adjustment = g_value_get_uint64 (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -1335,6 +1357,9 @@ gst_rtspsrc_get_property (GObject * object, guint prop_id, GValue * value,
     case PROP_RFC7273_SYNC:
       g_value_set_boolean (value, rtspsrc->rfc7273_sync);
       break;
+    case PROP_MAX_TS_OFFSET_ADJUSTMENT:
+      g_value_set_uint64 (value, rtspsrc->max_ts_offset_adjustment);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -3166,6 +3191,11 @@ gst_rtspsrc_stream_configure_manager (GstRTSPSrc * src, GstRTSPStream * stream,
             src->max_rtcp_rtp_time_diff, NULL);
       }
 
+      if (g_object_class_find_property (klass, "max-ts-offset-adjustment")) {
+        g_object_set (src->manager, "max-ts-offset-adjustment",
+            src->max_ts_offset_adjustment, NULL);
+      }
+
       /* buffer mode pauses are handled by adding offsets to buffer times,
        * but some depayloaders may have a hard time syncing output times
        * with such input times, e.g. container ones, most notably ASF */

gst/rtsp/gstrtspsrc.h

@@ -250,6 +250,7 @@ struct _GstRTSPSrc {
   gchar            *user_agent;
   GstClockTime      max_rtcp_rtp_time_diff;
   gboolean          rfc7273_sync;
+  guint64           max_ts_offset_adjustment;
 
   /* state */
   GstRTSPState       state;