mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-27 20:21:24 +00:00
Code Issues 1 Projects Releases Wiki Activity

rtpjitterbuffer: fix lost-event using dts instead of pts

Browse source 
The lost-event was using a different time-domain (dts) than the outgoing
buffers (pts). Given certain network-conditions these two would become
sufficiently different and the lost-event contained timestamp/duration
that was really wrong. As an example GstAudioDecoder could produce
a stream that jumps back and forth in time after receiving a lost-event.

The previous behavior calculated the pts (based on the rtptime) inside the
rtp_jitter_buffer_insert function, but now this functionality has been
refactored into a new function rtp_jitter_buffer_calculate_pts that is
called much earlier in the _chain function to make pts available to
various calculations that wrongly used dts previously
(like the lost-event).

There are however two calculations where using dts is the right thing to
do: calculating the receive-jitter and the rtx-round-trip-time, where the
arrival time of the buffer from the network is the right metric
(and is what dts in fact is today).

The patch also adds two tests regarding B-frames or the
“rtptime-going-backwards”-scenario, as there were some concerns that this
patch might break this behavior (which the tests shows it does not).
This commit is contained in:
Havard Graff 2016-10-09 15:59:05 +02:00 committed by Sebastian Dröge
parent bea35f97c8
commit fb9c75db36
4 changed files with 543 additions and 410 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

378
gst/rtpmanager/gstrtpjitterbuffer.c
View file

@ -304,7 +304,7 @@ struct _GstRtpJitterBufferPrivate
/* last output time */ /* last output time */
GstClockTime last_out_time; GstClockTime last_out_time;
/* last valid input timestamp and rtptime pair */ /* last valid input timestamp and rtptime pair */
GstClockTime ips_dts; GstClockTime ips_pts;
guint64 ips_rtptime; guint64 ips_rtptime;
GstClockTime packet_spacing; GstClockTime packet_spacing;
gint equidistant; gint equidistant;
@ -312,7 +312,7 @@ struct _GstRtpJitterBufferPrivate
GQueue gap_packets; GQueue gap_packets;
/* the next expected seqnum we receive */ /* the next expected seqnum we receive */
GstClockTime last_in_dts; GstClockTime last_in_pts;
guint32 next_in_seqnum; guint32 next_in_seqnum;
GArray *timers; GArray *timers;
@ -1298,7 +1298,7 @@ gst_rtp_jitter_buffer_set_active (GstRtpJitterBuffer * jbuf, gboolean active,
} }
if ((item = rtp_jitter_buffer_peek (priv->jbuf))) { if ((item = rtp_jitter_buffer_peek (priv->jbuf))) {
/* head buffer timestamp and offset gives our output time */ /* head buffer timestamp and offset gives our output time */
last_out = item->dts + priv->ts_offset; last_out = item->pts + priv->ts_offset;
} else { } else {
/* use last known time when the buffer is empty */ /* use last known time when the buffer is empty */
last_out = priv->last_out_time; last_out = priv->last_out_time;
@ -1554,7 +1554,7 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer)
priv->next_seqnum = -1; priv->next_seqnum = -1;
priv->seqnum_base = -1; priv->seqnum_base = -1;
priv->ips_rtptime = -1; priv->ips_rtptime = -1;
priv->ips_dts = GST_CLOCK_TIME_NONE; priv->ips_pts = GST_CLOCK_TIME_NONE;
priv->packet_spacing = 0; priv->packet_spacing = 0;
priv->next_in_seqnum = -1; priv->next_in_seqnum = -1;
priv->clock_rate = -1; priv->clock_rate = -1;
@ -1566,7 +1566,7 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer)
priv->avg_jitter = 0; priv->avg_jitter = 0;
priv->last_dts = -1; priv->last_dts = -1;
priv->last_rtptime = -1; priv->last_rtptime = -1;
priv->last_in_dts = 0; priv->last_in_pts = 0;
priv->equidistant = 0; priv->equidistant = 0;
GST_DEBUG_OBJECT (jitterbuffer, "flush and reset jitterbuffer"); GST_DEBUG_OBJECT (jitterbuffer, "flush and reset jitterbuffer");
rtp_jitter_buffer_flush (priv->jbuf, (GFunc) free_item, NULL); rtp_jitter_buffer_flush (priv->jbuf, (GFunc) free_item, NULL);
@ -1782,7 +1782,7 @@ queue_event (GstRtpJitterBuffer * jitterbuffer, GstEvent * event)
GST_DEBUG_OBJECT (jitterbuffer, "adding event"); GST_DEBUG_OBJECT (jitterbuffer, "adding event");
item = alloc_item (event, ITEM_TYPE_EVENT, -1, -1, -1, 0, -1); item = alloc_item (event, ITEM_TYPE_EVENT, -1, -1, -1, 0, -1);
rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL, -1); rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL);
if (head) if (head)
JBUF_SIGNAL_EVENT (priv); JBUF_SIGNAL_EVENT (priv);
@ -2304,8 +2304,8 @@ already_lost (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum)
*/ */
static void static void
update_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum, update_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum,
GstClockTime dts, gboolean do_next_seqnum, gboolean is_rtx, GstClockTime dts, GstClockTime pts, gboolean do_next_seqnum,
TimerData * timer) gboolean is_rtx, TimerData * timer)
{ {
GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; GstRtpJitterBufferPrivate *priv = jitterbuffer->priv;
@ -2351,16 +2351,16 @@ update_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum,
* several retransmitted packets. For accuracy we should update the * several retransmitted packets. For accuracy we should update the
* stats also when the redundant retransmitted packets arrives. */ * stats also when the redundant retransmitted packets arrives. */
timer_queue_append (priv->rtx_stats_timers, timer, timer_queue_append (priv->rtx_stats_timers, timer,
dts + priv->rtx_stats_timeout * GST_MSECOND, FALSE); pts + priv->rtx_stats_timeout * GST_MSECOND, FALSE);
} }
} }
} }
if (do_next_seqnum && dts != GST_CLOCK_TIME_NONE) { if (do_next_seqnum && pts != GST_CLOCK_TIME_NONE) {
GstClockTime expected, delay; GstClockTime expected, delay;
/* calculate expected arrival time of the next seqnum */ /* calculate expected arrival time of the next seqnum */
expected = dts + priv->packet_spacing; expected = pts + priv->packet_spacing;
delay = get_rtx_delay (priv); delay = get_rtx_delay (priv);
@ -2387,16 +2387,16 @@ update_timers (GstRtpJitterBuffer * jitterbuffer, guint16 seqnum,
static void static void
calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime, calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime,
GstClockTime dts) GstClockTime pts)
{ {
GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; GstRtpJitterBufferPrivate *priv = jitterbuffer->priv;
/* we need consecutive seqnums with a different /* we need consecutive seqnums with a different
* rtptime to estimate the packet spacing. */ * rtptime to estimate the packet spacing. */
if (priv->ips_rtptime != rtptime) { if (priv->ips_rtptime != rtptime) {
/* rtptime changed, check dts diff */ /* rtptime changed, check pts diff */
if (priv->ips_dts != -1 && dts != -1 && dts > priv->ips_dts) { if (priv->ips_pts != -1 && pts != -1 && pts > priv->ips_pts) {
GstClockTime new_packet_spacing = dts - priv->ips_dts; GstClockTime new_packet_spacing = pts - priv->ips_pts;
GstClockTime old_packet_spacing = priv->packet_spacing; GstClockTime old_packet_spacing = priv->packet_spacing;
/* Biased towards bigger packet spacings to prevent /* Biased towards bigger packet spacings to prevent
@ -2421,33 +2421,33 @@ calculate_packet_spacing (GstRtpJitterBuffer * jitterbuffer, guint32 rtptime,
GST_TIME_ARGS (priv->packet_spacing)); GST_TIME_ARGS (priv->packet_spacing));
} }
priv->ips_rtptime = rtptime; priv->ips_rtptime = rtptime;
priv->ips_dts = dts; priv->ips_pts = pts;
} }
} }
static void static void
calculate_expected (GstRtpJitterBuffer * jitterbuffer, guint32 expected, calculate_expected (GstRtpJitterBuffer * jitterbuffer, guint32 expected,
guint16 seqnum, GstClockTime dts, gint gap) guint16 seqnum, GstClockTime pts, gint gap)
{ {
GstRtpJitterBufferPrivate *priv = jitterbuffer->priv; GstRtpJitterBufferPrivate *priv = jitterbuffer->priv;
GstClockTime duration, expected_dts, delay; GstClockTime duration, expected_pts, delay;
TimerType type; TimerType type;
gboolean equidistant = priv->equidistant > 0; gboolean equidistant = priv->equidistant > 0;
GST_DEBUG_OBJECT (jitterbuffer, GST_DEBUG_OBJECT (jitterbuffer,
"dts %" GST_TIME_FORMAT ", last %" GST_TIME_FORMAT, "pts %" GST_TIME_FORMAT ", last %" GST_TIME_FORMAT,
GST_TIME_ARGS (dts), GST_TIME_ARGS (priv->last_in_dts)); GST_TIME_ARGS (pts), GST_TIME_ARGS (priv->last_in_pts));
if (dts == GST_CLOCK_TIME_NONE) { if (pts == GST_CLOCK_TIME_NONE) {
GST_WARNING_OBJECT (jitterbuffer, "Have no DTS"); GST_WARNING_OBJECT (jitterbuffer, "Have no PTS");
return; return;
} }
if (equidistant) { if (equidistant) {
GstClockTime total_duration; GstClockTime total_duration;
/* the total duration spanned by the missing packets */ /* the total duration spanned by the missing packets */
if (dts >= priv->last_in_dts) if (pts >= priv->last_in_pts)
total_duration = dts - priv->last_in_dts; total_duration = pts - priv->last_in_pts;
else else
total_duration = 0; total_duration = 0;
@ -2488,19 +2488,19 @@ calculate_expected (GstRtpJitterBuffer * jitterbuffer, guint32 expected,
* the timer thread */ * the timer thread */
if (lost_packets > 0) { if (lost_packets > 0) {
add_timer (jitterbuffer, TIMER_TYPE_LOST, expected, lost_packets, add_timer (jitterbuffer, TIMER_TYPE_LOST, expected, lost_packets,
priv->last_in_dts + duration, 0, gap_time); priv->last_in_pts + duration, 0, gap_time);
expected += lost_packets; expected += lost_packets;
priv->last_in_dts += gap_time; priv->last_in_pts += gap_time;
} }
} }
expected_dts = priv->last_in_dts + duration; expected_pts = priv->last_in_pts + duration;
} else { } else {
/* If we cannot assume equidistant packet spacing, the only thing we now /* If we cannot assume equidistant packet spacing, the only thing we now
* for sure is that the missing packets have expected dts not later than * for sure is that the missing packets have expected pts not later than
* the last received dts. */ * the last received pts. */
duration = 0; duration = 0;
expected_dts = dts; expected_pts = pts;
} }
delay = 0; delay = 0;
@ -2516,20 +2516,20 @@ calculate_expected (GstRtpJitterBuffer * jitterbuffer, guint32 expected,
GstClockTime timeout = timer->timeout; GstClockTime timeout = timer->timeout;
timer->duration = duration; timer->duration = duration;
if (timeout > (expected_dts + delay) && timer->num_rtx_retry == 0) { if (timeout > (expected_pts + delay) && timer->num_rtx_retry == 0) {
reschedule_timer (jitterbuffer, timer, timer->seqnum, expected_dts, reschedule_timer (jitterbuffer, timer, timer->seqnum, expected_pts,
delay, TRUE); delay, TRUE);
} }
expected++; expected++;
expected_dts += duration; expected_pts += duration;
} }
} else { } else {
type = TIMER_TYPE_LOST; type = TIMER_TYPE_LOST;
} }
while (gst_rtp_buffer_compare_seqnum (expected, seqnum) > 0) { while (gst_rtp_buffer_compare_seqnum (expected, seqnum) > 0) {
add_timer (jitterbuffer, type, expected, 0, expected_dts, delay, duration); add_timer (jitterbuffer, type, expected, 0, expected_pts, delay, duration);
expected_dts += duration; expected_pts += duration;
expected++; expected++;
} }
} }
@ -2617,13 +2617,13 @@ compare_buffer_seqnum (GstBuffer * a, GstBuffer * b, gpointer user_data)
} }
static gboolean static gboolean
handle_big_gap_buffer (GstRtpJitterBuffer * jitterbuffer, gboolean future, handle_big_gap_buffer (GstRtpJitterBuffer * jitterbuffer, GstBuffer * buffer,
GstBuffer * buffer, guint8 pt, guint16 seqnum, gint gap, guint max_dropout, guint8 pt, guint16 seqnum, gint gap, guint max_dropout, guint max_misorder)
guint max_misorder)
{ {
GstRtpJitterBufferPrivate *priv; GstRtpJitterBufferPrivate *priv;
guint gap_packets_length; guint gap_packets_length;
gboolean reset = FALSE; gboolean reset = FALSE;
gboolean future = gap > 0;
priv = jitterbuffer->priv; priv = jitterbuffer->priv;
@ -2711,6 +2711,78 @@ get_current_running_time (GstRtpJitterBuffer * jitterbuffer)
return running_time; return running_time;
} }
static GstFlowReturn
gst_rtp_jitter_buffer_reset (GstRtpJitterBuffer * jitterbuffer,
GstPad * pad, GstObject * parent, guint16 seqnum)
{
GstRtpJitterBufferPrivate *priv = jitterbuffer->priv;
GstFlowReturn ret = GST_FLOW_OK;
GList *events = NULL, *l;
GList *buffers;
gboolean head;
GST_DEBUG_OBJECT (jitterbuffer, "flush and reset jitterbuffer");
rtp_jitter_buffer_flush (priv->jbuf,
(GFunc) free_item_and_retain_events, &events);
rtp_jitter_buffer_reset_skew (priv->jbuf);
remove_all_timers (jitterbuffer);
priv->discont = TRUE;
priv->last_popped_seqnum = -1;
if (priv->gap_packets.head) {
GstBuffer *gap_buffer = priv->gap_packets.head->data;
GstRTPBuffer gap_rtp = GST_RTP_BUFFER_INIT;
gst_rtp_buffer_map (gap_buffer, GST_MAP_READ, &gap_rtp);
priv->next_seqnum = gst_rtp_buffer_get_seq (&gap_rtp);
gst_rtp_buffer_unmap (&gap_rtp);
} else {
priv->next_seqnum = seqnum;
}
priv->last_in_pts = -1;
priv->next_in_seqnum = -1;
/* Insert all sticky events again in order, otherwise we would
* potentially loose STREAM_START, CAPS or SEGMENT events
*/
events = g_list_reverse (events);
for (l = events; l; l = l->next) {
RTPJitterBufferItem *item;
item = alloc_item (l->data, ITEM_TYPE_EVENT, -1, -1, -1, 0, -1);
rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL);
}
g_list_free (events);
JBUF_SIGNAL_EVENT (priv);
/* reset spacing estimation when gap */
priv->ips_rtptime = -1;
priv->ips_pts = GST_CLOCK_TIME_NONE;
buffers = g_list_copy (priv->gap_packets.head);
g_queue_clear (&priv->gap_packets);
priv->ips_rtptime = -1;
priv->ips_pts = GST_CLOCK_TIME_NONE;
JBUF_UNLOCK (jitterbuffer->priv);
for (l = buffers; l; l = l->next) {
ret = gst_rtp_jitter_buffer_chain (pad, parent, l->data);
l->data = NULL;
if (ret != GST_FLOW_OK) {
l = l->next;
break;
}
}
for (; l; l = l->next)
gst_buffer_unref (l->data);
g_list_free (buffers);
return ret;
}
static GstFlowReturn static GstFlowReturn
gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent,
GstBuffer * buffer) GstBuffer * buffer)
@ -2730,7 +2802,7 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent,
RTPJitterBufferItem *item; RTPJitterBufferItem *item;
GstMessage *msg = NULL; GstMessage *msg = NULL;
gboolean estimated_dts = FALSE; gboolean estimated_dts = FALSE;
guint32 packet_rate, max_dropout, max_misorder; gint32 packet_rate, max_dropout, max_misorder;
TimerData *timer = NULL; TimerData *timer = NULL;
jitterbuffer = GST_RTP_JITTER_BUFFER_CAST (parent); jitterbuffer = GST_RTP_JITTER_BUFFER_CAST (parent);
@ -2828,7 +2900,6 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent,
"packet seqnum #%d before seqnum-base #%d", seqnum, "packet seqnum #%d before seqnum-base #%d", seqnum,
priv->seqnum_base); priv->seqnum_base);
gst_buffer_unref (buffer); gst_buffer_unref (buffer);
ret = GST_FLOW_OK;
goto finished; goto finished;
} else if (gap > 16384) { } else if (gap > 16384) {
/* From now on don't compare against the seqnum base anymore as /* From now on don't compare against the seqnum base anymore as
@ -2853,156 +2924,90 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent,
max_dropout, max_misorder); max_dropout, max_misorder);
/* now check against our expected seqnum */ /* now check against our expected seqnum */
if (G_LIKELY (expected != -1)) { if (G_UNLIKELY (expected == -1)) {
gint gap;
/* now calculate gap */
gap = gst_rtp_buffer_compare_seqnum (expected, seqnum);
GST_DEBUG_OBJECT (jitterbuffer, "expected #%d, got #%d, gap of %d",
expected, seqnum, gap);
if (G_LIKELY (gap == 0)) {
/* packet is expected */
calculate_packet_spacing (jitterbuffer, rtptime, dts);
do_next_seqnum = TRUE;
} else {
gboolean reset = FALSE;
if (gap < 0) {
/* we received an old packet */
if (G_UNLIKELY (gap != -1 && gap < -max_misorder)) {
reset =
handle_big_gap_buffer (jitterbuffer, FALSE, buffer, pt, seqnum,
gap, max_dropout, max_misorder);
buffer = NULL;
} else {
GST_DEBUG_OBJECT (jitterbuffer, "old packet received");
}
} else {
/* new packet, we are missing some packets */
if (G_UNLIKELY (priv->timers->len >= max_dropout)) {
/* If we have timers for more than RTP_MAX_DROPOUT packets
* pending this means that we have a huge gap overall. We can
* reset the jitterbuffer at this point because there's
* just too much data missing to be able to do anything
* sensible with the past data. Just try again from the
* next packet */
GST_WARNING_OBJECT (jitterbuffer,
"%d pending timers > %d - resetting", priv->timers->len,
max_dropout);
reset = TRUE;
gst_buffer_unref (buffer);
buffer = NULL;
} else if (G_UNLIKELY (gap >= max_dropout)) {
reset =
handle_big_gap_buffer (jitterbuffer, TRUE, buffer, pt, seqnum,
gap, max_dropout, max_misorder);
buffer = NULL;
} else {
GST_DEBUG_OBJECT (jitterbuffer, "%d missing packets", gap);
/* fill in the gap with EXPECTED timers */
calculate_expected (jitterbuffer, expected, seqnum, dts, gap);
do_next_seqnum = TRUE;
}
}
if (G_UNLIKELY (reset)) {
GList *events = NULL, *l;
GList *buffers;
GST_DEBUG_OBJECT (jitterbuffer, "flush and reset jitterbuffer");
rtp_jitter_buffer_flush (priv->jbuf,
(GFunc) free_item_and_retain_events, &events);
rtp_jitter_buffer_reset_skew (priv->jbuf);
remove_all_timers (jitterbuffer);
priv->discont = TRUE;
priv->last_popped_seqnum = -1;
if (priv->gap_packets.head) {
GstBuffer *gap_buffer = priv->gap_packets.head->data;
GstRTPBuffer gap_rtp = GST_RTP_BUFFER_INIT;
gst_rtp_buffer_map (gap_buffer, GST_MAP_READ, &gap_rtp);
priv->next_seqnum = gst_rtp_buffer_get_seq (&gap_rtp);
gst_rtp_buffer_unmap (&gap_rtp);
} else {
priv->next_seqnum = seqnum;
}
priv->last_in_dts = -1;
priv->next_in_seqnum = -1;
/* Insert all sticky events again in order, otherwise we would
* potentially loose STREAM_START, CAPS or SEGMENT events
*/
events = g_list_reverse (events);
for (l = events; l; l = l->next) {
RTPJitterBufferItem *item;
item = alloc_item (l->data, ITEM_TYPE_EVENT, -1, -1, -1, 0, -1);
rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL, -1);
}
g_list_free (events);
JBUF_SIGNAL_EVENT (priv);
/* reset spacing estimation when gap */
priv->ips_rtptime = -1;
priv->ips_dts = GST_CLOCK_TIME_NONE;
buffers = g_list_copy (priv->gap_packets.head);
g_queue_clear (&priv->gap_packets);
priv->ips_rtptime = -1;
priv->ips_dts = GST_CLOCK_TIME_NONE;
JBUF_UNLOCK (jitterbuffer->priv);
for (l = buffers; l; l = l->next) {
ret = gst_rtp_jitter_buffer_chain (pad, parent, l->data);
l->data = NULL;
if (ret != GST_FLOW_OK) {
l = l->next;
break;
}
}
for (; l; l = l->next)
gst_buffer_unref (l->data);
g_list_free (buffers);
return ret;
}
/* reset spacing estimation when gap */
priv->ips_rtptime = -1;
priv->ips_dts = GST_CLOCK_TIME_NONE;
}
} else {
GST_DEBUG_OBJECT (jitterbuffer, "First buffer #%d", seqnum); GST_DEBUG_OBJECT (jitterbuffer, "First buffer #%d", seqnum);
/* calculate a pts based on rtptime and arrival time (dts) */
pts = rtp_jitter_buffer_calculate_pts (priv->jbuf, dts, rtptime,
gst_element_get_base_time (GST_ELEMENT_CAST (jitterbuffer)));
/* we don't know what the next_in_seqnum should be, wait for the last /* we don't know what the next_in_seqnum should be, wait for the last
* possible moment to push this buffer, maybe we get an earlier seqnum * possible moment to push this buffer, maybe we get an earlier seqnum
* while we wait */ * while we wait */
set_timer (jitterbuffer, TIMER_TYPE_DEADLINE, seqnum, dts); set_timer (jitterbuffer, TIMER_TYPE_DEADLINE, seqnum, pts);
do_next_seqnum = TRUE;
/* take rtptime and dts to calculate packet spacing */
priv->ips_rtptime = rtptime;
priv->ips_dts = dts;
}
/* We had no huge gap, let's drop all the gap packets */ do_next_seqnum = TRUE;
if (buffer != NULL) { /* take rtptime and pts to calculate packet spacing */
priv->ips_rtptime = rtptime;
priv->ips_pts = pts;
} else {
gint gap;
/* now calculate gap */
gap = gst_rtp_buffer_compare_seqnum (expected, seqnum);
GST_DEBUG_OBJECT (jitterbuffer, "expected #%d, got #%d, gap of %d",
expected, seqnum, gap);
if (G_UNLIKELY (gap > 0 && priv->timers->len >= max_dropout)) {
/* If we have timers for more than RTP_MAX_DROPOUT packets
* pending this means that we have a huge gap overall. We can
* reset the jitterbuffer at this point because there's
* just too much data missing to be able to do anything
* sensible with the past data. Just try again from the
* next packet */
GST_WARNING_OBJECT (jitterbuffer, "%d pending timers > %d - resetting",
priv->timers->len, max_dropout);
gst_buffer_unref (buffer);
return gst_rtp_jitter_buffer_reset (jitterbuffer, pad, parent, seqnum);
}
/* Special handling of large gaps */
if ((gap != -1 && gap < -max_misorder) || (gap >= max_dropout)) {
gboolean reset = handle_big_gap_buffer (jitterbuffer, buffer, pt, seqnum,
gap, max_dropout, max_misorder);
if (reset) {
return gst_rtp_jitter_buffer_reset (jitterbuffer, pad, parent, seqnum);
} else {
GST_DEBUG_OBJECT (jitterbuffer,
"Had big gap, waiting for more consecutive packets");
goto finished;
}
}
/* We had no huge gap, let's drop all the gap packets */
GST_DEBUG_OBJECT (jitterbuffer, "Clearing gap packets"); GST_DEBUG_OBJECT (jitterbuffer, "Clearing gap packets");
g_queue_foreach (&priv->gap_packets, (GFunc) gst_buffer_unref, NULL); g_queue_foreach (&priv->gap_packets, (GFunc) gst_buffer_unref, NULL);
g_queue_clear (&priv->gap_packets); g_queue_clear (&priv->gap_packets);
} else {
GST_DEBUG_OBJECT (jitterbuffer, /* calculate a pts based on rtptime and arrival time (dts) */
"Had big gap, waiting for more consecutive packets"); pts = rtp_jitter_buffer_calculate_pts (priv->jbuf, dts, rtptime,
JBUF_UNLOCK (jitterbuffer->priv); gst_element_get_base_time (GST_ELEMENT_CAST (jitterbuffer)));
return GST_FLOW_OK;
if (G_LIKELY (gap == 0)) {
/* packet is expected */
calculate_packet_spacing (jitterbuffer, rtptime, pts);
do_next_seqnum = TRUE;
} else {
/* we have a gap */
if (gap > 0) {
GST_DEBUG_OBJECT (jitterbuffer, "%d missing packets", gap);
/* fill in the gap with EXPECTED timers */
calculate_expected (jitterbuffer, expected, seqnum, pts, gap);
do_next_seqnum = TRUE;
} else {
GST_DEBUG_OBJECT (jitterbuffer, "old packet received");
do_next_seqnum = FALSE;
}
/* reset spacing estimation when gap */
priv->ips_rtptime = -1;
priv->ips_pts = GST_CLOCK_TIME_NONE;
}
} }
if (do_next_seqnum) { if (do_next_seqnum) {
priv->last_in_dts = dts; priv->last_in_pts = pts;
priv->next_in_seqnum = (seqnum + 1) & 0xffff; priv->next_in_seqnum = (seqnum + 1) & 0xffff;
} }
@ -3073,23 +3078,22 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent,
if (estimated_dts) if (estimated_dts)
item = item =
alloc_item (buffer, ITEM_TYPE_BUFFER, GST_CLOCK_TIME_NONE, alloc_item (buffer, ITEM_TYPE_BUFFER, GST_CLOCK_TIME_NONE,
GST_CLOCK_TIME_NONE, seqnum, 1, rtptime); pts, seqnum, 1, rtptime);
else else
item = alloc_item (buffer, ITEM_TYPE_BUFFER, dts, pts, seqnum, 1, rtptime); item = alloc_item (buffer, ITEM_TYPE_BUFFER, dts, pts, seqnum, 1, rtptime);
/* now insert the packet into the queue in sorted order. This function returns /* now insert the packet into the queue in sorted order. This function returns
* FALSE if a packet with the same seqnum was already in the queue, meaning we * FALSE if a packet with the same seqnum was already in the queue, meaning we
* have a duplicate. */ * have a duplicate. */
if (G_UNLIKELY (!rtp_jitter_buffer_insert (priv->jbuf, item, if (G_UNLIKELY (!rtp_jitter_buffer_insert (priv->jbuf, item, &head,
&head, &percent, &percent))) {
gst_element_get_base_time (GST_ELEMENT_CAST (jitterbuffer))))) {
if (GST_BUFFER_IS_RETRANSMISSION (buffer) && timer) if (GST_BUFFER_IS_RETRANSMISSION (buffer) && timer)
update_rtx_stats (jitterbuffer, timer, dts, FALSE); update_rtx_stats (jitterbuffer, timer, dts, FALSE);
goto duplicate; goto duplicate;
} }
/* update timers */ /* update timers */
update_timers (jitterbuffer, seqnum, dts, do_next_seqnum, update_timers (jitterbuffer, seqnum, dts, pts, do_next_seqnum,
GST_BUFFER_IS_RETRANSMISSION (buffer), timer); GST_BUFFER_IS_RETRANSMISSION (buffer), timer);
/* we had an unhandled SR, handle it now */ /* we had an unhandled SR, handle it now */
@ -3258,7 +3262,7 @@ update_estimated_eos (GstRtpJitterBuffer * jitterbuffer,
} }
/* this is the current time as running-time */ /* this is the current time as running-time */
out_time = item->dts; out_time = item->pts;
if (elapsed > 0) if (elapsed > 0)
estimated = gst_util_uint64_scale (out_time, total, elapsed); estimated = gst_util_uint64_scale (out_time, total, elapsed);
@ -3731,7 +3735,7 @@ do_lost_timeout (GstRtpJitterBuffer * jitterbuffer, TimerData * timer,
/* we now only accept seqnum bigger than this */ /* we now only accept seqnum bigger than this */
if (gst_rtp_buffer_compare_seqnum (priv->next_in_seqnum, next_in_seqnum) > 0) { if (gst_rtp_buffer_compare_seqnum (priv->next_in_seqnum, next_in_seqnum) > 0) {
priv->next_in_seqnum = next_in_seqnum; priv->next_in_seqnum = next_in_seqnum;
priv->last_in_dts = apply_offset (jitterbuffer, timer->timeout); priv->last_in_pts = apply_offset (jitterbuffer, timer->timeout);
} }
/* Avoid creating events if we don't need it. Note that we still need to create /* Avoid creating events if we don't need it. Note that we still need to create
@ -3752,7 +3756,7 @@ do_lost_timeout (GstRtpJitterBuffer * jitterbuffer, TimerData * timer,
"retry", G_TYPE_UINT, num_rtx_retry, NULL)); "retry", G_TYPE_UINT, num_rtx_retry, NULL));
} }
item = alloc_item (event, ITEM_TYPE_LOST, -1, -1, seqnum, lost_packets, -1); item = alloc_item (event, ITEM_TYPE_LOST, -1, -1, seqnum, lost_packets, -1);
rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL, -1); rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL);
if (GST_CLOCK_TIME_IS_VALID (timer->rtx_last)) { if (GST_CLOCK_TIME_IS_VALID (timer->rtx_last)) {
/* Store info to update stats if the packet arrives too late */ /* Store info to update stats if the packet arrives too late */
@ -4250,7 +4254,7 @@ gst_rtp_jitter_buffer_sink_query (GstPad * pad, GstObject * parent,
RTP_JITTER_BUFFER_MODE_BUFFER) { RTP_JITTER_BUFFER_MODE_BUFFER) {
GST_DEBUG_OBJECT (jitterbuffer, "adding serialized query"); GST_DEBUG_OBJECT (jitterbuffer, "adding serialized query");
item = alloc_item (query, ITEM_TYPE_QUERY, -1, -1, -1, 0, -1); item = alloc_item (query, ITEM_TYPE_QUERY, -1, -1, -1, 0, -1);
rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL, -1); rtp_jitter_buffer_insert (priv->jbuf, item, &head, NULL);
if (head) if (head)
JBUF_SIGNAL_EVENT (priv); JBUF_SIGNAL_EVENT (priv);
JBUF_WAIT_QUERY (priv, out_flushing); JBUF_WAIT_QUERY (priv, out_flushing);

195
gst/rtpmanager/rtpjitterbuffer.c
View file

@ -688,92 +688,16 @@ queue_do_insert (RTPJitterBuffer * jbuf, GList * list, GList * item)
queue->length++; queue->length++;
} }
/** GstClockTime
* rtp_jitter_buffer_insert: rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts,
* @jbuf: an #RTPJitterBuffer guint32 rtptime, GstClockTime base_time)
* @item: an #RTPJitterBufferItem to insert
* @head: TRUE when the head element changed.
* @percent: the buffering percent after insertion
* @base_time: base time of the pipeline
*
* Inserts @item into the packet queue of @jbuf. The sequence number of the
* packet will be used to sort the packets. This function takes ownerhip of
* @buf when the function returns %TRUE.
*
* When @head is %TRUE, the new packet was added at the head of the queue and
* will be available with the next call to rtp_jitter_buffer_pop() and
* rtp_jitter_buffer_peek().
*
* Returns: %FALSE if a packet with the same number already existed.
*/
gboolean
rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
gboolean * head, gint * percent, GstClockTime base_time)
{ {
GList *list, *event = NULL;
guint32 rtptime;
guint64 ext_rtptime; guint64 ext_rtptime;
guint16 seqnum; GstClockTime gstrtptime, pts;
GstClockTime gstrtptime, dts;
GstClock *media_clock, *pipeline_clock; GstClock *media_clock, *pipeline_clock;
guint64 media_clock_offset; guint64 media_clock_offset;
gboolean rfc7273_mode; gboolean rfc7273_mode;
g_return_val_if_fail (jbuf != NULL, FALSE);
g_return_val_if_fail (item != NULL, FALSE);
list = jbuf->packets->tail;
/* no seqnum, simply append then */
if (item->seqnum == -1)
goto append;
seqnum = item->seqnum;
/* loop the list to skip strictly larger seqnum buffers */
for (; list; list = g_list_previous (list)) {
guint16 qseq;
gint gap;
RTPJitterBufferItem *qitem = (RTPJitterBufferItem *) list;
if (qitem->seqnum == -1) {
/* keep a pointer to the first consecutive event if not already
* set. we will insert the packet after the event if we can't find
* a packet with lower sequence number before the event. */
if (event == NULL)
event = list;
continue;
}
qseq = qitem->seqnum;
/* compare the new seqnum to the one in the buffer */
gap = gst_rtp_buffer_compare_seqnum (seqnum, qseq);
/* we hit a packet with the same seqnum, notify a duplicate */
if (G_UNLIKELY (gap == 0))
goto duplicate;
/* seqnum > qseq, we can stop looking */
if (G_LIKELY (gap < 0))
break;
/* if we've found a packet with greater sequence number, cleanup the
* event pointer as the packet will be inserted before the event */
event = NULL;
}
/* if event is set it means that packets before the event had smaller
* sequence number, so we will insert our packet after the event */
if (event)
list = event;
dts = item->dts;
if (item->rtptime == -1)
goto append;
rtptime = item->rtptime;
/* rtp time jumps are checked for during skew calculation, but bypassed /* rtp time jumps are checked for during skew calculation, but bypassed
* in other mode, so mind those here and reset jb if needed. * in other mode, so mind those here and reset jb if needed.
* Only reset if valid input time, which is likely for UDP input * Only reset if valid input time, which is likely for UDP input
@ -797,8 +721,7 @@ rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
/* Return the last time if we got the same RTP timestamp again */ /* Return the last time if we got the same RTP timestamp again */
ext_rtptime = gst_rtp_buffer_ext_timestamp (&jbuf->ext_rtptime, rtptime); ext_rtptime = gst_rtp_buffer_ext_timestamp (&jbuf->ext_rtptime, rtptime);
if (jbuf->last_rtptime != -1 && ext_rtptime == jbuf->last_rtptime) { if (jbuf->last_rtptime != -1 && ext_rtptime == jbuf->last_rtptime) {
item->pts = jbuf->prev_out_time; return jbuf->prev_out_time;
goto append;
} }
/* keep track of the last extended rtptime */ /* keep track of the last extended rtptime */
@ -906,12 +829,12 @@ rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
external, rate_num, rate_denom); external, rate_num, rate_denom);
if (rtpsystime > base_time) if (rtpsystime > base_time)
item->pts = rtpsystime - base_time; pts = rtpsystime - base_time;
else else
item->pts = 0; pts = 0;
GST_DEBUG ("RFC7273 clock time %" GST_TIME_FORMAT ", out %" GST_TIME_FORMAT, GST_DEBUG ("RFC7273 clock time %" GST_TIME_FORMAT ", out %" GST_TIME_FORMAT,
GST_TIME_ARGS (rtpsystime), GST_TIME_ARGS (item->pts)); GST_TIME_ARGS (rtpsystime), GST_TIME_ARGS (pts));
} else if (rfc7273_mode && (jbuf->mode == RTP_JITTER_BUFFER_MODE_SLAVE } else if (rfc7273_mode && (jbuf->mode == RTP_JITTER_BUFFER_MODE_SLAVE
|| jbuf->mode == RTP_JITTER_BUFFER_MODE_SYNCED) || jbuf->mode == RTP_JITTER_BUFFER_MODE_SYNCED)
&& media_clock_offset != -1 && jbuf->rfc7273_sync) { && media_clock_offset != -1 && jbuf->rfc7273_sync) {
@ -960,12 +883,12 @@ rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
/* All this assumes that the pipeline has enough additional /* All this assumes that the pipeline has enough additional
* latency to cover for the network delay */ * latency to cover for the network delay */
if (rtpsystime > base_time) if (rtpsystime > base_time)
item->pts = rtpsystime - base_time; pts = rtpsystime - base_time;
else else
item->pts = 0; pts = 0;
GST_DEBUG ("RFC7273 clock time %" GST_TIME_FORMAT ", out %" GST_TIME_FORMAT, GST_DEBUG ("RFC7273 clock time %" GST_TIME_FORMAT ", out %" GST_TIME_FORMAT,
GST_TIME_ARGS (rtpsystime), GST_TIME_ARGS (item->pts)); GST_TIME_ARGS (rtpsystime), GST_TIME_ARGS (pts));
} else { } else {
/* If we used the RFC7273 clock before and not anymore, /* If we used the RFC7273 clock before and not anymore,
* we need to resync it later again */ * we need to resync it later again */
@ -973,39 +896,40 @@ rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
/* do skew calculation by measuring the difference between rtptime and the /* do skew calculation by measuring the difference between rtptime and the
* receive dts, this function will return the skew corrected rtptime. */ * receive dts, this function will return the skew corrected rtptime. */
item->pts = calculate_skew (jbuf, ext_rtptime, gstrtptime, dts); pts = calculate_skew (jbuf, ext_rtptime, gstrtptime, dts);
} }
/* check if timestamps are not going backwards, we can only check this if we /* check if timestamps are not going backwards, we can only check this if we
* have a previous out time and a previous send_diff */ * have a previous out time and a previous send_diff */
if (G_LIKELY (item->pts != -1 && jbuf->prev_out_time != -1 if (G_LIKELY (pts != -1 && jbuf->prev_out_time != -1
&& jbuf->prev_send_diff != -1)) { && jbuf->prev_send_diff != -1)) {
/* now check for backwards timestamps */ /* now check for backwards timestamps */
if (G_UNLIKELY ( if (G_UNLIKELY (
/* if the server timestamps went up and the out_time backwards */ /* if the server timestamps went up and the out_time backwards */
(gstrtptime - jbuf->base_rtptime > jbuf->prev_send_diff (gstrtptime - jbuf->base_rtptime > jbuf->prev_send_diff
&& item->pts < jbuf->prev_out_time) || && pts < jbuf->prev_out_time) ||
/* if the server timestamps went backwards and the out_time forwards */ /* if the server timestamps went backwards and the out_time forwards */
(gstrtptime - jbuf->base_rtptime < jbuf->prev_send_diff (gstrtptime - jbuf->base_rtptime < jbuf->prev_send_diff
&& item->pts > jbuf->prev_out_time) || && pts > jbuf->prev_out_time) ||
/* if the server timestamps did not change */ /* if the server timestamps did not change */
gstrtptime - jbuf->base_rtptime == jbuf->prev_send_diff)) { gstrtptime - jbuf->base_rtptime == jbuf->prev_send_diff)) {
GST_DEBUG ("backwards timestamps, using previous time"); GST_DEBUG ("backwards timestamps, using previous time");
item->pts = jbuf->prev_out_time; pts = jbuf->prev_out_time;
} }
} }
if (dts != -1 && item->pts + jbuf->delay < dts) {
if (dts != -1 && pts + jbuf->delay < dts) {
/* if we are going to produce a timestamp that is later than the input /* if we are going to produce a timestamp that is later than the input
* timestamp, we need to reset the jitterbuffer. Likely the server paused * timestamp, we need to reset the jitterbuffer. Likely the server paused
* temporarily */ * temporarily */
GST_DEBUG ("out %" GST_TIME_FORMAT " + %" G_GUINT64_FORMAT " < time %" GST_DEBUG ("out %" GST_TIME_FORMAT " + %" G_GUINT64_FORMAT " < time %"
GST_TIME_FORMAT ", reset jitterbuffer", GST_TIME_ARGS (item->pts), GST_TIME_FORMAT ", reset jitterbuffer", GST_TIME_ARGS (pts),
jbuf->delay, GST_TIME_ARGS (dts)); jbuf->delay, GST_TIME_ARGS (dts));
rtp_jitter_buffer_resync (jbuf, dts, gstrtptime, ext_rtptime, TRUE); rtp_jitter_buffer_resync (jbuf, dts, gstrtptime, ext_rtptime, TRUE);
item->pts = dts; pts = dts;
} }
jbuf->prev_out_time = item->pts; jbuf->prev_out_time = pts;
jbuf->prev_send_diff = gstrtptime - jbuf->base_rtptime; jbuf->prev_send_diff = gstrtptime - jbuf->base_rtptime;
if (media_clock) if (media_clock)
@ -1013,6 +937,83 @@ rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
if (pipeline_clock) if (pipeline_clock)
gst_object_unref (pipeline_clock); gst_object_unref (pipeline_clock);
return pts;
}
/**
* rtp_jitter_buffer_insert:
* @jbuf: an #RTPJitterBuffer
* @item: an #RTPJitterBufferItem to insert
* @head: TRUE when the head element changed.
* @percent: the buffering percent after insertion
*
* Inserts @item into the packet queue of @jbuf. The sequence number of the
* packet will be used to sort the packets. This function takes ownerhip of
* @buf when the function returns %TRUE.
*
* When @head is %TRUE, the new packet was added at the head of the queue and
* will be available with the next call to rtp_jitter_buffer_pop() and
* rtp_jitter_buffer_peek().
*
* Returns: %FALSE if a packet with the same number already existed.
*/
gboolean
rtp_jitter_buffer_insert (RTPJitterBuffer * jbuf, RTPJitterBufferItem * item,
gboolean * head, gint * percent)
{
GList *list, *event = NULL;
guint16 seqnum;
g_return_val_if_fail (jbuf != NULL, FALSE);
g_return_val_if_fail (item != NULL, FALSE);
list = jbuf->packets->tail;
/* no seqnum, simply append then */
if (item->seqnum == -1)
goto append;
seqnum = item->seqnum;
/* loop the list to skip strictly larger seqnum buffers */
for (; list; list = g_list_previous (list)) {
guint16 qseq;
gint gap;
RTPJitterBufferItem *qitem = (RTPJitterBufferItem *) list;
if (qitem->seqnum == -1) {
/* keep a pointer to the first consecutive event if not already
* set. we will insert the packet after the event if we can't find
* a packet with lower sequence number before the event. */
if (event == NULL)
event = list;
continue;
}
qseq = qitem->seqnum;
/* compare the new seqnum to the one in the buffer */
gap = gst_rtp_buffer_compare_seqnum (seqnum, qseq);
/* we hit a packet with the same seqnum, notify a duplicate */
if (G_UNLIKELY (gap == 0))
goto duplicate;
/* seqnum > qseq, we can stop looking */
if (G_LIKELY (gap < 0))
break;
/* if we've found a packet with greater sequence number, cleanup the
* event pointer as the packet will be inserted before the event */
event = NULL;
}
/* if event is set it means that packets before the event had smaller
* sequence number, so we will insert our packet after the event */
if (event)
list = event;
append: append:
queue_do_insert (jbuf, list, (GList *) item); queue_do_insert (jbuf, list, (GList *) item);

6
gst/rtpmanager/rtpjitterbuffer.h
View file

@ -168,8 +168,7 @@ void rtp_jitter_buffer_reset_skew (RTPJitterBuffer *jbuf)
gboolean rtp_jitter_buffer_insert (RTPJitterBuffer *jbuf, gboolean rtp_jitter_buffer_insert (RTPJitterBuffer *jbuf,
RTPJitterBufferItem *item, RTPJitterBufferItem *item,
gboolean *head, gint *percent, gboolean *head, gint *percent);
GstClockTime base_time);
void rtp_jitter_buffer_disable_buffering (RTPJitterBuffer *jbuf, gboolean disabled); void rtp_jitter_buffer_disable_buffering (RTPJitterBuffer *jbuf, gboolean disabled);
@ -190,4 +189,7 @@ void rtp_jitter_buffer_get_sync (RTPJitterBuffer *jbuf,
guint64 *timestamp, guint32 *clock_rate, guint64 *timestamp, guint32 *clock_rate,
guint64 *last_rtptime); guint64 *last_rtptime);
GstClockTime rtp_jitter_buffer_calculate_pts (RTPJitterBuffer * jbuf, GstClockTime dts,
guint32 rtptime, GstClockTime base_time);
#endif /* __RTP_JITTER_BUFFER_H__ */ #endif /* __RTP_JITTER_BUFFER_H__ */

374
tests/check/elements/rtpjitterbuffer.c
View file

@ -860,6 +860,148 @@ GST_START_TEST (test_num_late_when_considered_lost_arrives)
GST_END_TEST; GST_END_TEST;
GST_START_TEST (test_lost_event_uses_pts)
{
GstHarness *h = gst_harness_new ("rtpjitterbuffer");
GstEvent *out_event;
GstClockTime now;
gint jb_latency_ms = 100;
gint i;
gst_harness_set_src_caps (h, generate_caps ());
g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL);
/* push the first buffer through */
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (0)));
fail_unless (gst_harness_crank_single_clock_wait (h));
gst_buffer_unref (gst_harness_pull (h));
/* push some buffers arriving in perfect time! */
for (i = 1; i <= 2; i++) {
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (i)));
gst_buffer_unref (gst_harness_pull (h));
}
/* hop over 1 packets (seqnum 3) and make another one (gap of 1), but due to
network delays, this packets is also grossly late */
i = 4;
/* advance the clock to the latest possible time buffer 4 could arrive */
now = i * PCMU_BUF_DURATION + jb_latency_ms * GST_MSECOND;
gst_harness_set_time (h, now);
gst_harness_push (h, generate_test_buffer_full (now, FALSE, i, i * PCMU_RTP_TS_DURATION));
/* drop GstEventStreamStart & GstEventCaps & GstEventSegment */
for (int i = 0; i < 3; i++)
gst_event_unref (gst_harness_pull_event (h));
/* we should now have received a packet-lost-event for buffer 3 */
out_event = gst_harness_pull_event (h);
verify_lost_event (out_event, 3, 3 * PCMU_BUF_DURATION, PCMU_BUF_DURATION);
/* and pull out packet 4 */
gst_buffer_unref (gst_harness_pull (h));
fail_unless (verify_jb_stats (h->element,
gst_structure_new ("application/x-rtp-jitterbuffer-stats",
"num-pushed", G_TYPE_UINT64, (guint64) 4,
"num-lost", G_TYPE_UINT64, (guint64) 1, NULL)));
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (test_lost_event_with_backwards_rtptime)
{
GstHarness *h = gst_harness_new ("rtpjitterbuffer");
GstEvent *out_event;
gint jb_latency_ms = 100;
gint i;
gst_harness_set_src_caps (h, generate_caps ());
g_object_set (h->element, "do-lost", TRUE, "latency", jb_latency_ms, NULL);
/* push the first buffer through */
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (0)));
fail_unless (gst_harness_crank_single_clock_wait (h));
gst_buffer_unref (gst_harness_pull (h));
/* push some buffers arriving in perfect time! */
for (i = 1; i <= 2; i++) {
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (i)));
gst_buffer_unref (gst_harness_pull (h));
}
/*
For video using B-frames, an expected sequence
could be like this:
(I = I-frame, P = P-frame, B = B-frame)
___ ___ ___ ___ ___
... | 3 | | 4 | | 5 | | 6 | | 7 |
––– ––– ––– ––– –––
rtptime: 3(I) 5(P) 5(P) 4(B) 6(P)
arrival(dts): 3 5 5 5 6
Notice here that packet 6 (the B frame) make
the rtptime go backwards.
But we get this:
___ ___ _ _ ___ ___
... | 3 | | 4 | | | | 6 | | 7 |
––– ––– - - ––– –––
rtptime: 3(I) 5(P) 4(B) 6(P)
arrival(dts): 3 5 5 6
*/
/* seqnum 3 */
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (3)));
gst_buffer_unref (gst_harness_pull (h));
/* seqnum 4, arriving at time 5 with rtptime 5 */
gst_harness_push (h, generate_test_buffer_full (
5 * PCMU_BUF_DURATION, FALSE, 4, 5 * PCMU_RTP_TS_DURATION));
gst_buffer_unref (gst_harness_pull (h));
/* seqnum 6, arriving at time 5 with rtptime 4,
making a gap for missing seqnum 5 */
gst_harness_push (h, generate_test_buffer_full (
5 * PCMU_BUF_DURATION, FALSE, 6, 4 * PCMU_RTP_TS_DURATION));
/* seqnum 7, arriving at time 6 with rtptime 6 */
gst_harness_push (h, generate_test_buffer_full (
6 * PCMU_BUF_DURATION, FALSE, 7, 6 * PCMU_RTP_TS_DURATION));
/* drop GstEventStreamStart & GstEventCaps & GstEventSegment */
for (int i = 0; i < 3; i++)
gst_event_unref (gst_harness_pull_event (h));
/* we should now have received a packet-lost-event for seqnum 5,
with time 5 and 0 duration */
gst_harness_crank_single_clock_wait (h);
out_event = gst_harness_pull_event (h);
verify_lost_event (out_event, 5, 5 * PCMU_BUF_DURATION, 0);
/* and pull out 6 and 7 */
gst_buffer_unref (gst_harness_pull (h));
gst_buffer_unref (gst_harness_pull (h));
fail_unless (verify_jb_stats (h->element,
gst_structure_new ("application/x-rtp-jitterbuffer-stats",
"num-pushed", G_TYPE_UINT64, (guint64) 7,
"num-lost", G_TYPE_UINT64, (guint64) 1, NULL)));
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (test_all_packets_are_timestamped_zero) GST_START_TEST (test_all_packets_are_timestamped_zero)
{ {
GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstHarness *h = gst_harness_new ("rtpjitterbuffer");
@ -1228,110 +1370,6 @@ GST_START_TEST (test_rtx_two_missing)
GST_END_TEST; GST_END_TEST;
GST_START_TEST (text_rtx_two_missing_early)
{
GstHarness *h = gst_harness_new ("rtpjitterbuffer");
GstTestClock *testclock;
gint latency_ms = 30;
GstClockTime last_rtx_request, now;
testclock = gst_harness_get_testclock (h);
gst_harness_set_src_caps (h, generate_caps ());
g_object_set (h->element, "do-retransmission", TRUE, "latency", latency_ms,
NULL);
for (gint i = 0; i <= latency_ms / PCMU_BUF_MS; i++) {
gst_test_clock_set_time (testclock, i * PCMU_BUF_DURATION);
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (i)));
gst_harness_wait_for_clock_id_waits (h, 1, 60);
}
gst_harness_crank_single_clock_wait (h);
fail_unless_equals_int64 (latency_ms * GST_MSECOND,
gst_clock_get_time (GST_CLOCK (testclock)));
for (gint i = 0; i <= latency_ms / PCMU_BUF_MS; i++)
gst_buffer_unref (gst_harness_pull (h));
/* drop reconfigure event */
gst_event_unref (gst_harness_pull_upstream_event (h));
/*
The expected sequence of buffers is this:
___ ___ ___ ___ ___
| 0 | | 1 | | 2 | | 3 | | 4 |
––– ––– ––– ––– –––
0ms 20ms 40ms 60ms 80ms
But instead we get this:
___ ___ _ _ _ _ ___
| 0 | | 1 | | | | | | 4 |
––– ––– – – – – –––
0ms 20ms 41ms
*/
now = 41 * GST_MSECOND;
gst_harness_set_time (h, now);
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h,
generate_test_buffer_full (now, TRUE, 4, 4 * PCMU_RTP_TS_DURATION)));
/*
With the now calculated packet-spacing of (41-20) / 3 = 7,
giving us these expected times:
___ ___ ___ ___ ___
| 0 | | 1 | | 2 | | 3 | | 4 |
––– ––– ––– ––– –––
0ms 20ms 27ms 34ms 41ms
For RTX, the inital RTX-timeouts for the missing buffers are
the expected arrival time + half the packet-spacing time, like this:
___ ___
| 2 | | 3 |
––– –––
50ms 70ms
But since we have re-calculated the estimated arrival-time
of these buffers, we have to adjust the RTX timeout as well,
and we use the original delay (packet-spacing / 2) = 10ms,
and add it on:
___ ___
| 2 | | 3 |
––– –––
37ms 44ms
Also note that the first RTX request is now scheduled for a
time that is prior to NOW (37ms < 41ms), so it will be sent straight
away without us needing to "crank" the timer-thread
*/
/* The RTX request for packet 2 has timestamp 27ms and delay 10ms */
verify_rtx_event (gst_harness_pull_upstream_event (h),
2, 27 * GST_MSECOND, 10, PCMU_BUF_DURATION);
/* and is sent immediately after packet 4 arrives (41ms) */
last_rtx_request = gst_clock_get_time (GST_CLOCK (testclock));
fail_unless_equals_int64 (last_rtx_request, now);
/* crank the timer thread */
gst_harness_crank_single_clock_wait (h);
/* The RTX request for packet 3 has timestamp 34ms and delay 10ms */
verify_rtx_event (gst_harness_pull_upstream_event (h),
3, 34 * GST_MSECOND, 10, PCMU_BUF_DURATION);
/* and is sent at 44ms */
last_rtx_request = gst_clock_get_time (GST_CLOCK (testclock));
fail_unless_equals_int64 (last_rtx_request, 44 * GST_MSECOND);
gst_object_unref (testclock);
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (test_rtx_packet_delay) GST_START_TEST (test_rtx_packet_delay)
{ {
GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstHarness *h = gst_harness_new ("rtpjitterbuffer");
@ -1365,9 +1403,10 @@ GST_START_TEST (test_rtx_packet_delay)
* the estimate for 2 could be refined now to 20ms. also packet 2, 3 and 4 * the estimate for 2 could be refined now to 20ms. also packet 2, 3 and 4
* are exceeding the max allowed reorder distance and should request a * are exceeding the max allowed reorder distance and should request a
* retransmission right away */ * retransmission right away */
gst_harness_set_time (h, 1 * PCMU_BUF_DURATION);
fail_unless_equals_int (GST_FLOW_OK, fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer_full (20 * GST_MSECOND, TRUE, 8, gst_harness_push (h, generate_test_buffer_full (1 * PCMU_BUF_DURATION, TRUE, 8,
8 * PCMU_RTP_TS_DURATION))); 1 * PCMU_RTP_TS_DURATION)));
/* drop reconfigure event */ /* drop reconfigure event */
gst_event_unref (gst_harness_pull_upstream_event (h)); gst_event_unref (gst_harness_pull_upstream_event (h));
@ -1377,23 +1416,23 @@ GST_START_TEST (test_rtx_packet_delay)
/* we should now receive retransmission requests for 2 -> 5 */ /* we should now receive retransmission requests for 2 -> 5 */
out_event = gst_harness_pull_upstream_event (h); out_event = gst_harness_pull_upstream_event (h);
verify_rtx_event (out_event, 2, 20 * GST_MSECOND, 17, PCMU_BUF_DURATION); verify_rtx_event (out_event, 2, 20 * GST_MSECOND, 10, PCMU_BUF_DURATION);
for (i = 3; i < 5; i++) { for (i = 3; i < 5; i++) {
GST_DEBUG ("popping %d", i); GST_DEBUG ("popping %d", i);
out_event = gst_harness_pull_upstream_event (h); out_event = gst_harness_pull_upstream_event (h);
verify_rtx_event (out_event, i, 20 * GST_MSECOND, 17, PCMU_BUF_DURATION); verify_rtx_event (out_event, i, 20 * GST_MSECOND, 10, PCMU_BUF_DURATION);
} }
fail_unless_equals_int (0, gst_harness_upstream_events_in_queue (h)); fail_unless_equals_int (0, gst_harness_upstream_events_in_queue (h));
/* push 9, this should immediately request retransmission of 5 */ /* push 9, this should immediately request retransmission of 5 */
fail_unless_equals_int (GST_FLOW_OK, fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer_full (20 * GST_MSECOND, TRUE, 9, gst_harness_push (h, generate_test_buffer_full (1 * PCMU_BUF_DURATION, TRUE, 9,
9 * PCMU_RTP_TS_DURATION))); 1 * PCMU_RTP_TS_DURATION)));
/* we should now receive retransmission requests for 5 */ /* we should now receive retransmission requests for 5 */
out_event = gst_harness_pull_upstream_event (h); out_event = gst_harness_pull_upstream_event (h);
verify_rtx_event (out_event, 5, 20 * GST_MSECOND, 17, PCMU_BUF_DURATION); verify_rtx_event (out_event, 5, 20 * GST_MSECOND, 10, PCMU_BUF_DURATION);
/* wait for timeout for rtx 6 -> 7 */ /* wait for timeout for rtx 6 -> 7 */
gst_test_clock_set_time_and_process (testclock, 60 * GST_MSECOND); gst_test_clock_set_time_and_process (testclock, 60 * GST_MSECOND);
@ -1401,7 +1440,7 @@ GST_START_TEST (test_rtx_packet_delay)
for (i = 6; i < 8; i++) { for (i = 6; i < 8; i++) {
GST_DEBUG ("popping %d", i); GST_DEBUG ("popping %d", i);
out_event = gst_harness_pull_upstream_event (h); out_event = gst_harness_pull_upstream_event (h);
verify_rtx_event (out_event, i, 20 * GST_MSECOND, 17, PCMU_BUF_DURATION); verify_rtx_event (out_event, i, 20 * GST_MSECOND, 10, PCMU_BUF_DURATION);
} }
/* churn through 7 sync_times until the new buffer gets pushed out */ /* churn through 7 sync_times until the new buffer gets pushed out */
@ -2137,6 +2176,83 @@ GST_START_TEST (test_rtx_same_delay_and_retry_timeout)
GST_END_TEST; GST_END_TEST;
GST_START_TEST (test_rtx_with_backwards_rtptime)
{
GstHarness *h = gst_harness_new ("rtpjitterbuffer");
GstEvent *event;
gint jb_latency_ms = 40;
gint i;
gst_harness_set_src_caps (h, generate_caps ());
g_object_set (h->element,
"do-retransmission", TRUE,
"latency", jb_latency_ms,
NULL);
/* push the first buffer through */
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (0)));
fail_unless (gst_harness_crank_single_clock_wait (h));
gst_buffer_unref (gst_harness_pull (h));
/* push some buffers arriving in perfect time! */
for (i = 1; i <= 2; i++) {
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (i)));
gst_buffer_unref (gst_harness_pull (h));
}
/*
For video using B-frames, an expected sequence
could be like this:
(I = I-frame, P = P-frame, B = B-frame)
___ ___ ___
... | 3 | | 4 | | 5 |
––– ––– –––
rtptime: 3(I) 5(P) 4(B)
arrival(dts): 3 5 5
Notice here that packet 5 (the B frame) make
the rtptime go backwards.
*/
/* seqnum 3, arriving at time 3 with rtptime 3 */
fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer (3)));
gst_buffer_unref (gst_harness_pull (h));
/* seqnum 4, arriving at time 5 with rtptime 5 */
gst_harness_push (h, generate_test_buffer_full (
5 * PCMU_BUF_DURATION, FALSE, 4, 5 * PCMU_RTP_TS_DURATION));
gst_buffer_unref (gst_harness_pull (h));
/* seqnum 5, arriving at time 5 with rtptime 4 */
gst_harness_push (h, generate_test_buffer_full (
5 * PCMU_BUF_DURATION, FALSE, 5, 4 * PCMU_RTP_TS_DURATION));
/* drop reconfigure event */
gst_event_unref (gst_harness_pull_upstream_event (h));
/* crank to time-out the rtx-request for seqnum 6, the point here
being that the backwards rtptime did not mess up the timeout for
the rtx event */
gst_harness_crank_single_clock_wait (h);
event = gst_harness_pull_upstream_event (h);
verify_rtx_event (event, 6, 5 * PCMU_BUF_DURATION + 15 * GST_MSECOND,
17, 35 * GST_MSECOND);
fail_unless (verify_jb_stats (h->element,
gst_structure_new ("application/x-rtp-jitterbuffer-stats",
"num-pushed", G_TYPE_UINT64, (guint64) 6,
"rtx-count", G_TYPE_UINT64, (guint64) 1,
"num-lost", G_TYPE_UINT64, (guint64) 0, NULL)));
gst_harness_teardown (h);
}
GST_END_TEST;
GST_START_TEST (test_gap_exceeds_latency) GST_START_TEST (test_gap_exceeds_latency)
{ {
GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstHarness *h = gst_harness_new ("rtpjitterbuffer");
@ -2325,13 +2441,12 @@ GST_START_TEST (test_deadline_ts_offset)
GST_END_TEST; GST_END_TEST;
GST_START_TEST (test_dts_gap_larger_than_latency) GST_START_TEST (test_gap_larger_than_latency)
{ {
GstHarness *h = gst_harness_new ("rtpjitterbuffer"); GstHarness *h = gst_harness_new ("rtpjitterbuffer");
GstTestClock *testclock; GstTestClock *testclock;
GstEvent *out_event; GstEvent *out_event;
gint jb_latency_ms = 100; gint jb_latency_ms = 100;
GstClockTime dts_after_gap = (jb_latency_ms + 1) * GST_MSECOND;
gst_harness_set_src_caps (h, generate_caps ()); gst_harness_set_src_caps (h, generate_caps ());
testclock = gst_harness_get_testclock (h); testclock = gst_harness_get_testclock (h);
@ -2343,26 +2458,34 @@ GST_START_TEST (test_dts_gap_larger_than_latency)
fail_unless (gst_harness_crank_single_clock_wait (h)); fail_unless (gst_harness_crank_single_clock_wait (h));
gst_buffer_unref (gst_harness_pull (h)); gst_buffer_unref (gst_harness_pull (h));
/* Push packet with DTS larger than latency */ /* we have 100ms latency, so drop 6 packets (120ms) */
gst_harness_set_time (h, 7 * PCMU_BUF_DURATION);
fail_unless_equals_int (GST_FLOW_OK, fail_unless_equals_int (GST_FLOW_OK,
gst_harness_push (h, generate_test_buffer_full (dts_after_gap, gst_harness_push (h, generate_test_buffer (7)));
TRUE, 5, 5 * PCMU_RTP_TS_DURATION)));
/* drop GstEventStreamStart & GstEventCaps & GstEventSegment */ /* drop GstEventStreamStart & GstEventCaps & GstEventSegment */
for (int i = 0; i < 3; i++) for (int i = 0; i < 3; i++)
gst_event_unref (gst_harness_pull_event (h)); gst_event_unref (gst_harness_pull_event (h));
/* Time out and verify lost events */ /* Packet 1 and 2 are already lost at this point */
for (gint i = 1; i < 5; i++) { for (gint i = 1; i <= 2; i++) {
GstClockTime dur = dts_after_gap / 5; out_event = gst_harness_pull_event (h);
verify_lost_event (out_event, i, i * PCMU_BUF_DURATION, PCMU_BUF_DURATION);
}
/* Packets 3-6 have to be timed out */
for (gint i = 3; i <= 6; i++) {
fail_unless (gst_harness_crank_single_clock_wait (h)); fail_unless (gst_harness_crank_single_clock_wait (h));
out_event = gst_harness_pull_event (h); out_event = gst_harness_pull_event (h);
verify_lost_event (out_event, i, i * dur, dur); verify_lost_event (out_event, i, i * PCMU_BUF_DURATION, PCMU_BUF_DURATION);
} }
/* and finally pull out buffer 7 */
gst_buffer_unref (gst_harness_pull (h));
fail_unless (verify_jb_stats (h->element, fail_unless (verify_jb_stats (h->element,
gst_structure_new ("application/x-rtp-jitterbuffer-stats", gst_structure_new ("application/x-rtp-jitterbuffer-stats",
"num-lost", G_TYPE_UINT64, (guint64) 4, NULL))); "num-lost", G_TYPE_UINT64, (guint64) 6, NULL)));
gst_object_unref (testclock); gst_object_unref (testclock);
gst_harness_teardown (h); gst_harness_teardown (h);
@ -2555,6 +2678,9 @@ rtpjitterbuffer_suite (void)
tcase_add_test (tc_chain, test_only_one_lost_event_on_large_gaps); tcase_add_test (tc_chain, test_only_one_lost_event_on_large_gaps);
tcase_add_test (tc_chain, test_two_lost_one_arrives_in_time); tcase_add_test (tc_chain, test_two_lost_one_arrives_in_time);
tcase_add_test (tc_chain, test_late_packets_still_makes_lost_events); tcase_add_test (tc_chain, test_late_packets_still_makes_lost_events);
tcase_add_test (tc_chain, test_lost_event_uses_pts);
tcase_add_test (tc_chain, test_lost_event_with_backwards_rtptime);
tcase_add_test (tc_chain, test_all_packets_are_timestamped_zero); tcase_add_test (tc_chain, test_all_packets_are_timestamped_zero);
tcase_add_loop_test (tc_chain, test_num_late_when_considered_lost_arrives, 0, tcase_add_loop_test (tc_chain, test_num_late_when_considered_lost_arrives, 0,
2); 2);
@ -2564,7 +2690,6 @@ rtpjitterbuffer_suite (void)
tcase_add_test (tc_chain, test_rtx_expected_next); tcase_add_test (tc_chain, test_rtx_expected_next);
tcase_add_test (tc_chain, test_rtx_two_missing); tcase_add_test (tc_chain, test_rtx_two_missing);
tcase_add_test (tc_chain, text_rtx_two_missing_early);
tcase_add_test (tc_chain, test_rtx_packet_delay); tcase_add_test (tc_chain, test_rtx_packet_delay);
tcase_add_test (tc_chain, test_rtx_buffer_arrives_just_in_time); tcase_add_test (tc_chain, test_rtx_buffer_arrives_just_in_time);
tcase_add_test (tc_chain, test_rtx_buffer_arrives_too_late); tcase_add_test (tc_chain, test_rtx_buffer_arrives_too_late);
@ -2575,10 +2700,11 @@ rtpjitterbuffer_suite (void)
tcase_add_test (tc_chain, test_rtx_rtt_larger_than_retry_timeout); tcase_add_test (tc_chain, test_rtx_rtt_larger_than_retry_timeout);
tcase_add_test (tc_chain, test_rtx_no_request_if_time_past_retry_period); tcase_add_test (tc_chain, test_rtx_no_request_if_time_past_retry_period);
tcase_add_test (tc_chain, test_rtx_same_delay_and_retry_timeout); tcase_add_test (tc_chain, test_rtx_same_delay_and_retry_timeout);
tcase_add_test (tc_chain, test_rtx_with_backwards_rtptime);
tcase_add_test (tc_chain, test_gap_exceeds_latency); tcase_add_test (tc_chain, test_gap_exceeds_latency);
tcase_add_test (tc_chain, test_deadline_ts_offset); tcase_add_test (tc_chain, test_deadline_ts_offset);
tcase_add_test (tc_chain, test_dts_gap_larger_than_latency); tcase_add_test (tc_chain, test_gap_larger_than_latency);
tcase_add_test (tc_chain, test_push_big_gap); tcase_add_test (tc_chain, test_push_big_gap);
tcase_add_loop_test (tc_chain, tcase_add_loop_test (tc_chain,