From 9d5179ad3f6966873b4b65edeb94fd0c200b808d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 26 Apr 2022 22:17:51 +0300 Subject: [PATCH] rtpjitterbuffer: add the reference timestamp meta in more situations Previously, we only added it when actually performing synchronization based on the NTP time. The information can be useful downstream in other situations too, and we can compute a NTP time as soon as we get a sender report with the relevant information. Co-authored-by: Mathieu Duponchelle Part-of: --- .../docs/gst_plugins_cache.json | 4 +- .../gst/rtpmanager/gstrtpbin.c | 6 +- .../gst/rtpmanager/gstrtpjitterbuffer.c | 72 ++++++++++++++++--- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index 83f6ee13a6..6ea3cbec48 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -17047,7 +17047,7 @@ }, "properties": { "add-reference-timestamp-meta": { - "blurb": "Add Reference Timestamp Meta to buffers with the original clock timestamp before any adjustments when syncing to an RFC7273 clock.", + "blurb": "Add Reference Timestamp Meta to buffers with the original clock timestamp before any adjustments when syncing to an RFC7273 clock or after clock synchronization via RTCP or inband NTP-64 header extensions has happened.", "conditionally-available": false, "construct": false, "construct-only": false, @@ -18115,7 +18115,7 @@ }, "properties": { "add-reference-timestamp-meta": { - "blurb": "Add Reference Timestamp Meta to buffers with the original clock timestamp before any adjustments when syncing to an RFC7273 clock.", + "blurb": "Add Reference Timestamp Meta to buffers with the original clock timestamp before any adjustments when syncing to an RFC7273 clock or after clock synchronization via RTCP or inband NTP-64 header extensions has happened.", "conditionally-available": false, "construct": false, "construct-only": false, diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c index c7ab4395a4..e39c10d20c 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpbin.c @@ -2836,7 +2836,8 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) /** * GstRtpBin:add-reference-timestamp-meta: * - * When syncing to a RFC7273 clock, add #GstReferenceTimestampMeta + * When syncing to a RFC7273 clock or after clock synchronization via RTCP or + * inband NTP-64 header extensions has happened, add #GstReferenceTimestampMeta * to buffers with the original reconstructed reference clock timestamp. * * Since: 1.22 @@ -2846,7 +2847,8 @@ gst_rtp_bin_class_init (GstRtpBinClass * klass) g_param_spec_boolean ("add-reference-timestamp-meta", "Add Reference Timestamp Meta", "Add Reference Timestamp Meta to buffers with the original clock timestamp " - "before any adjustments when syncing to an RFC7273 clock.", + "before any adjustments when syncing to an RFC7273 clock or after clock " + "synchronization via RTCP or inband NTP-64 header extensions has happened.", DEFAULT_ADD_REFERENCE_TIMESTAMP_META, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); diff --git a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c index 3ba70c8d3d..cb27dea0b4 100644 --- a/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c +++ b/subprojects/gst-plugins-good/gst/rtpmanager/gstrtpjitterbuffer.c @@ -344,7 +344,7 @@ struct _GstRtpJitterBufferPrivate guint sync_interval; /* Reference for GstReferenceTimestampMeta */ - GstCaps *rfc7273_ref; + GstCaps *reference_timestamp_caps; /* RTP header extension ID for RFC6051 64-bit NTP timestamps */ guint8 ntp64_ext_id; @@ -408,11 +408,14 @@ struct _GstRtpJitterBufferPrivate /* the latency of the upstream peer, we have to take this into account when * synchronizing the buffers. */ GstClockTime peer_latency; - guint64 ext_rtptime; + guint64 last_sr_ext_rtptime; GstBuffer *last_sr; guint32 last_sr_ssrc; GstClockTime last_sr_ntpnstime; + GstClockTime last_known_ntpnstime; + guint64 last_known_ext_rtptime; + /* some accounting */ guint64 num_pushed; guint64 num_lost; @@ -952,7 +955,8 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) /** * GstRtpJitterBuffer:add-reference-timestamp-meta: * - * When syncing to a RFC7273 clock, add #GstReferenceTimestampMeta + * When syncing to a RFC7273 clock or after clock synchronization via RTCP or + * inband NTP-64 header extensions has happened, add #GstReferenceTimestampMeta * to buffers with the original reconstructed reference clock timestamp. * * Since: 1.22 @@ -962,7 +966,8 @@ gst_rtp_jitter_buffer_class_init (GstRtpJitterBufferClass * klass) g_param_spec_boolean ("add-reference-timestamp-meta", "Add Reference Timestamp Meta", "Add Reference Timestamp Meta to buffers with the original clock timestamp " - "before any adjustments when syncing to an RFC7273 clock.", + "before any adjustments when syncing to an RFC7273 clock or after clock " + "synchronization via RTCP or inband NTP-64 header extensions has happened.", DEFAULT_ADD_REFERENCE_TIMESTAMP_META, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); @@ -1134,6 +1139,8 @@ gst_rtp_jitter_buffer_init (GstRtpJitterBuffer * jitterbuffer) priv->last_pts = -1; priv->last_rtptime = -1; priv->last_ntpnstime = -1; + priv->last_known_ext_rtptime = -1; + priv->last_known_ntpnstime = -1; priv->avg_jitter = 0; priv->last_drop_msg_timestamp = GST_CLOCK_TIME_NONE; priv->num_too_late = 0; @@ -1644,6 +1651,8 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, ts_meta_ref = gst_caps_new_simple ("timestamp/x-ptp", "version", G_TYPE_STRING, "IEEE1588-2008", "domain", G_TYPE_INT, domain, NULL); + } else if (!g_strcmp0 (ts_refclk, "local")) { + ts_meta_ref = gst_caps_new_empty_simple ("timestamp/x-ntp"); } else { GST_FIXME_OBJECT (jitterbuffer, "Unsupported timestamp reference clock"); } @@ -1664,9 +1673,10 @@ gst_jitter_buffer_sink_parse_caps (GstRtpJitterBuffer * jitterbuffer, rtp_jitter_buffer_set_media_clock (priv->jbuf, clock, clock_offset); } else { rtp_jitter_buffer_set_media_clock (priv->jbuf, NULL, -1); + ts_meta_ref = gst_caps_new_empty_simple ("timestamp/x-ntp"); } - gst_caps_take (&priv->rfc7273_ref, ts_meta_ref); + gst_caps_take (&priv->reference_timestamp_caps, ts_meta_ref); _get_cname_ssrc_mappings (jitterbuffer, caps_struct); priv->ntp64_ext_id = @@ -1738,6 +1748,8 @@ gst_rtp_jitter_buffer_flush_stop (GstRtpJitterBuffer * jitterbuffer) priv->last_dts = -1; priv->last_rtptime = -1; priv->last_ntpnstime = -1; + priv->last_known_ext_rtptime = -1; + priv->last_known_ntpnstime = -1; priv->last_in_pts = 0; priv->equidistant = 0; priv->segment_seqnum = GST_SEQNUM_INVALID; @@ -1869,7 +1881,7 @@ gst_rtp_jitter_buffer_change_state (GstElement * element, JBUF_UNLOCK (priv); g_thread_join (priv->timer_thread); priv->timer_thread = NULL; - gst_clear_caps (&priv->rfc7273_ref); + gst_clear_caps (&priv->reference_timestamp_caps); break; case GST_STATE_CHANGE_READY_TO_NULL: break; @@ -3172,6 +3184,9 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, /* reset clock-rate so that we get a new one */ priv->clock_rate = -1; + priv->last_known_ext_rtptime = -1; + priv->last_known_ntpnstime = -1; + /* Try to get the clock-rate from the caps first if we can. If there are no * caps we must fire the signal to get the clock-rate. */ if ((caps = gst_pad_get_current_caps (pad))) { @@ -3190,12 +3205,16 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, goto no_clock_rate; gst_rtp_packet_rate_ctx_reset (&priv->packet_rate_ctx, priv->clock_rate); + priv->last_known_ext_rtptime = -1; + priv->last_known_ntpnstime = -1; } if (G_UNLIKELY (priv->last_ssrc != ssrc)) { GST_DEBUG_OBJECT (jitterbuffer, "SSRC changed from %u to %u", priv->last_ssrc, ssrc); priv->last_ssrc = ssrc; + priv->last_known_ext_rtptime = -1; + priv->last_known_ntpnstime = -1; } /* don't accept more data on EOS */ @@ -3362,6 +3381,16 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, priv->next_in_seqnum = (seqnum + 1) & 0xffff; } + if (inband_ntp_time != GST_CLOCK_TIME_NONE) { + guint64 ext_rtptime; + + ext_rtptime = priv->jbuf->ext_rtptime; + ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); + + priv->last_known_ext_rtptime = ext_rtptime; + priv->last_known_ntpnstime = inband_ntp_time; + } + if (is_rtx) timer->num_rtx_received++; @@ -3438,13 +3467,31 @@ gst_rtp_jitter_buffer_chain (GstPad * pad, GstObject * parent, JBUF_SIGNAL_EVENT (priv); } } - // ntp_time will only be set in case of RFC7273 sync + // If we can calculate a NTP time based solely on the Sender Report, or + // inband NTP header extension do that so that we can still add a reference + // timestamp meta to the buffer + if (!GST_CLOCK_TIME_IS_VALID (ntp_time) && + GST_CLOCK_TIME_IS_VALID (priv->last_known_ntpnstime) && + priv->last_known_ext_rtptime != -1) { + guint64 ext_time = priv->last_known_ext_rtptime; + + ext_time = gst_rtp_buffer_ext_timestamp (&ext_time, rtptime); + + ntp_time = + priv->last_known_ntpnstime + gst_util_uint64_scale (ext_time - + priv->last_known_ext_rtptime, GST_SECOND, priv->clock_rate); + } + if (priv->add_reference_timestamp_meta && GST_CLOCK_TIME_IS_VALID (ntp_time) - && priv->rfc7273_ref != NULL) { + && priv->reference_timestamp_caps != NULL) { buffer = gst_buffer_make_writable (buffer); + GST_TRACE_OBJECT (jitterbuffer, + "adding NTP time reference meta: %" GST_TIME_FORMAT, + GST_TIME_ARGS (ntp_time)); + gst_buffer_add_reference_timestamp_meta (buffer, - priv->rfc7273_ref, ntp_time, GST_CLOCK_TIME_NONE); + priv->reference_timestamp_caps, ntp_time, GST_CLOCK_TIME_NONE); } /* If we estimated the DTS, don't consider it in the clock skew calculations @@ -4546,7 +4593,7 @@ do_handle_sync (GstRtpJitterBuffer * jitterbuffer) &clock_rate, &last_rtptime); clock_base = priv->clock_base; - ext_rtptime = priv->ext_rtptime; + ext_rtptime = priv->last_sr_ext_rtptime; GST_DEBUG_OBJECT (jitterbuffer, "ext SR %" G_GUINT64_FORMAT ", NTP %" G_GUINT64_FORMAT ", base %" @@ -4728,10 +4775,13 @@ gst_rtp_jitter_buffer_chain_rtcp (GstPad * pad, GstObject * parent, ext_rtptime = priv->jbuf->ext_rtptime; ext_rtptime = gst_rtp_buffer_ext_timestamp (&ext_rtptime, rtptime); - priv->ext_rtptime = ext_rtptime; + priv->last_sr_ext_rtptime = ext_rtptime; priv->last_sr_ssrc = ssrc; priv->last_sr_ntpnstime = ntpnstime; + priv->last_known_ext_rtptime = ext_rtptime; + priv->last_known_ntpnstime = ntpnstime; + if (priv->last_ntpnstime != GST_CLOCK_TIME_NONE && ntpnstime - priv->last_ntpnstime < priv->sync_interval * GST_MSECOND) { gst_buffer_replace (&priv->last_sr, NULL);