From c5b75394a97b1de895014ac9a1a4b11d7987fada Mon Sep 17 00:00:00 2001 From: Carlos Rafael Giani Date: Mon, 9 Dec 2013 17:08:15 +0100 Subject: [PATCH] audiobasesink: added custom clock slaving method This new clock slaving method allows for installing a callback that is invoked during playback. Inside this callback, a custom slaving mechanism can be used (for example, a control loop adjusting a PLL or an asynchronous resampler). Upon request, it can skew the playout pointer just like the "skew" method. This is useful if the clocks drifted apart too much, and a quick reset is necessary. Signed-off-by: Carlos Rafael Giani https://bugzilla.gnome.org/show_bug.cgi?id=708362 --- gst-libs/gst/audio/gstaudiobasesink.c | 196 ++++++++++++++++++++++++++ gst-libs/gst/audio/gstaudiobasesink.h | 71 +++++++++- 2 files changed, 266 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/audio/gstaudiobasesink.c b/gst-libs/gst/audio/gstaudiobasesink.c index 32013fe6d5..28988f40a7 100644 --- a/gst-libs/gst/audio/gstaudiobasesink.c +++ b/gst-libs/gst/audio/gstaudiobasesink.c @@ -69,6 +69,11 @@ struct _GstAudioBaseSinkPrivate /* number of nanoseconds to wait until creating a discontinuity */ GstClockTime discont_wait; + + /* custom slaving algorithm callback */ + GstAudioBaseSinkCustomSlavingCallback custom_slaving_callback; + gpointer custom_slaving_cb_data; + GDestroyNotify custom_slaving_cb_notify; }; /* BaseAudioSink signals and args */ @@ -124,6 +129,8 @@ gst_audio_base_sink_slave_method_get_type (void) "resample"}, {GST_AUDIO_BASE_SINK_SLAVE_SKEW, "GST_AUDIO_BASE_SINK_SLAVE_SKEW", "skew"}, {GST_AUDIO_BASE_SINK_SLAVE_NONE, "GST_AUDIO_BASE_SINK_SLAVE_NONE", "none"}, + {GST_AUDIO_BASE_SINK_SLAVE_CUSTOM, "GST_AUDIO_BASE_SINK_SLAVE_CUSTOM", + "custom"}, {0, NULL, NULL}, }; @@ -304,6 +311,9 @@ gst_audio_base_sink_init (GstAudioBaseSink * audiobasesink) audiobasesink->priv->drift_tolerance = DEFAULT_DRIFT_TOLERANCE; audiobasesink->priv->alignment_threshold = DEFAULT_ALIGNMENT_THRESHOLD; audiobasesink->priv->discont_wait = DEFAULT_DISCONT_WAIT; + audiobasesink->priv->custom_slaving_callback = NULL; + audiobasesink->priv->custom_slaving_cb_data = NULL; + audiobasesink->priv->custom_slaving_cb_notify = NULL; audiobasesink->provided_clock = gst_audio_clock_new ("GstAudioSinkClock", (GstAudioClockGetTimeFunc) gst_audio_base_sink_get_time, audiobasesink, @@ -327,6 +337,9 @@ gst_audio_base_sink_dispose (GObject * object) sink = GST_AUDIO_BASE_SINK (object); + if (sink->priv->custom_slaving_cb_notify) + sink->priv->custom_slaving_cb_notify (sink->priv->custom_slaving_cb_data); + if (sink->provided_clock) { gst_audio_clock_invalidate (sink->provided_clock); gst_object_unref (sink->provided_clock); @@ -743,6 +756,73 @@ gst_audio_base_sink_set_discont_wait (GstAudioBaseSink * sink, GST_OBJECT_UNLOCK (sink); } +/** + * gst_audio_base_sink_set_custom_slaving_callback: + * @sink: a #GstAudioBaseSink + * @callback: a #GstAudioBaseSinkCustomSlavingCallback + * @user_data: user data passed to the callback + * @notify : called when user_data becomes unused + * + * Sets the custom slaving callback. This callback will + * be invoked if the slave-method property is set to + * GST_AUDIO_BASE_SINK_SLAVE_CUSTOM and the audio sink + * receives and plays samples. + * + * Setting the callback to NULL causes the sink to + * behave as if the GST_AUDIO_BASE_SINK_SLAVE_NONE + * method were used. + * + * Since: 1.6 + */ +void +gst_audio_base_sink_set_custom_slaving_callback (GstAudioBaseSink * sink, + GstAudioBaseSinkCustomSlavingCallback callback, + gpointer user_data, GDestroyNotify notify) +{ + g_return_if_fail (GST_IS_AUDIO_BASE_SINK (sink)); + + GST_OBJECT_LOCK (sink); + sink->priv->custom_slaving_callback = callback; + sink->priv->custom_slaving_cb_data = user_data; + sink->priv->custom_slaving_cb_notify = notify; + GST_OBJECT_UNLOCK (sink); +} + +static void +gst_audio_base_sink_custom_cb_report_discont (GstAudioBaseSink * sink, + GstAudioBaseSinkDiscontReason discont_reason) +{ + if ((sink->priv->custom_slaving_callback != NULL) && + (sink->priv->slave_method == GST_AUDIO_BASE_SINK_SLAVE_CUSTOM)) { + sink->priv->custom_slaving_callback (sink, GST_CLOCK_TIME_NONE, + GST_CLOCK_TIME_NONE, NULL, discont_reason, + sink->priv->custom_slaving_cb_data); + } +} + +/** + * gst_audio_base_sink_report_device_failure: + * @sink: a #GstAudioBaseSink + * + * Informs this base class that the audio output device has failed for + * some reason, causing a discontinuity (for example, because the device + * recovered from the error, but lost all contents of its ring buffer). + * This function is typically called by derived classes, and is useful + * for the custom slave method. + * + * Since: 1.6 + */ +void +gst_audio_base_sink_report_device_failure (GstAudioBaseSink * sink) +{ + g_return_if_fail (GST_IS_AUDIO_BASE_SINK (sink)); + + GST_OBJECT_LOCK (sink); + gst_audio_base_sink_custom_cb_report_discont (sink, + GST_AUDIO_BASE_SINK_DISCONT_REASON_DEVICE_FAILURE); + GST_OBJECT_UNLOCK (sink); +} + /** * gst_audio_base_sink_get_discont_wait: * @sink: a #GstAudioBaseSink @@ -903,6 +983,9 @@ gst_audio_base_sink_setcaps (GstBaseSink * bsink, GstCaps * caps) /* We need to resync since the ringbuffer restarted */ gst_audio_base_sink_reset_sync (sink); + gst_audio_base_sink_custom_cb_report_discont (sink, + GST_AUDIO_BASE_SINK_DISCONT_REASON_NEW_CAPS); + if (bsink->pad_mode == GST_PAD_MODE_PUSH) { GST_DEBUG_OBJECT (sink, "activate ringbuffer"); gst_audio_ring_buffer_activate (sink->ringbuffer, TRUE); @@ -1096,6 +1179,10 @@ gst_audio_base_sink_event (GstBaseSink * bsink, GstEvent * event) case GST_EVENT_FLUSH_STOP: /* always resync on sample after a flush */ gst_audio_base_sink_reset_sync (sink); + + gst_audio_base_sink_custom_cb_report_discont (sink, + GST_AUDIO_BASE_SINK_DISCONT_REASON_FLUSH); + if (sink->ringbuffer) gst_audio_ring_buffer_set_flushing (sink->ringbuffer, FALSE); break; @@ -1179,6 +1266,104 @@ clock_convert_external (GstClockTime external, GstClockTime cinternal, return external; } + +/* apply the clock offset and invoke a custom callback + * which might also request changes to the playout pointer + * + * this reuses code from the skewing algorithm, but leaves + * decision on whether or not to skew (and how much to skew) + * up to the callback */ +static void +gst_audio_base_sink_custom_slaving (GstAudioBaseSink * sink, + GstClockTime render_start, GstClockTime render_stop, + GstClockTime * srender_start, GstClockTime * srender_stop) +{ + GstClockTime cinternal, cexternal, crate_num, crate_denom; + GstClockTime etime, itime; + GstClockTimeDiff requested_skew; + gint driftsamples; + gint64 last_align; + + /* get calibration parameters to compensate for offsets */ + gst_clock_get_calibration (sink->provided_clock, &cinternal, &cexternal, + &crate_num, &crate_denom); + + /* sample clocks and figure out clock skew */ + etime = gst_clock_get_time (GST_ELEMENT_CLOCK (sink)); + itime = gst_audio_clock_get_time (sink->provided_clock); + itime = gst_audio_clock_adjust (sink->provided_clock, itime); + + GST_DEBUG_OBJECT (sink, + "internal %" GST_TIME_FORMAT " external %" GST_TIME_FORMAT + " cinternal %" GST_TIME_FORMAT " cexternal %" GST_TIME_FORMAT, + GST_TIME_ARGS (itime), GST_TIME_ARGS (etime), + GST_TIME_ARGS (cinternal), GST_TIME_ARGS (cexternal)); + + /* make sure we never go below 0 */ + etime = etime > cexternal ? etime - cexternal : 0; + itime = itime > cinternal ? itime - cinternal : 0; + + /* don't do any skewing unless the callback explicitely requests one */ + requested_skew = 0; + + if (sink->priv->custom_slaving_callback != NULL) { + sink->priv->custom_slaving_callback (sink, etime, itime, &requested_skew, + FALSE, sink->priv->custom_slaving_cb_data); + GST_DEBUG_OBJECT (sink, "custom slaving requested skew %" G_GINT64_FORMAT, + requested_skew); + } else { + GST_DEBUG_OBJECT (sink, + "no custom slaving callback set - clock drift will not be compensated"); + } + + if (requested_skew > 0) { + cexternal = (cexternal > requested_skew) ? (cexternal - requested_skew) : 0; + + driftsamples = + (sink->ringbuffer->spec.info.rate * requested_skew) / GST_SECOND; + last_align = sink->priv->last_align; + + /* if we were aligning in the wrong direction or we aligned more than what we + * will correct, resync */ + if ((last_align < 0) || (last_align > driftsamples)) + sink->next_sample = -1; + + GST_DEBUG_OBJECT (sink, + "last_align %" G_GINT64_FORMAT " driftsamples %u, next %" + G_GUINT64_FORMAT, last_align, driftsamples, sink->next_sample); + + gst_clock_set_calibration (sink->provided_clock, cinternal, cexternal, + crate_num, crate_denom); + } else if (requested_skew < 0) { + cexternal += ABS (requested_skew); + + driftsamples = + (sink->ringbuffer->spec.info.rate * ABS (requested_skew)) / GST_SECOND; + last_align = sink->priv->last_align; + + /* if we were aligning in the wrong direction or we aligned more than what we + * will correct, resync */ + if ((last_align > 0) || (-last_align > driftsamples)) + sink->next_sample = -1; + + GST_DEBUG_OBJECT (sink, + "last_align %" G_GINT64_FORMAT " driftsamples %u, next %" + G_GUINT64_FORMAT, last_align, driftsamples, sink->next_sample); + + gst_clock_set_calibration (sink->provided_clock, cinternal, cexternal, + crate_num, crate_denom); + } + + /* convert, ignoring speed */ + render_start = clock_convert_external (render_start, cinternal, cexternal, + crate_num, crate_denom); + render_stop = clock_convert_external (render_stop, cinternal, cexternal, + crate_num, crate_denom); + + *srender_start = render_start; + *srender_stop = render_stop; +} + /* algorithm to calculate sample positions that will result in resampling to * match the clock rate of the master */ static void @@ -1395,6 +1580,10 @@ gst_audio_base_sink_handle_slaving (GstAudioBaseSink * sink, gst_audio_base_sink_none_slaving (sink, render_start, render_stop, srender_start, srender_stop); break; + case GST_AUDIO_BASE_SINK_SLAVE_CUSTOM: + gst_audio_base_sink_custom_slaving (sink, render_start, render_stop, + srender_start, srender_stop); + break; default: g_warning ("unknown slaving method %d", sink->priv->slave_method); break; @@ -1511,12 +1700,16 @@ gst_audio_base_sink_sync_latency (GstBaseSink * bsink, GstMiniObject * obj) break; case GST_AUDIO_BASE_SINK_SLAVE_SKEW: case GST_AUDIO_BASE_SINK_SLAVE_NONE: + case GST_AUDIO_BASE_SINK_SLAVE_CUSTOM: default: break; } gst_audio_base_sink_reset_sync (sink); + gst_audio_base_sink_custom_cb_report_discont (sink, + GST_AUDIO_BASE_SINK_DISCONT_REASON_SYNC_LATENCY); + return GST_FLOW_OK; /* ERRORS */ @@ -1607,6 +1800,9 @@ gst_audio_base_sink_get_alignment (GstAudioBaseSink * sink, "%s%" GST_TIME_FORMAT ", resyncing", sample_offset > sink->next_sample ? "+" : "-", GST_TIME_ARGS (diff_s)); align = 0; + + gst_audio_base_sink_custom_cb_report_discont (sink, + GST_AUDIO_BASE_SINK_DISCONT_REASON_ALIGNMENT); } return align; diff --git a/gst-libs/gst/audio/gstaudiobasesink.h b/gst-libs/gst/audio/gstaudiobasesink.h index 49ee19d3f3..d17fc97c14 100644 --- a/gst-libs/gst/audio/gstaudiobasesink.h +++ b/gst-libs/gst/audio/gstaudiobasesink.h @@ -86,6 +86,7 @@ G_BEGIN_DECLS * @GST_AUDIO_BASE_SINK_SLAVE_SKEW: Adjust playout pointer when master clock * drifts too much. * @GST_AUDIO_BASE_SINK_SLAVE_NONE: No adjustment is done. + * @GST_AUDIO_BASE_SINK_SLAVE_CUSTOM: Use custom clock slaving algorithm (Since: 1.6) * * Different possible clock slaving algorithms used when the internal audio * clock is not selected as the pipeline master clock. @@ -94,7 +95,8 @@ typedef enum { GST_AUDIO_BASE_SINK_SLAVE_RESAMPLE, GST_AUDIO_BASE_SINK_SLAVE_SKEW, - GST_AUDIO_BASE_SINK_SLAVE_NONE + GST_AUDIO_BASE_SINK_SLAVE_NONE, + GST_AUDIO_BASE_SINK_SLAVE_CUSTOM } GstAudioBaseSinkSlaveMethod; #define GST_TYPE_AUDIO_BASE_SINK_SLAVE_METHOD (gst_audio_base_sink_slave_method_get_type ()) @@ -103,6 +105,65 @@ typedef struct _GstAudioBaseSink GstAudioBaseSink; typedef struct _GstAudioBaseSinkClass GstAudioBaseSinkClass; typedef struct _GstAudioBaseSinkPrivate GstAudioBaseSinkPrivate; +/** + * GstAudioBaseSinkDiscontReason: + * GST_AUDIO_BASE_SINK_DISCONT_REASON_NO_DISCONT: No discontinuity occurred + * GST_AUDIO_BASE_SINK_DISCONT_REASON_NEW_CAPS: New caps are set, causing renegotiotion + * GST_AUDIO_BASE_SINK_DISCONT_REASON_FLUSH: Samples have been flushed + * GST_AUDIO_BASE_SINK_DISCONT_REASON_SYNC_LATENCY: Sink was synchronized to the estimated latency (occurs during initialization) + * GST_AUDIO_BASE_SINK_DISCONT_REASON_ALIGNMENT: Aligning buffers failed because the timestamps are too discontinuous + * GST_AUDIO_BASE_SINK_DISCONT_REASON_DEVICE_FAILURE: Audio output device experienced and recovered from an error but introduced latency in the process (see also @gst_audio_base_sink_report_device_failure) + * + * Different possible reasons for discontinuities. This enum is useful for the custom + * slave method. + * + * Since: 1.6 + */ +typedef enum +{ + GST_AUDIO_BASE_SINK_DISCONT_REASON_NO_DISCONT, + GST_AUDIO_BASE_SINK_DISCONT_REASON_NEW_CAPS, + GST_AUDIO_BASE_SINK_DISCONT_REASON_FLUSH, + GST_AUDIO_BASE_SINK_DISCONT_REASON_SYNC_LATENCY, + GST_AUDIO_BASE_SINK_DISCONT_REASON_ALIGNMENT, + GST_AUDIO_BASE_SINK_DISCONT_REASON_DEVICE_FAILURE +} GstAudioBaseSinkDiscontReason; + +/** + * GstAudioBaseSinkCustomSlavingCallback: + * @sink: a #GstAudioBaseSink + * @etime: external clock time + * @itime: internal clock time + * @requested_skew: skew amount requested by the callback + * @discont: TRUE if there was a discontinuity in the average skew + * @user_data: user data + * + * This function is set with gst_audio_base_sink_set_custom_slaving_callback() + * and is called during playback. It receives the current time of external and + * internal clocks, which the callback can then use to apply any custom + * slaving/synchronization schemes. The external clock is the sink's element clock, + * the internal one is the internal audio clock. The internal audio clock's + * calibration is applied to the timestamps before they are passed to the + * callback. The difference between etime and itime is the skew; how much + * internal and external clock lie apart from each other. A skew of 0 means + * both clocks are perfectly in sync. itime > etime means the external clock + * is going slower, while itime < etime means it is going faster than the + * internal clock. etime and itime are always valid timestamps, except for when + * discont is set to TRUE. + * requested_skew is an output value the callback can write to. It informs the sink + * of whether or not it should move the playout pointer, and if so, by how much. + * This pointer is only NULL if discont is true; otherwise, it is safe to write + * to *requested_skew. The default skew is 0. + * + * The sink may experience discontinuities. If one happens, discont is TRUE, + * itime, etime are set to GST_CLOCK_TIME_NONE, and requested_skew is NULL. + * This makes it possible to reset custom clock slaving algorithms when a + * discontinuity happens. + * + * Since: 1.6 + */ +typedef void (*GstAudioBaseSinkCustomSlavingCallback) (GstAudioBaseSink *sink, GstClockTime etime, GstClockTime itime, GstClockTimeDiff *requested_skew, GstAudioBaseSinkDiscontReason discont_reason, gpointer user_data); + /** * GstAudioBaseSink: * @@ -187,6 +248,14 @@ void gst_audio_base_sink_set_discont_wait (GstAudioBaseSink * sink, GstClockTime gst_audio_base_sink_get_discont_wait (GstAudioBaseSink * sink); +void +gst_audio_base_sink_set_custom_slaving_callback (GstAudioBaseSink * sink, + GstAudioBaseSinkCustomSlavingCallback callback, + gpointer user_data, + GDestroyNotify notify); + +void gst_audio_base_sink_report_device_failure (GstAudioBaseSink * sink); + G_END_DECLS #endif /* __GST_AUDIO_BASE_SINK_H__ */