From 396ef0cbcf01d2cb28d8a732897e84dfbfc49db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 5 Sep 2024 22:07:24 +0300 Subject: [PATCH] video: Don't overshoot QoS earliest time by a factor of 2 By setting the earliest time to timestamp + 2 * diff there would be a difference of 1 * diff between the current clock time and the earliest time the element would let through in the future. If e.g. a frame is arriving 30s late at the sink, then not just all frames up to that point would be dropped but also 30s of frames after the current clock time. Part-of: --- .../gst-libs/gst/adaptivedemux/gstadaptivedemux.c | 2 +- .../gst-libs/gst/pbutils/gstaudiovisualizer.c | 2 +- .../gst-libs/gst/video/gstvideoaggregator.c | 3 ++- .../gst-libs/gst/video/gstvideodecoder.c | 3 ++- .../gst-libs/gst/video/gstvideoencoder.c | 11 ++++++----- .../ext/adaptivedemux2/gstadaptivedemux.c | 2 +- .../gst-plugins-good/gst/deinterlace/gstdeinterlace.c | 2 +- .../gst-plugins-good/gst/monoscope/gstmonoscope.c | 2 +- .../gst-plugins-good/gst/shapewipe/gstshapewipe.c | 3 ++- .../gst-plugins-good/gst/videomixer/videomixer2.c | 3 ++- 10 files changed, 19 insertions(+), 14 deletions(-) diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/adaptivedemux/gstadaptivedemux.c b/subprojects/gst-plugins-bad/gst-libs/gst/adaptivedemux/gstadaptivedemux.c index b783a21967..cf2ede64ff 100644 --- a/subprojects/gst-plugins-bad/gst-libs/gst/adaptivedemux/gstadaptivedemux.c +++ b/subprojects/gst-plugins-bad/gst-libs/gst/adaptivedemux/gstadaptivedemux.c @@ -1917,7 +1917,7 @@ gst_adaptive_demux_src_event (GstPad * pad, GstObject * parent, gst_event_parse_qos (event, NULL, NULL, &diff, ×tamp); /* Only take into account lateness if late */ if (diff > 0) - earliest_time = timestamp + 2 * diff; + earliest_time = timestamp + MIN (2 * diff, GST_SECOND); else earliest_time = timestamp; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c b/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c index e46af8506d..c7a4c42733 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/pbutils/gstaudiovisualizer.c @@ -1264,7 +1264,7 @@ gst_audio_visualizer_src_event (GstPad * pad, GstObject * parent, if (diff >= 0) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ - scope->priv->earliest_time = timestamp + 2 * diff + + scope->priv->earliest_time = timestamp + MIN (2 * diff, GST_SECOND) + scope->priv->frame_duration; else scope->priv->earliest_time = timestamp + diff; diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoaggregator.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoaggregator.c index 3bc376eb74..1caa934f64 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoaggregator.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoaggregator.c @@ -1667,7 +1667,8 @@ gst_video_aggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { if (!live && G_UNLIKELY (diff > 0)) vagg->priv->earliest_time = - timestamp + 2 * diff + gst_util_uint64_scale_int_round (GST_SECOND, + timestamp + MIN (2 * diff, + GST_SECOND) + gst_util_uint64_scale_int_round (GST_SECOND, GST_VIDEO_INFO_FPS_D (&vagg->info), GST_VIDEO_INFO_FPS_N (&vagg->info)); else diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c index 47f2ec477a..9d4a5b8ecf 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideodecoder.c @@ -1834,7 +1834,8 @@ gst_video_decoder_src_event_default (GstVideoDecoder * decoder, priv->proportion = proportion; if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (timestamp))) { if (G_UNLIKELY (diff > 0)) { - priv->earliest_time = timestamp + 2 * diff + priv->qos_frame_duration; + priv->earliest_time = + timestamp + MIN (2 * diff, GST_SECOND) + priv->qos_frame_duration; } else { priv->earliest_time = timestamp + diff; } diff --git a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c index 23ea21f2fc..e24c827180 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c +++ b/subprojects/gst-plugins-base/gst-libs/gst/video/gstvideoencoder.c @@ -1372,7 +1372,8 @@ gst_video_encoder_src_event_default (GstVideoEncoder * encoder, priv->proportion = proportion; if (G_LIKELY (GST_CLOCK_TIME_IS_VALID (timestamp))) { if (G_UNLIKELY (diff > 0)) { - priv->earliest_time = timestamp + 2 * diff + priv->qos_frame_duration; + priv->earliest_time = + timestamp + MIN (2 * diff, GST_SECOND) + priv->qos_frame_duration; } else { priv->earliest_time = timestamp + diff; } @@ -2172,7 +2173,7 @@ gst_video_encoder_release_frame_unlocked (GstVideoEncoder * enc, * @encoder: a #GstVideoEncoder * @frame: (transfer full): a #GstVideoCodecFrame * - * Removes @frame from list of pending frames and releases it, similar + * Removes @frame from list of pending frames and releases it, similar * to calling gst_video_encoder_finish_frame() without a buffer attached * to the frame, but does not post a QoS message or do any additional * processing. Events from @frame are moved to the pending events list. @@ -2198,11 +2199,11 @@ gst_video_encoder_release_frame (GstVideoEncoder * enc, * @encoder: a #GstVideoEncoder * @frame: (transfer full): a #GstVideoCodecFrame * - * Removes @frame from the list of pending frames, releases it + * Removes @frame from the list of pending frames, releases it * and posts a QoS message with the frame's details on the bus. - * Similar to calling gst_video_encoder_finish_frame() without a buffer + * Similar to calling gst_video_encoder_finish_frame() without a buffer * attached to @frame, but this function additionally stores events - * from @frame as pending, to be pushed out alongside the next frame + * from @frame as pending, to be pushed out alongside the next frame * submitted via gst_video_encoder_finish_frame(). * * Since: 1.26 diff --git a/subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.c b/subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.c index 7597618ead..d953d27be9 100644 --- a/subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.c +++ b/subprojects/gst-plugins-good/ext/adaptivedemux2/gstadaptivedemux.c @@ -2561,7 +2561,7 @@ gst_adaptive_demux_src_event (GstPad * pad, GstObject * parent, gst_event_parse_qos (event, NULL, NULL, &diff, ×tamp); /* Only take into account lateness if late */ if (diff > 0) - earliest_time = timestamp + 2 * diff; + earliest_time = timestamp + MIN (2 * diff, GST_SECOND); else earliest_time = timestamp; diff --git a/subprojects/gst-plugins-good/gst/deinterlace/gstdeinterlace.c b/subprojects/gst-plugins-good/gst/deinterlace/gstdeinterlace.c index d1254a7d4b..998b3a4af4 100644 --- a/subprojects/gst-plugins-good/gst/deinterlace/gstdeinterlace.c +++ b/subprojects/gst-plugins-good/gst/deinterlace/gstdeinterlace.c @@ -1288,7 +1288,7 @@ gst_deinterlace_update_qos (GstDeinterlace * self, gdouble proportion, if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { if (G_UNLIKELY (diff > 0)) self->earliest_time = - timestamp + 2 * diff + ((self->fields == + timestamp + MIN (2 * diff, GST_SECOND) + ((self->fields == GST_DEINTERLACE_ALL) ? self->field_duration : 2 * self->field_duration); else diff --git a/subprojects/gst-plugins-good/gst/monoscope/gstmonoscope.c b/subprojects/gst-plugins-good/gst/monoscope/gstmonoscope.c index 867e7ec0e0..5ddae26880 100644 --- a/subprojects/gst-plugins-good/gst/monoscope/gstmonoscope.c +++ b/subprojects/gst-plugins-good/gst/monoscope/gstmonoscope.c @@ -534,7 +534,7 @@ gst_monoscope_src_event (GstPad * pad, GstObject * parent, GstEvent * event) /* we're late, this is a good estimate for next displayable * frame (see part-qos.txt) */ monoscope->earliest_time = - timestamp + 2 * diff + monoscope->frame_duration; + timestamp + MIN (2 * diff, GST_SECOND) + monoscope->frame_duration; else monoscope->earliest_time = timestamp + diff; GST_OBJECT_UNLOCK (monoscope); diff --git a/subprojects/gst-plugins-good/gst/shapewipe/gstshapewipe.c b/subprojects/gst-plugins-good/gst/shapewipe/gstshapewipe.c index 06aa5b7ea5..442e666463 100644 --- a/subprojects/gst-plugins-good/gst/shapewipe/gstshapewipe.c +++ b/subprojects/gst-plugins-good/gst/shapewipe/gstshapewipe.c @@ -695,7 +695,8 @@ gst_shape_wipe_update_qos (GstShapeWipe * self, gdouble proportion, self->proportion = proportion; if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { if (G_UNLIKELY (diff > 0)) - self->earliest_time = timestamp + 2 * diff + self->frame_duration; + self->earliest_time = + timestamp + MIN (2 * diff, GST_SECOND) + self->frame_duration; else self->earliest_time = timestamp + diff; } else { diff --git a/subprojects/gst-plugins-good/gst/videomixer/videomixer2.c b/subprojects/gst-plugins-good/gst/videomixer/videomixer2.c index 0ecc0c9224..a220aba7bc 100644 --- a/subprojects/gst-plugins-good/gst/videomixer/videomixer2.c +++ b/subprojects/gst-plugins-good/gst/videomixer/videomixer2.c @@ -816,7 +816,8 @@ gst_videomixer2_update_qos (GstVideoMixer2 * mix, gdouble proportion, if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { if (!mix->live && G_UNLIKELY (diff > 0)) mix->earliest_time = - timestamp + 2 * diff + gst_util_uint64_scale_int_round (GST_SECOND, + timestamp + MIN (2 * diff, + GST_SECOND) + gst_util_uint64_scale_int_round (GST_SECOND, GST_VIDEO_INFO_FPS_D (&mix->info), GST_VIDEO_INFO_FPS_N (&mix->info)); else mix->earliest_time = timestamp + diff;