diff --git a/gst/isomp4/qtdemux.c b/gst/isomp4/qtdemux.c
index b38e88ecf3..91a22dc9e8 100644
--- a/gst/isomp4/qtdemux.c
+++ b/gst/isomp4/qtdemux.c
@@ -5811,8 +5811,10 @@ gst_qtdemux_loop_state_movie (GstQTDemux * qtdemux)
       GST_TIME_ARGS (dts), GST_TIME_ARGS (pts), GST_TIME_ARGS (duration));
 
   if (G_UNLIKELY (empty)) {
-    /* empty segment, push a gap and move to the next one */
-    gst_pad_push_event (stream->pad, gst_event_new_gap (pts, duration));
+    /* empty segment, push a gap if there's more than a second
+     * difference and move to the next one */
+    if ((pts + duration - stream->segment.position) > GST_SECOND)
+      gst_pad_push_event (stream->pad, gst_event_new_gap (pts, duration));
     stream->segment.position = pts + duration;
     goto next;
   }