diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs
index 96d6b912..68c62540 100644
--- a/mux/mp4/src/mp4mux/imp.rs
+++ b/mux/mp4/src/mp4mux/imp.rs
@@ -779,6 +779,16 @@ impl MP4Mux {
                 && buffer.buffer.size() == 0
             {
                 gst::trace!(CAT, obj: stream.sinkpad, "Skipping gap buffer {buffer:?}");
+
+                // If a new chunk was just started for the gap buffer, don't bother and get rid
+                // of this chunk again for now and search for the next stream.
+                if let Some(chunk) = stream.chunks.last() {
+                    if chunk.samples.is_empty() {
+                        let _ = stream.chunks.pop();
+                        state.current_stream_idx = None;
+                    }
+                }
+
                 continue;
             }