diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs
index 3701a772..cd3ace96 100644
--- a/mux/mp4/src/mp4mux/imp.rs
+++ b/mux/mp4/src/mp4mux/imp.rs
@@ -143,6 +143,7 @@ struct Stream {
 impl Stream {
     fn get_elst_infos(
         &self,
+        start_time: gst::ClockTime,
         min_earliest_pts: gst::ClockTime,
     ) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
         let mut elst_infos = self.elst_infos.clone();
@@ -179,10 +180,11 @@ impl Stream {
         }
 
         // Add a gap at the beginning if needed
-        if min_earliest_pts != earliest_pts {
-            let gap_duration = (earliest_pts - min_earliest_pts).nseconds())
-                .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-                .context("too big gap")?;
+        if min_earliest_pts != earliest_pts || start_time > gst::ClockTime::ZERO {
+            let gap_duration = (start_time.nseconds()
+                + (earliest_pts - min_earliest_pts).nseconds())
+            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+            .context("too big gap")?;
 
             if gap_duration > 0 {
                 elst_infos.insert(
@@ -1516,6 +1518,13 @@ impl AggregatorImpl for MP4Mux {
                 state.mdat_size
             );
 
+            let obj = self.obj();
+            let start_time =
+                if obj.start_time_selection() == gst_base::AggregatorStartTimeSelection::Set {
+                    gst::ClockTime::from_nseconds(AggregatorExt::start_time(&*obj))
+                } else {
+                    gst::ClockTime::ZERO
+                };
             let min_earliest_pts = state
                 .streams
                 .iter()
@@ -1537,7 +1546,7 @@ impl AggregatorImpl for MP4Mux {
                     earliest_pts,
                     end_pts,
                     elst_infos: stream
-                        .get_elst_infos(min_earliest_pts)
+                        .get_elst_infos(start_time, min_earliest_pts)
                         .unwrap_or_else(|e| {
                             gst::error!(CAT, "Could not prepare edit lists: {e:?}");