fmp4mux: Implement interleaving based on start timestamps instead of accumulated durations

Durations might accumulate rounding errors and streams might not
actually start at the same time. For that reason also start with the
stream that has the lowest timestamp.

generic/fmp4/src/fmp4mux/imp.rs

@@ -639,30 +639,47 @@ impl FMP4Mux {
         // Interleave buffers according to the settings into a single vec
         let mut interleaved_buffers =
             Vec::with_capacity(drain_buffers.iter().map(|bs| bs.len()).sum());
-        while drain_buffers.iter().any(|bs| !bs.is_empty()) {
+        while let Some((idx, bs)) =
-            for (idx, bs) in drain_buffers.iter_mut().enumerate() {
+            drain_buffers
-                let mut dequeued_time = gst::ClockTime::ZERO;
+                .iter_mut()
+                .enumerate()
+                .min_by(|(a_idx, a), (b_idx, b)| {
+                    let (a, b) = match (a.front(), b.front()) {
+                        (None, None) => return std::cmp::Ordering::Equal,
+                        (None, _) => return std::cmp::Ordering::Greater,
+                        (_, None) => return std::cmp::Ordering::Less,
+                        (Some(a), Some(b)) => (a, b),
+                    };
+
+                    match a.dts.unwrap_or(a.pts).cmp(&b.dts.unwrap_or(b.pts)) {
+                        std::cmp::Ordering::Equal => a_idx.cmp(b_idx),
+                        cmp => cmp,
+                    }
+                })
+        {
+            let start_time = match bs.front() {
+                None => {
+                    // No more buffers now
+                    break;
+                }
+                Some(buf) => buf.dts.unwrap_or(buf.pts),
+            };
+            let mut current_end_time = start_time;
             let mut dequeued_bytes = 0;
 
             while settings
                 .interleave_bytes
                 .map_or(true, |max_bytes| dequeued_bytes <= max_bytes)
-                    && settings
+                && settings.interleave_time.map_or(true, |max_time| {
-                        .interleave_time
+                    current_end_time.saturating_sub(start_time) <= max_time
-                        .map_or(true, |max_time| dequeued_time <= max_time)
+                })
             {
                 if let Some(buffer) = bs.pop_front() {
-                        dequeued_time += match bs.front() {
+                    current_end_time = match bs.front() {
-                            Some(next_buffer) => match Option::zip(next_buffer.dts, buffer.dts) {
+                        Some(next_buffer) => next_buffer.dts.unwrap_or(next_buffer.pts),
-                                Some((b, a)) => b.saturating_sub(a),
-                                None => next_buffer.pts.saturating_sub(buffer.pts),
-                            },
                         None => {
                             let timing_info = timing_infos[idx].as_ref().unwrap();
-                                match Option::zip(timing_info.end_dts, buffer.dts) {
+                            timing_info.end_dts.unwrap_or(timing_info.end_pts)
-                                    Some((b, a)) => b.saturating_sub(a),
-                                    None => timing_info.end_pts.saturating_sub(buffer.pts),
-                                }
                         }
                     };
                     dequeued_bytes += buffer.buffer.size() as u64;
@@ -673,7 +690,8 @@ impl FMP4Mux {
                 }
             }
         }
-        }
+
+        assert!(drain_buffers.iter().all(|bs| bs.is_empty()));
 
         let mut buffer_list = None;