fmp4mux: Use correct timescales for edit lists

The duration is using the movie timescale while the media time is using
the media / track timescale.

Previously both were using the track timescale.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/2167>

mux/fmp4/src/fmp4mux/boxes.rs

@@ -543,32 +543,70 @@ struct TrackReference {
     track_ids: Vec<u32>,
 }
 
-fn write_edts(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
-    write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
+fn write_edts(
+    v: &mut Vec<u8>,
+    cfg: &super::HeaderConfiguration,
+    stream: &super::HeaderStream,
+) -> Result<(), Error> {
+    write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| {
+        write_elst(v, cfg, stream)
+    })?;
 
     Ok(())
 }
 
-fn write_elst(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
+fn write_elst(
+    v: &mut Vec<u8>,
+    cfg: &super::HeaderConfiguration,
+    stream: &super::HeaderStream,
+) -> Result<(), Error> {
+    let movie_timescale = header_configuration_to_timescale(cfg);
+    let track_timescale = header_stream_to_timescale(stream);
+
     // Entry count
-    v.extend((stream.elst_infos.len() as u32).to_be_bytes());
+    let mut num_entries = 0u32;
+    let entry_count_position = v.len();
+    // Entry count, rewritten in the end
+    v.extend(0u32.to_be_bytes());
 
     for elst_info in &stream.elst_infos {
-        v.extend(
-            elst_info
-                .duration
-                .expect("Should have been set by `get_elst_infos`")
-                .to_be_bytes(),
-        );
+        // Edit duration (in movie timescale)
+        let edit_duration = elst_info
+            .duration
+            .expect("Should have been set by `get_elst_infos`")
+            .nseconds()
+            .mul_div_round(movie_timescale as u64, gst::ClockTime::SECOND.nseconds())
+            .unwrap();
 
-        // Media time
-        v.extend(elst_info.start.to_be_bytes());
+        if edit_duration == 0 {
+            continue;
+        }
+        v.extend(edit_duration.to_be_bytes());
+
+        // Media time (in media timescale)
+        let media_time = elst_info
+            .start
+            .map(|start| {
+                i64::try_from(start)
+                    .unwrap()
+                    .mul_div_round(
+                        track_timescale as i64,
+                        gst::ClockTime::SECOND.nseconds() as i64,
+                    )
+                    .unwrap()
+            })
+            .unwrap_or(-1i64);
+        v.extend(media_time.to_be_bytes());
 
         // Media rate
         v.extend(1u16.to_be_bytes());
         v.extend(0u16.to_be_bytes());
+        num_entries += 1;
     }
 
+    // Rewrite entry count
+    v[entry_count_position..][..4].copy_from_slice(&num_entries.to_be_bytes());
+
     Ok(())
 }
 
@@ -591,7 +629,7 @@ fn write_trak(
     // TODO: write edts optionally for negative DTS instead of offsetting the DTS
     write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
     if !stream.elst_infos.is_empty() && cfg.write_edts {
-        if let Err(e) = write_edts(v, stream) {
+        if let Err(e) = write_edts(v, cfg, stream) {
             gst::warning!(CAT, "Failed to write edts: {e}");
         }
     }

mux/fmp4/src/fmp4mux/imp.rs

@@ -12,7 +12,7 @@ use gst::subclass::prelude::*;
 use gst_base::prelude::*;
 use gst_base::subclass::prelude::*;
 
-use anyhow::Context;
+use anyhow::{bail, Context};
 use std::collections::BTreeSet;
 use std::collections::VecDeque;
 use std::mem;
@@ -268,29 +268,28 @@ struct Stream {
 impl Stream {
     fn get_elst_infos(&self) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
         let mut elst_infos = self.elst_infos.clone();
-        let timescale = self.timescale();
         let earliest_pts = self.earliest_pts.unwrap_or(gst::ClockTime::ZERO);
         let end_pts = self
             .end_pts
             .unwrap_or(gst::ClockTime::from_nseconds(u64::MAX - 1));
 
-        let mut iter = elst_infos.iter_mut().peekable();
+        let mut iter = elst_infos
+            .iter_mut()
+            .filter(|e| e.start.is_some())
+            .peekable();
         while let Some(&mut ref mut elst_info) = iter.next() {
-            if elst_info.duration.unwrap_or(0) == 0 {
+            if elst_info
+                .duration
+                .map_or(true, |duration| duration.is_zero())
+            {
                 elst_info.duration = if let Some(next) = iter.peek_mut() {
-                    let duration = next.start - elst_info.start;
-                    if duration < 0 {
-                        anyhow::bail!("Invalid edit list entry, negative duration");
-                    }
-                    Some(duration as u64)
-                } else {
                     Some(
-                        (end_pts.checked_sub(earliest_pts))
-                            .expect("stream end < stream start?")
-                            .nseconds()
-                            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-                            .context("too big track duration")?,
+                        (next.start.unwrap() - elst_info.start.unwrap())
+                            .positive()
+                            .unwrap_or(gst::ClockTime::ZERO),
                     )
+                } else {
+                    Some(end_pts - earliest_pts)
                 }
             }
         }
@@ -447,60 +446,57 @@ impl FMP4Mux {
         buffer: &PreQueuedBuffer,
         stream: &mut Stream,
     ) -> Result<(), anyhow::Error> {
-        if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
-            let timescale = stream
-                .caps
-                .structure(0)
-                .unwrap()
-                .get::<i32>("rate")
-                .unwrap_or_else(|_| stream.timescale() as i32);
+        let cmeta = if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
+            cmeta
+        } else {
+            return Ok(());
+        };
 
-            let gstclocktime_to_samples = move |v: gst::ClockTime| {
-                v.nseconds()
-                    .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-                    .context("Invalid start in the AudioClipMeta")
-            };
+        let timescale = stream
+            .caps
+            .structure(0)
+            .unwrap()
+            .get::<i32>("rate")
+            .unwrap_or_else(|_| stream.timescale() as i32);
 
-            let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
-                if let gst::GenericFormattedValue::Default(Some(v)) = t {
-                    let v = v.into();
-                    Ok(Some(v).filter(|x| x != &0u64))
-                } else if let gst::GenericFormattedValue::Time(Some(v)) = t {
-                    Ok(Some(gstclocktime_to_samples(v)?))
-                } else {
-                    Ok(None)
-                }
-            };
+        let samples_to_gstclocktime = move |v: u64| {
+            let nseconds = v
+                .mul_div_round(gst::ClockTime::SECOND.nseconds(), timescale as u64)
+                .context("Invalid start in the AudioClipMeta")?;
+            Ok::<_, anyhow::Error>(gst::ClockTime::from_nseconds(nseconds))
+        };
 
-            let start = generic_to_samples(cmeta.start())?;
-            let end = generic_to_samples(cmeta.end())?;
-            if end.is_none() && start.is_none() {
-                return Err(anyhow::anyhow!(
-                    "No start or end time in `default` format in the AudioClipingMeta"
-                ));
+        let generic_to_gstclocktime = move |t| -> Result<Option<gst::ClockTime>, anyhow::Error> {
+            if let gst::GenericFormattedValue::Default(Some(v)) = t {
+                let v = u64::from(v);
+                let v = samples_to_gstclocktime(v)?;
+                Ok(Some(v).filter(|x| !x.is_zero()))
+            } else if let gst::GenericFormattedValue::Time(Some(v)) = t {
+                Ok(Some(v).filter(|x| !x.is_zero()))
+            } else {
+                Ok(None)
             }
+        };
 
-            let start = if let Some(start) = generic_to_samples(cmeta.start())? {
-                start + gstclocktime_to_samples(buffer.pts)?
-            } else {
-                0
-            };
-            let duration = if let Some(e) = end {
-                Some(
-                    gstclocktime_to_samples(buffer.end_pts)?
-                        .checked_sub(e)
-                        .context("Invalid end in the AudioClipMeta")?,
-                )
-            } else {
-                None
-            };
+        let start = generic_to_gstclocktime(cmeta.start())?;
+        let end = generic_to_gstclocktime(cmeta.end())?;
 
-            stream.elst_infos.push(super::ElstInfo {
-                start: start as i64,
-                duration,
-            });
+        if end.is_none() && start.is_none() {
+            bail!("No start or end time in `default` format in the AudioClipingMeta");
         }
 
+        let start = if let Some(start) = generic_to_gstclocktime(cmeta.start())? {
+            start + buffer.pts
+        } else {
+            gst::ClockTime::ZERO
+        };
+        let duration = end.map(|end| buffer.end_pts - end);
+
+        stream.elst_infos.push(super::ElstInfo {
+            start: Some(start.into()),
+            duration,
+        });
+
         Ok(())
     }
 

mux/fmp4/src/fmp4mux/mod.rs

@@ -197,8 +197,8 @@ pub(crate) struct HeaderConfiguration {
 
 #[derive(Debug, Clone)]
 pub(crate) struct ElstInfo {
-    start: i64,
-    duration: Option<u64>,
+    start: Option<gst::Signed<gst::ClockTime>>,
+    duration: Option<gst::ClockTime>,
 }
 
 #[derive(Debug)]