fmp4mux: Add support to write edts to handle audio priming

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1434>

docs/plugins/gst_plugins_cache.json

@@ -2425,6 +2425,18 @@
                         "type": "guint",
                         "writable": true
                     },
+                    "write-edts-mode": {
+                        "blurb": "Mode for writing EDTS, when in auto mode, edts written only for non-live streams.",
+                        "conditionally-available": false,
+                        "construct": false,
+                        "construct-only": false,
+                        "controllable": false,
+                        "default": "auto (0)",
+                        "mutable": "ready",
+                        "readable": true,
+                        "type": "GstFMP4MuxWriteEdtsMode",
+                        "writable": true
+                    },
                     "write-mehd": {
                         "blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)",
                         "conditionally-available": false,
@@ -2516,6 +2528,26 @@
                         "writable": true
                     }
                 }
+            },
+            "GstFMP4MuxWriteEdtsMode": {
+                "kind": "enum",
+                "values": [
+                    {
+                        "desc": "Auto",
+                        "name": "auto",
+                        "value": "0"
+                    },
+                    {
+                        "desc": "Always",
+                        "name": "always",
+                        "value": "1"
+                    },
+                    {
+                        "desc": "Never",
+                        "name": "never",
+                        "value": "2"
+                    }
+                ]
             }
         },
         "package": "gst-plugin-fmp4",

mux/fmp4/src/fmp4mux/boxes.rs

@@ -6,6 +6,7 @@
 //
 // SPDX-License-Identifier: MPL-2.0
 
+use crate::fmp4mux::imp::CAT;
 use gst::prelude::*;
 
 use anyhow::{anyhow, bail, Context, Error};
@@ -555,6 +556,35 @@ struct TrackReference {
     track_ids: Vec<u32>,
 }
 
+fn write_edts(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
+    write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
+
+    Ok(())
+}
+
+fn write_elst(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
+    // Entry count
+    v.extend((stream.elst_infos.len() as u32).to_be_bytes());
+
+    for elst_info in &stream.elst_infos {
+        v.extend(
+            elst_info
+                .duration
+                .expect("Should have been set by `get_elst_infos`")
+                .to_be_bytes(),
+        );
+
+        // Media time
+        v.extend(elst_info.start.to_be_bytes());
+
+        // Media rate
+        v.extend(1u16.to_be_bytes());
+        v.extend(0u16.to_be_bytes());
+    }
+
+    Ok(())
+}
+
 fn write_trak(
     v: &mut Vec<u8>,
     cfg: &super::HeaderConfiguration,
@@ -571,10 +601,13 @@ fn write_trak(
         |v| write_tkhd(v, cfg, idx, stream, creation_time),
     )?;
 
-    // TODO: write edts if necessary: for audio tracks to remove initialization samples
     // TODO: write edts optionally for negative DTS instead of offsetting the DTS
-
     write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
+    if !stream.elst_infos.is_empty() {
+        if let Err(e) = write_edts(v, stream) {
+            gst::warning!(CAT, "Failed to write edts: {e}");
+        }
+    }
 
     if !references.is_empty() {
         write_box(v, b"tref", |v| write_tref(v, cfg, references))?;

mux/fmp4/src/fmp4mux/imp.rs

@@ -12,6 +12,7 @@ use gst::subclass::prelude::*;
 use gst_base::prelude::*;
 use gst_base::subclass::prelude::*;
 
+use anyhow::Context;
 use std::collections::BTreeSet;
 use std::collections::VecDeque;
 use std::mem;
@@ -90,7 +91,7 @@ fn utc_time_to_running_time(
         .and_then(|res| res.positive())
 }
 
-static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
+pub static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
     gst::DebugCategory::new(
         "fmp4mux",
         gst::DebugColorFlags::empty(),
@@ -234,6 +235,88 @@ struct Stream {
     running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>,
 
     extra_header_data: Option<Vec<u8>>,
+
+    /// Earliest PTS of the whole stream
+    earliest_pts: Option<gst::ClockTime>,
+    /// Current end PTS of the whole stream
+    end_pts: Option<gst::ClockTime>,
+
+    /// Edit list entries for this stream.
+    elst_infos: Vec<super::ElstInfo>,
+}
+
+impl Stream {
+    fn get_elst_infos(&self) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
+        let mut elst_infos = self.elst_infos.clone();
+        let timescale = self.timescale();
+        let earliest_pts = self.earliest_pts.unwrap_or(gst::ClockTime::ZERO);
+        let end_pts = self
+            .end_pts
+            .unwrap_or(gst::ClockTime::from_nseconds(u64::MAX - 1));
+
+        let mut iter = elst_infos.iter_mut().peekable();
+        while let Some(&mut ref mut elst_info) = iter.next() {
+            if elst_info.duration.unwrap_or(0) == 0 {
+                elst_info.duration = if let Some(next) = iter.peek_mut() {
+                    let duration = next.start - elst_info.start;
+                    if duration < 0 {
+                        anyhow::bail!("Invalid edit list entry, negative duration");
+                    }
+                    Some(duration as u64)
+                } else {
+                    Some(
+                        (end_pts.checked_sub(earliest_pts))
+                            .expect("stream end < stream start?")
+                            .nseconds()
+                            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                            .context("too big track duration")?,
+                    )
+                }
+            }
+        }
+
+        Ok(elst_infos)
+    }
+
+    fn timescale(&self) -> u32 {
+        let trak_timescale = { self.sinkpad.imp().state.lock().unwrap().trak_timescale };
+
+        if trak_timescale > 0 {
+            return trak_timescale;
+        }
+
+        let s = self.caps.structure(0).unwrap();
+
+        if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
+            if fps.numer() == 0 {
+                return 10_000;
+            }
+
+            if fps.denom() != 1 && fps.denom() != 1001 {
+                if let Some(fps) = (fps.denom() as u64)
+                    .nseconds()
+                    .mul_div_round(1_000_000_000, fps.numer() as u64)
+                    .and_then(gst_video::guess_framerate)
+                {
+                    return (fps.numer() as u32)
+                        .mul_div_round(100, fps.denom() as u32)
+                        .unwrap_or(10_000);
+                }
+            }
+
+            if fps.denom() == 1001 {
+                fps.numer() as u32
+            } else {
+                (fps.numer() as u32)
+                    .mul_div_round(100, fps.denom() as u32)
+                    .unwrap_or(10_000)
+            }
+        } else if let Ok(rate) = s.get::<i32>("rate") {
+            rate as u32
+        } else {
+            10_000
+        }
+    }
 }
 
 #[derive(Default)]
@@ -295,6 +378,68 @@ pub(crate) struct FMP4Mux {
 }
 
 impl FMP4Mux {
+    fn add_elst_info(
+        &self,
+        buffer: &PreQueuedBuffer,
+        stream: &mut Stream,
+    ) -> Result<(), anyhow::Error> {
+        if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
+            let timescale = stream
+                .caps
+                .structure(0)
+                .unwrap()
+                .get::<i32>("rate")
+                .unwrap_or_else(|_| stream.timescale() as i32);
+
+            let gstclocktime_to_samples = move |v: gst::ClockTime| {
+                v.nseconds()
+                    .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                    .context("Invalid start in the AudioClipMeta")
+            };
+
+            let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
+                if let gst::GenericFormattedValue::Default(Some(v)) = t {
+                    let v = v.into();
+                    Ok(Some(v).filter(|x| x != &0u64))
+                } else if let gst::GenericFormattedValue::Time(Some(v)) = t {
+                    Ok(Some(gstclocktime_to_samples(v)?))
+                } else {
+                    Ok(None)
+                }
+            };
+
+            let start = generic_to_samples(cmeta.start())?;
+            let end = generic_to_samples(cmeta.end())?;
+            if end.is_none() && start.is_none() {
+                return Err(anyhow::anyhow!(
+                    "No start or end time in `default` format in the AudioClipingMeta"
+                ));
+            }
+
+            let start = if let Some(start) = generic_to_samples(cmeta.start())? {
+                start + gstclocktime_to_samples(buffer.pts)?
+            } else {
+                0
+            };
+            let duration = if let Some(e) = end {
+                Some(
+                    gstclocktime_to_samples(buffer.end_pts)?
+                        .checked_sub(e)
+                        .context("Invalid end in the AudioClipMeta")?,
+                )
+            } else {
+                None
+            };
+
+            stream.elst_infos.push(super::ElstInfo {
+                start: start as i64,
+                duration,
+            });
+        }
+
+        Ok(())
+    }
+
     /// Checks if a buffer is valid according to the stream configuration.
     fn check_buffer(buffer: &gst::BufferRef, stream: &Stream) -> Result<(), gst::FlowError> {
         let Stream {
@@ -400,6 +545,17 @@ impl FMP4Mux {
                 gst::ClockTime::ZERO
             });
 
+        if stream
+            .earliest_pts
+            .map_or(true, |earliest_pts| earliest_pts > pts)
+        {
+            stream.end_pts = Some(pts);
+        }
+
+        if stream.end_pts.opt_lt(end_pts).unwrap_or(true) {
+            stream.end_pts = Some(end_pts);
+        }
+
         let (dts, end_dts) = if !stream.delta_frames.requires_dts() {
             (None, None)
         } else {
@@ -665,7 +821,13 @@ impl FMP4Mux {
             assert!(stream.running_time_utc_time_mapping.is_some());
         }
 
-        stream.pre_queue.pop_front().unwrap()
+        let buffer = stream.pre_queue.pop_front().unwrap();
+
+        if let Err(err) = self.add_elst_info(&buffer, stream) {
+            gst::error!(CAT, "Failed to add elst info: {err:#}");
+        }
+
+        buffer
     }
 
     // Caps/tag changes are allowed only in case that the
@@ -2083,8 +2245,6 @@ impl FMP4Mux {
         );
 
         for (idx, stream) in state.streams.iter_mut().enumerate() {
-            let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
-
             let gops = self.drain_buffers_one_stream(
                 settings,
                 stream,
@@ -2163,6 +2323,7 @@ impl FMP4Mux {
                 }
             }
 
+            let trak_timescale = stream.sinkpad.imp().state.lock().unwrap().trak_timescale;
             if gops.is_empty() {
                 gst::info!(CAT, obj = stream.sinkpad, "Draining no buffers",);
 
@@ -2171,7 +2332,7 @@ impl FMP4Mux {
                         caps: stream.caps.clone(),
                         start_time: None,
                         delta_frames: stream.delta_frames,
-                        trak_timescale: stream_settings.trak_timescale,
+                        trak_timescale,
                     },
                     VecDeque::new(),
                 ));
@@ -2222,7 +2383,7 @@ impl FMP4Mux {
                             caps: stream.caps.clone(),
                             start_time: None,
                             delta_frames: stream.delta_frames,
-                            trak_timescale: stream_settings.trak_timescale,
+                            trak_timescale,
                         },
                         VecDeque::new(),
                     ));
@@ -2270,7 +2431,7 @@ impl FMP4Mux {
                     caps: stream.caps.clone(),
                     start_time: Some(start_time),
                     delta_frames: stream.delta_frames,
-                    trak_timescale: stream_settings.trak_timescale,
+                    trak_timescale,
                 },
                 buffers,
             ));
@@ -2860,6 +3021,9 @@ impl FMP4Mux {
                 current_position: gst::ClockTime::ZERO,
                 running_time_utc_time_mapping: None,
                 extra_header_data: None,
+                earliest_pts: None,
+                end_pts: None,
+                elst_infos: Vec::new(),
             });
         }
 
@@ -2923,11 +3087,19 @@ impl FMP4Mux {
         let streams = state
             .streams
             .iter()
-            .map(|s| super::HeaderStream {
+            .map(|s| {
-                trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale,
+                let trak_timescale = { s.sinkpad.imp().settings.lock().unwrap().trak_timescale };
-                delta_frames: s.delta_frames,
+                super::HeaderStream {
-                caps: s.caps.clone(),
+                    trak_timescale,
-                extra_header_data: s.extra_header_data.clone(),
+                    delta_frames: s.delta_frames,
+                    caps: s.caps.clone(),
+                    extra_header_data: s.extra_header_data.clone(),
+                    elst_infos: s.get_elst_infos().unwrap_or_else(|e| {
+                        gst::error!(CAT, "Could not prepare edit lists: {e:?}");
+
+                        Vec::new()
+                    }),
+                }
             })
             .collect::<Vec<_>>();
 
@@ -3598,6 +3770,12 @@ impl AggregatorImpl for FMP4Mux {
     fn start(&self) -> Result<(), gst::ErrorMessage> {
         gst::trace!(CAT, imp = self, "Starting");
 
+        for pad in self.obj().sink_pads() {
+            let pad = pad.downcast_ref::<super::FMP4MuxPad>().unwrap().imp();
+
+            pad.state.lock().unwrap().trak_timescale = pad.settings.lock().unwrap().trak_timescale;
+        }
+
         self.parent_start()?;
 
         // For non-single-stream variants configure a default segment that allows for negative
@@ -4239,9 +4417,16 @@ struct PadSettings {
     trak_timescale: u32,
 }
 
+#[derive(Default, Clone)]
+struct PadState {
+    // the selected trak_timescale
+    trak_timescale: u32,
+}
+
 #[derive(Default)]
 pub(crate) struct FMP4MuxPad {
     settings: Mutex<PadSettings>,
+    state: Mutex<PadState>,
 }
 
 #[glib::object_subclass]

mux/fmp4/src/fmp4mux/mod.rs

@@ -169,6 +169,12 @@ pub(crate) struct HeaderConfiguration {
     start_utc_time: Option<u64>,
 }
 
+#[derive(Debug, Clone)]
+pub(crate) struct ElstInfo {
+    start: i64,
+    duration: Option<u64>,
+}
+
 #[derive(Debug)]
 pub(crate) struct HeaderStream {
     /// Caps of this stream
@@ -182,6 +188,9 @@ pub(crate) struct HeaderStream {
 
     // More data to be included in the fragmented stream header
     extra_header_data: Option<Vec<u8>>,
+
+    /// Edit list clipping information
+    elst_infos: Vec<ElstInfo>,
 }
 
 #[derive(Debug)]

mux/mp4/src/mp4mux/boxes.rs

@@ -225,50 +225,12 @@ fn write_moov(v: &mut Vec<u8>, header: &super::Header) -> Result<(), Error> {
     Ok(())
 }
 
-fn stream_to_timescale(stream: &super::Stream) -> u32 {
-    if stream.trak_timescale > 0 {
-        stream.trak_timescale
-    } else {
-        let s = stream.caps.structure(0).unwrap();
-
-        if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
-            if fps.numer() == 0 {
-                return 10_000;
-            }
-
-            if fps.denom() != 1 && fps.denom() != 1001 {
-                if let Some(fps) = (fps.denom() as u64)
-                    .nseconds()
-                    .mul_div_round(1_000_000_000, fps.numer() as u64)
-                    .and_then(gst_video::guess_framerate)
-                {
-                    return (fps.numer() as u32)
-                        .mul_div_round(100, fps.denom() as u32)
-                        .unwrap_or(10_000);
-                }
-            }
-
-            if fps.denom() == 1001 {
-                fps.numer() as u32
-            } else {
-                (fps.numer() as u32)
-                    .mul_div_round(100, fps.denom() as u32)
-                    .unwrap_or(10_000)
-            }
-        } else if let Ok(rate) = s.get::<i32>("rate") {
-            rate as u32
-        } else {
-            10_000
-        }
-    }
-}
-
 fn header_to_timescale(header: &super::Header) -> u32 {
     if header.movie_timescale > 0 {
         header.movie_timescale
     } else {
         // Use the reference track timescale
-        stream_to_timescale(&header.streams[0])
+        header.streams[0].timescale
     }
 }
 
@@ -352,7 +314,7 @@ fn write_trak(
     if !references.is_empty() {
         write_box(v, b"tref", |v| write_tref(v, header, references))?;
     }
-    write_box(v, b"edts", |v| write_edts(v, header, stream))?;
+    write_box(v, b"edts", |v| write_edts(v, stream))?;
 
     Ok(())
 }
@@ -480,7 +442,7 @@ fn write_mdhd(
     stream: &super::Stream,
     creation_time: u64,
 ) -> Result<(), Error> {
-    let timescale = stream_to_timescale(stream);
+    let timescale = stream.timescale;
 
     // Creation time
     v.extend(creation_time.to_be_bytes());
@@ -1454,7 +1416,7 @@ fn write_stts(
     _header: &super::Header,
     stream: &super::Stream,
 ) -> Result<(), Error> {
-    let timescale = stream_to_timescale(stream);
+    let timescale = stream.timescale;
 
     let entry_count_position = v.len();
     // Entry count, rewritten in the end
@@ -1508,7 +1470,7 @@ fn write_ctts(
     stream: &super::Stream,
     version: u8,
 ) -> Result<(), Error> {
-    let timescale = stream_to_timescale(stream);
+    let timescale = stream.timescale;
 
     let entry_count_position = v.len();
     // Entry count, rewritten in the end
@@ -1578,7 +1540,7 @@ fn write_cslg(
     _header: &super::Header,
     stream: &super::Stream,
 ) -> Result<(), Error> {
-    let timescale = stream_to_timescale(stream);
+    let timescale = stream.timescale;
 
     let (min_ctts, max_ctts) = stream
         .chunks
@@ -1794,82 +1756,31 @@ fn write_tref(
     Ok(())
 }
 
-fn write_edts(
+fn write_edts(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
-    v: &mut Vec<u8>,
+    write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
-    header: &super::Header,
-    stream: &super::Stream,
-) -> Result<(), Error> {
-    write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| {
-        write_elst(v, header, stream)
-    })?;
 
     Ok(())
 }
 
-fn write_elst(
+fn write_elst(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
-    v: &mut Vec<u8>,
+    // Entry count
-    header: &super::Header,
+    v.extend((stream.elst_infos.len() as u32).to_be_bytes());
-    stream: &super::Stream,
-) -> Result<(), Error> {
-    // In movie header timescale
-    let timescale = header_to_timescale(header);
 
-    let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap();
+    for elst_info in &stream.elst_infos {
+        v.extend(
+            elst_info
+                .duration
+                .expect("Should have been set by `get_elst_infos`")
+                .to_be_bytes(),
+        );
 
-    if min_earliest_pts != stream.earliest_pts {
+        // Media time
-        let gap = (stream.earliest_pts - min_earliest_pts)
+        v.extend(elst_info.start.to_be_bytes());
-            .nseconds()
-            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-            .context("too big gap")?;
 
-        if gap > 0 {
+        // Media rate
-            // Entry count
+        v.extend(1u16.to_be_bytes());
-            v.extend(2u32.to_be_bytes());
+        v.extend(0u16.to_be_bytes());
-
-            // First entry for the gap
-
-            // Edit duration
-            v.extend(gap.to_be_bytes());
-
-            // Media time
-            v.extend((-1i64).to_be_bytes());
-
-            // Media rate
-            v.extend(1u16.to_be_bytes());
-            v.extend(0u16.to_be_bytes());
-        } else {
-            // Entry count
-            v.extend(1u32.to_be_bytes());
-        }
-    } else {
-        // Entry count
-        v.extend(1u32.to_be_bytes());
     }
 
-    // Edit duration
-    let duration = (stream.end_pts - stream.earliest_pts)
-        .nseconds()
-        .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-        .context("too big track duration")?;
-    v.extend(duration.to_be_bytes());
-
-    // Media time
-    if let Some(start_dts) = stream.start_dts {
-        let shift = (gst::Signed::Positive(stream.earliest_pts) - start_dts)
-            .nseconds()
-            .positive()
-            .unwrap_or(0)
-            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
-            .context("too big track duration")?;
-
-        v.extend(shift.to_be_bytes());
-    } else {
-        v.extend(0u64.to_be_bytes());
-    }
-
-    // Media rate
-    v.extend(1u16.to_be_bytes());
-    v.extend(0u16.to_be_bytes());
-
     Ok(())
 }

mux/mp4/src/mp4mux/imp.rs

@@ -6,6 +6,7 @@
 //
 // SPDX-License-Identifier: MPL-2.0
 
+use anyhow::{anyhow, Context};
 use gst::glib;
 use gst::prelude::*;
 use gst::subclass::prelude::*;
@@ -131,6 +132,10 @@ struct Stream {
 
     /// Earliest PTS.
     earliest_pts: Option<gst::ClockTime>,
+
+    /// Edit list entries for this stream.
+    elst_infos: Vec<super::ElstInfo>,
+
     /// Current end PTS.
     end_pts: Option<gst::ClockTime>,
 
@@ -143,6 +148,122 @@ struct Stream {
     orientation: Option<ImageOrientation>,
 }
 
+impl Stream {
+    fn get_elst_infos(
+        &self,
+        min_earliest_pts: gst::ClockTime,
+    ) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
+        let mut elst_infos = self.elst_infos.clone();
+        let timescale = self.timescale();
+        let earliest_pts = self
+            .earliest_pts
+            .expect("Streams without earliest_pts should have been skipped");
+        let end_pts = self
+            .end_pts
+            .expect("Streams without end_pts should have been skipped");
+
+        // If no elst info were set, use the whole track
+        if self.elst_infos.is_empty() {
+            let start = if let Some(start_dts) = self.start_dts {
+                ((gst::Signed::Positive(earliest_pts) - start_dts)
+                    .nseconds()
+                    .positive()
+                    .unwrap_or(0)
+                    .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                    .context("too big track duration")?) as i64
+            } else {
+                0i64
+            };
+
+            elst_infos.push(super::ElstInfo {
+                start,
+                duration: Some(
+                    (end_pts - earliest_pts)
+                        .nseconds()
+                        .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                        .context("too big track duration")?,
+                ),
+            });
+        }
+
+        // Add a gap at the beginning if needed
+        if min_earliest_pts != earliest_pts {
+            let gap_duration = (earliest_pts - min_earliest_pts)
+                .nseconds()
+                .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                .context("too big gap")?;
+
+            if gap_duration > 0 {
+                elst_infos.insert(
+                    0,
+                    super::ElstInfo {
+                        start: -1,
+                        duration: Some(gap_duration),
+                    },
+                );
+            }
+        }
+
+        let mut iter = elst_infos.iter_mut().peekable();
+        while let Some(&mut ref mut elst_info) = iter.next() {
+            if elst_info.duration.unwrap_or(0u64) == 0u64 {
+                elst_info.duration = if let Some(next) = iter.peek_mut() {
+                    Some((next.start - elst_info.start) as u64)
+                } else {
+                    Some(
+                        (end_pts - earliest_pts)
+                            .nseconds()
+                            .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                            .context("too big track duration")?,
+                    )
+                }
+            }
+        }
+
+        Ok(elst_infos)
+    }
+
+    fn timescale(&self) -> u32 {
+        let trak_timescale = { self.sinkpad.imp().settings.lock().unwrap().trak_timescale };
+
+        if trak_timescale > 0 {
+            return trak_timescale;
+        }
+
+        let s = self.caps.structure(0).unwrap();
+
+        if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
+            if fps.numer() == 0 {
+                return 10_000;
+            }
+
+            if fps.denom() != 1 && fps.denom() != 1001 {
+                if let Some(fps) = (fps.denom() as u64)
+                    .nseconds()
+                    .mul_div_round(1_000_000_000, fps.numer() as u64)
+                    .and_then(gst_video::guess_framerate)
+                {
+                    return (fps.numer() as u32)
+                        .mul_div_round(100, fps.denom() as u32)
+                        .unwrap_or(10_000);
+                }
+            }
+
+            if fps.denom() == 1001 {
+                fps.numer() as u32
+            } else {
+                (fps.numer() as u32)
+                    .mul_div_round(100, fps.denom() as u32)
+                    .unwrap_or(10_000)
+            }
+        } else if let Ok(rate) = s.get::<i32>("rate") {
+            rate as u32
+        } else {
+            10_000
+        }
+    }
+}
+
 #[derive(Default)]
 struct State {
     /// List of streams when the muxer was started.
@@ -200,6 +321,77 @@ impl MP4Mux {
         Ok(())
     }
 
+    fn add_elst_info(
+        &self,
+        buffer: &PendingBuffer,
+        stream: &mut Stream,
+    ) -> Result<(), anyhow::Error> {
+        let cmeta = if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
+            cmeta
+        } else {
+            return Ok(());
+        };
+
+        let timescale = stream
+            .caps
+            .structure(0)
+            .unwrap()
+            .get::<i32>("rate")
+            .unwrap_or_else(|_| stream.timescale() as i32);
+
+        let gstclocktime_to_samples = move |v: gst::ClockTime| {
+            v.nseconds()
+                .mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
+                .context("Invalid start in the AudioClipMeta")
+        };
+
+        let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
+            if let gst::GenericFormattedValue::Default(Some(v)) = t {
+                let v: u64 = v.into();
+                Ok(Some(v).filter(|x| x != &0u64))
+            } else if let gst::GenericFormattedValue::Time(Some(v)) = t {
+                Ok(Some(gstclocktime_to_samples(v)?))
+            } else {
+                Ok(None)
+            }
+        };
+
+        let start: Option<u64> = generic_to_samples(cmeta.start())?;
+        let end: Option<u64> = generic_to_samples(cmeta.end())?;
+
+        if end.is_none() && start.is_none() {
+            return Err(anyhow!(
+                "No start or end time in `default` format in the AudioClipingMeta"
+            ));
+        }
+
+        let start = if let Some(start) = generic_to_samples(cmeta.start())? {
+            start + gstclocktime_to_samples(buffer.pts)?
+        } else {
+            0
+        };
+        let duration = if let Some(e) = end {
+            Some(
+                gstclocktime_to_samples(buffer.pts)?
+                    + gstclocktime_to_samples(
+                        buffer
+                            .duration
+                            .context("No duration on buffer, we can't add edit list")?,
+                    )?
+                    - e,
+            )
+        } else {
+            None
+        };
+
+        stream.elst_infos.push(super::ElstInfo {
+            start: start as i64,
+            duration,
+        });
+
+        Ok(())
+    }
+
     fn peek_buffer(
         &self,
         sinkpad: &super::MP4MuxPad,
@@ -889,6 +1081,11 @@ impl MP4Mux {
 
             let duration = buffer.duration.unwrap();
             let composition_time_offset = buffer.composition_time_offset;
+
+            if let Err(err) = self.add_elst_info(&buffer, stream) {
+                gst::error!(CAT, "Failed to add elst info: {:#}", err);
+            }
+
             let mut buffer = buffer.buffer;
 
             stream.queued_chunk_time += duration;
@@ -1018,6 +1215,7 @@ impl MP4Mux {
                 queued_chunk_bytes: 0,
                 start_dts: None,
                 earliest_pts: None,
+                elst_infos: Default::default(),
                 end_pts: None,
                 running_time_utc_time_mapping: None,
                 extra_header_data: None,
@@ -1468,9 +1666,14 @@ impl AggregatorImpl for MP4Mux {
                 state.mdat_size
             );
 
+            let min_earliest_pts = state
+                .streams
+                .iter()
+                .filter_map(|s| s.earliest_pts)
+                .min()
+                .unwrap();
             let mut streams = Vec::with_capacity(state.streams.len());
             for stream in state.streams.drain(..) {
-                let pad_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
                 let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts)
                 {
                     Some(res) => res,
@@ -1480,10 +1683,14 @@ impl AggregatorImpl for MP4Mux {
                 streams.push(super::Stream {
                     caps: stream.caps.clone(),
                     delta_frames: stream.delta_frames,
-                    trak_timescale: pad_settings.trak_timescale,
+                    timescale: stream.timescale(),
-                    start_dts: stream.start_dts,
                     earliest_pts,
                     end_pts,
+                    elst_infos: stream.get_elst_infos(min_earliest_pts).unwrap_or_else(|e| {
+                        gst::error!(CAT, "Could not prepare edit lists: {e:?}");
+
+                        Vec::new()
+                    }),
                     chunks: stream.chunks,
                     extra_header_data: stream.extra_header_data.clone(),
                     orientation: stream.orientation,

mux/mp4/src/mp4mux/mod.rs

@@ -172,6 +172,12 @@ pub(crate) struct Chunk {
     samples: Vec<Sample>,
 }
 
+#[derive(Debug, Clone)]
+pub(crate) struct ElstInfo {
+    start: i64,
+    duration: Option<u64>,
+}
+
 #[derive(Debug)]
 pub(crate) struct Stream {
     /// Caps of this stream
@@ -181,15 +187,7 @@ pub(crate) struct Stream {
     delta_frames: DeltaFrames,
 
     /// Pre-defined trak timescale if not 0.
-    trak_timescale: u32,
+    timescale: u32,
-
-    /// Start DTS
-    ///
-    /// If this is negative then an edit list entry is needed to
-    /// make all sample times positive.
-    ///
-    /// This is `None` for streams that have no concept of DTS.
-    start_dts: Option<gst::Signed<gst::ClockTime>>,
 
     /// Earliest PTS
     ///
@@ -207,6 +205,9 @@ pub(crate) struct Stream {
 
     /// Orientation from tags
     orientation: Option<ImageOrientation>,
+
+    /// Edit list clipping information
+    elst_infos: Vec<ElstInfo>,
 }
 
 #[derive(Debug)]