fmp4mux: Write prft box into each fragment with the NTP / media time mapping if possible

The NTP time is based on the reference timestamp meta of the buffer that
has the start media time of the fragment.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/2193>

mux/fmp4/src/fmp4mux/boxes.rs

@@ -1800,7 +1800,18 @@ pub(super) fn create_fmp4_fragment_header(
         })?;
     }
 
-    let styp_len = v.len();
+    // Write prft for the first stream if we can
+    if let Some(stream) = cfg.streams.first() {
+        if let Some((start_time, start_ntp_time)) =
+            Option::zip(stream.start_time, stream.start_ntp_time)
+        {
+            write_full_box(&mut v, b"prft", FULL_BOX_VERSION_1, 8, |v| {
+                write_prft(v, &cfg, 0, stream, start_time, start_ntp_time)
+            })?;
+        }
+    }
+
+    let moof_pos = v.len();
 
     let data_offset_offsets = write_box(&mut v, b"moof", |v| write_moof(v, &cfg))?;
 
@@ -1818,7 +1829,7 @@ pub(super) fn create_fmp4_fragment_header(
         v.extend((size + 16).to_be_bytes());
     }
 
-    let data_offset = v.len() - styp_len;
+    let data_offset = v.len() - moof_pos;
     for data_offset_offset in data_offset_offsets {
         let val = u32::from_be_bytes(v[data_offset_offset..][..4].try_into()?)
             .checked_add(u32::try_from(data_offset)?)
@@ -1826,7 +1837,33 @@ pub(super) fn create_fmp4_fragment_header(
         v[data_offset_offset..][..4].copy_from_slice(&val.to_be_bytes());
     }
 
-    Ok((gst::Buffer::from_mut_slice(v), styp_len as u64))
+    Ok((gst::Buffer::from_mut_slice(v), moof_pos as u64))
+}
+
+fn write_prft(
+    v: &mut Vec<u8>,
+    _cfg: &super::FragmentHeaderConfiguration,
+    idx: usize,
+    stream: &super::FragmentHeaderStream,
+    start_time: gst::ClockTime,
+    start_ntp_time: gst::ClockTime,
+) -> Result<(), Error> {
+    // Reference track ID
+    v.extend((idx as u32 + 1).to_be_bytes());
+    // NTP timestamp
+    let start_ntp_time = start_ntp_time
+        .nseconds()
+        .mul_div_floor(1u64 << 32, gst::ClockTime::SECOND.nseconds())
+        .unwrap();
+    v.extend(start_ntp_time.to_be_bytes());
+    // Media time
+    let timescale = fragment_header_stream_to_timescale(stream);
+    let media_time = start_time
+        .mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds())
+        .unwrap();
+    v.extend(media_time.to_be_bytes());
+
+    Ok(())
 }
 
 fn write_moof(

mux/fmp4/src/fmp4mux/imp.rs

@@ -2437,6 +2437,10 @@ impl FMP4Mux {
             Option<gst::ClockTime>,
             // End DTS
             Option<gst::ClockTime>,
+            // Start time (either matches start_dts if required or earliest-pts)
+            gst::ClockTime,
+            // Start NTP time (either matches start_dts if required or earliest_pts)
+            Option<gst::ClockTime>,
         )>,
         gst::FlowError,
     > {
@@ -2456,6 +2460,7 @@ impl FMP4Mux {
         let mut earliest_pts_position = None;
         let mut start_dts = None;
         let mut start_dts_position = None;
+        let mut start_ntp_time = None;
 
         let mut gop_buffers = gop_buffers.into_iter();
         while let Some(buffer) = gop_buffers.next() {
@@ -2472,6 +2477,11 @@ impl FMP4Mux {
 
             if earliest_pts.is_none_or(|earliest_pts| buffer.pts < earliest_pts) {
                 earliest_pts = Some(buffer.pts);
+                if !stream.delta_frames.requires_dts() {
+                    let utc_time = get_utc_time_from_buffer(&buffer.buffer)
+                        .and_then(|t| t.checked_add(NTP_UNIX_OFFSET.seconds()));
+                    start_ntp_time = utc_time;
+                }
             }
             if earliest_pts_position.is_none_or(|earliest_pts_position| {
                 buffer.buffer.pts().unwrap() < earliest_pts_position
@@ -2480,6 +2490,11 @@ impl FMP4Mux {
             }
             if stream.delta_frames.requires_dts() && start_dts.is_none() {
                 start_dts = Some(buffer.dts.unwrap());
+                if stream.delta_frames.requires_dts() {
+                    let utc_time = get_utc_time_from_buffer(&buffer.buffer)
+                        .and_then(|t| t.checked_add(NTP_UNIX_OFFSET.seconds()));
+                    start_ntp_time = utc_time;
+                }
             }
             if stream.delta_frames.requires_dts() && start_dts_position.is_none() {
                 start_dts_position = Some(buffer.buffer.dts().unwrap());
@@ -2557,6 +2572,12 @@ impl FMP4Mux {
         let start_dts = start_dts;
         let start_dts_position = start_dts_position;
 
+        let start_time = if !stream.delta_frames.requires_dts() {
+            earliest_pts
+        } else {
+            start_dts.unwrap()
+        };
+
         Ok(Some((
             buffers,
             earliest_pts,
@@ -2565,6 +2586,8 @@ impl FMP4Mux {
             start_dts,
             start_dts_position,
             end_dts,
+            start_time,
+            start_ntp_time,
         )))
     }
 
@@ -2760,6 +2783,7 @@ impl FMP4Mux {
                     super::FragmentHeaderStream {
                         caps: stream.caps.clone(),
                         start_time: None,
+                        start_ntp_time: None,
                         delta_frames: stream.delta_frames,
                         trak_timescale,
                     },
@@ -2802,6 +2826,8 @@ impl FMP4Mux {
                 start_dts,
                 start_dts_position,
                 _end_dts,
+                start_time,
+                start_ntp_time,
             ) = match buffers {
                 Some(res) => res,
                 None => {
@@ -2811,6 +2837,7 @@ impl FMP4Mux {
                         super::FragmentHeaderStream {
                             caps: stream.caps.clone(),
                             start_time: None,
+                            start_ntp_time: None,
                             delta_frames: stream.delta_frames,
                             trak_timescale,
                         },
@@ -2831,12 +2858,6 @@ impl FMP4Mux {
                 stream.dts_offset.display(),
             );
 
-            let start_time = if !stream.delta_frames.requires_dts() {
-                earliest_pts
-            } else {
-                start_dts.unwrap()
-            };
-
             if min_earliest_pts.opt_gt(earliest_pts).unwrap_or(true) {
                 min_earliest_pts = Some(earliest_pts);
             }
@@ -2859,6 +2880,7 @@ impl FMP4Mux {
                 super::FragmentHeaderStream {
                     caps: stream.caps.clone(),
                     start_time: Some(start_time),
+                    start_ntp_time,
                     delta_frames: stream.delta_frames,
                     trak_timescale,
                 },

mux/fmp4/src/fmp4mux/mod.rs

@@ -252,6 +252,13 @@ pub(crate) struct FragmentHeaderStream {
     ///
     /// `None` if this stream has no buffers in this fragment.
     start_time: Option<gst::ClockTime>,
+
+    /// Start NTP time of this fragment
+    ///
+    /// This is in nanoseconds since epoch and is used for writing the prft box if present.
+    ///
+    /// Only the first track is ever used.
+    start_ntp_time: Option<gst::ClockTime>,
 }
 
 #[derive(Debug, Copy, Clone)]