From 51987b6f1f9f42475582f1b819f5e15422954b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 9 Apr 2025 14:19:02 +0300 Subject: [PATCH] fmp4mux: Write prft box into each fragment with the NTP / media time mapping if possible The NTP time is based on the reference timestamp meta of the buffer that has the start media time of the fragment. Part-of: --- mux/fmp4/src/fmp4mux/boxes.rs | 43 ++++++++++++++++++++++++++++++++--- mux/fmp4/src/fmp4mux/imp.rs | 34 ++++++++++++++++++++++----- mux/fmp4/src/fmp4mux/mod.rs | 7 ++++++ 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 2dd13efce..19c4fd1a1 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -1800,7 +1800,18 @@ pub(super) fn create_fmp4_fragment_header( })?; } - let styp_len = v.len(); + // Write prft for the first stream if we can + if let Some(stream) = cfg.streams.first() { + if let Some((start_time, start_ntp_time)) = + Option::zip(stream.start_time, stream.start_ntp_time) + { + write_full_box(&mut v, b"prft", FULL_BOX_VERSION_1, 8, |v| { + write_prft(v, &cfg, 0, stream, start_time, start_ntp_time) + })?; + } + } + + let moof_pos = v.len(); let data_offset_offsets = write_box(&mut v, b"moof", |v| write_moof(v, &cfg))?; @@ -1818,7 +1829,7 @@ pub(super) fn create_fmp4_fragment_header( v.extend((size + 16).to_be_bytes()); } - let data_offset = v.len() - styp_len; + let data_offset = v.len() - moof_pos; for data_offset_offset in data_offset_offsets { let val = u32::from_be_bytes(v[data_offset_offset..][..4].try_into()?) .checked_add(u32::try_from(data_offset)?) @@ -1826,7 +1837,33 @@ pub(super) fn create_fmp4_fragment_header( v[data_offset_offset..][..4].copy_from_slice(&val.to_be_bytes()); } - Ok((gst::Buffer::from_mut_slice(v), styp_len as u64)) + Ok((gst::Buffer::from_mut_slice(v), moof_pos as u64)) +} + +fn write_prft( + v: &mut Vec, + _cfg: &super::FragmentHeaderConfiguration, + idx: usize, + stream: &super::FragmentHeaderStream, + start_time: gst::ClockTime, + start_ntp_time: gst::ClockTime, +) -> Result<(), Error> { + // Reference track ID + v.extend((idx as u32 + 1).to_be_bytes()); + // NTP timestamp + let start_ntp_time = start_ntp_time + .nseconds() + .mul_div_floor(1u64 << 32, gst::ClockTime::SECOND.nseconds()) + .unwrap(); + v.extend(start_ntp_time.to_be_bytes()); + // Media time + let timescale = fragment_header_stream_to_timescale(stream); + let media_time = start_time + .mul_div_floor(timescale as u64, gst::ClockTime::SECOND.nseconds()) + .unwrap(); + v.extend(media_time.to_be_bytes()); + + Ok(()) } fn write_moof( diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index 2e9143e55..ca9ae1ffb 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -2437,6 +2437,10 @@ impl FMP4Mux { Option, // End DTS Option, + // Start time (either matches start_dts if required or earliest-pts) + gst::ClockTime, + // Start NTP time (either matches start_dts if required or earliest_pts) + Option, )>, gst::FlowError, > { @@ -2456,6 +2460,7 @@ impl FMP4Mux { let mut earliest_pts_position = None; let mut start_dts = None; let mut start_dts_position = None; + let mut start_ntp_time = None; let mut gop_buffers = gop_buffers.into_iter(); while let Some(buffer) = gop_buffers.next() { @@ -2472,6 +2477,11 @@ impl FMP4Mux { if earliest_pts.is_none_or(|earliest_pts| buffer.pts < earliest_pts) { earliest_pts = Some(buffer.pts); + if !stream.delta_frames.requires_dts() { + let utc_time = get_utc_time_from_buffer(&buffer.buffer) + .and_then(|t| t.checked_add(NTP_UNIX_OFFSET.seconds())); + start_ntp_time = utc_time; + } } if earliest_pts_position.is_none_or(|earliest_pts_position| { buffer.buffer.pts().unwrap() < earliest_pts_position @@ -2480,6 +2490,11 @@ impl FMP4Mux { } if stream.delta_frames.requires_dts() && start_dts.is_none() { start_dts = Some(buffer.dts.unwrap()); + if stream.delta_frames.requires_dts() { + let utc_time = get_utc_time_from_buffer(&buffer.buffer) + .and_then(|t| t.checked_add(NTP_UNIX_OFFSET.seconds())); + start_ntp_time = utc_time; + } } if stream.delta_frames.requires_dts() && start_dts_position.is_none() { start_dts_position = Some(buffer.buffer.dts().unwrap()); @@ -2557,6 +2572,12 @@ impl FMP4Mux { let start_dts = start_dts; let start_dts_position = start_dts_position; + let start_time = if !stream.delta_frames.requires_dts() { + earliest_pts + } else { + start_dts.unwrap() + }; + Ok(Some(( buffers, earliest_pts, @@ -2565,6 +2586,8 @@ impl FMP4Mux { start_dts, start_dts_position, end_dts, + start_time, + start_ntp_time, ))) } @@ -2760,6 +2783,7 @@ impl FMP4Mux { super::FragmentHeaderStream { caps: stream.caps.clone(), start_time: None, + start_ntp_time: None, delta_frames: stream.delta_frames, trak_timescale, }, @@ -2802,6 +2826,8 @@ impl FMP4Mux { start_dts, start_dts_position, _end_dts, + start_time, + start_ntp_time, ) = match buffers { Some(res) => res, None => { @@ -2811,6 +2837,7 @@ impl FMP4Mux { super::FragmentHeaderStream { caps: stream.caps.clone(), start_time: None, + start_ntp_time: None, delta_frames: stream.delta_frames, trak_timescale, }, @@ -2831,12 +2858,6 @@ impl FMP4Mux { stream.dts_offset.display(), ); - let start_time = if !stream.delta_frames.requires_dts() { - earliest_pts - } else { - start_dts.unwrap() - }; - if min_earliest_pts.opt_gt(earliest_pts).unwrap_or(true) { min_earliest_pts = Some(earliest_pts); } @@ -2859,6 +2880,7 @@ impl FMP4Mux { super::FragmentHeaderStream { caps: stream.caps.clone(), start_time: Some(start_time), + start_ntp_time, delta_frames: stream.delta_frames, trak_timescale, }, diff --git a/mux/fmp4/src/fmp4mux/mod.rs b/mux/fmp4/src/fmp4mux/mod.rs index 3d658625b..45d7c425d 100644 --- a/mux/fmp4/src/fmp4mux/mod.rs +++ b/mux/fmp4/src/fmp4mux/mod.rs @@ -252,6 +252,13 @@ pub(crate) struct FragmentHeaderStream { /// /// `None` if this stream has no buffers in this fragment. start_time: Option, + + /// Start NTP time of this fragment + /// + /// This is in nanoseconds since epoch and is used for writing the prft box if present. + /// + /// Only the first track is ever used. + start_ntp_time: Option, } #[derive(Debug, Copy, Clone)]