From 06939540a19fb8a89674a0fc8cc40a253b26caf4 Mon Sep 17 00:00:00 2001 From: Brad Hards Date: Sat, 7 Jun 2025 09:56:15 +1000 Subject: [PATCH] mp4mux: add TAI timestamp muxing This is an implementation of the TAI timestamp functionality described in ISO/IEC 23001-17 Amendment 1 Section 8.1.2 and 8.1.3. Part-of: --- docs/plugins/gst_plugins_cache.json | 12 ++ mux/mp4/Cargo.toml | 1 + mux/mp4/src/mp4mux/boxes.rs | 103 ++++++++++++ mux/mp4/src/mp4mux/imp.rs | 200 ++++++++++++++++++++++- mux/mp4/src/mp4mux/mod.rs | 17 ++ mux/mp4/tests/tests.rs | 243 +++++++++++++++++++++++++++- 6 files changed, 572 insertions(+), 4 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 564681874..b40958ac6 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -5514,6 +5514,18 @@ "readable": true, "type": "guint", "writable": true + }, + "tai-precision-timestamps": { + "blurb": "Whether to encode ISO/IEC 23001-17 TAI timestamps as auxiliary data", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true } } }, diff --git a/mux/mp4/Cargo.toml b/mux/mp4/Cargo.toml index 483c94670..74f51d9a2 100644 --- a/mux/mp4/Cargo.toml +++ b/mux/mp4/Cargo.toml @@ -37,6 +37,7 @@ default = [] static = [] capi = [] doc = [] +v1_28 = ["gst/v1_28"] [package.metadata.capi] min_version = "0.9.21" diff --git a/mux/mp4/src/mp4mux/boxes.rs b/mux/mp4/src/mp4mux/boxes.rs index e6862388c..83e37d7a3 100644 --- a/mux/mp4/src/mp4mux/boxes.rs +++ b/mux/mp4/src/mp4mux/boxes.rs @@ -14,6 +14,8 @@ use std::str::FromStr; use std::sync::LazyLock; use std::{collections::BTreeMap, convert::TryFrom}; +use crate::mp4mux::{AuxiliaryInformation, AuxiliaryInformationEntry}; + fn write_box) -> Result>( vec: &mut Vec, fourcc: impl std::borrow::Borrow<[u8; 4]>, @@ -656,6 +658,107 @@ fn write_stbl( })?; } + for auxiliary_information in &stream.auxiliary_info { + if !auxiliary_information.entries.is_empty() { + write_full_saiz(v, auxiliary_information)?; + write_full_saio(v, auxiliary_information)?; + } + } + + Ok(()) +} + +fn write_full_saiz( + v: &mut Vec, + auxiliary_information: &AuxiliaryInformation, +) -> Result<(), Error> { + if let Some(aux_info_type) = auxiliary_information.aux_info_type { + write_full_box(v, b"saiz", FULL_BOX_VERSION_0, 1, |v| { + v.extend(aux_info_type); + v.extend(auxiliary_information.aux_info_type_parameter.to_be_bytes()); + write_saiz_entries(v, &auxiliary_information.entries) + })?; + } else { + write_full_box(v, b"saiz", FULL_BOX_VERSION_0, 0, |v| { + write_saiz_entries(v, &auxiliary_information.entries) + })?; + } + Ok(()) +} + +fn write_saiz_entries(v: &mut Vec, entries: &[AuxiliaryInformationEntry]) -> Result<(), Error> { + assert!(!entries.is_empty()); + let first_entry_length = entries[0].entry_len; + if entries[1..] + .iter() + .all(|entry| entry.entry_len == first_entry_length) + { + v.extend(first_entry_length.to_be_bytes()); + v.extend((entries.len() as u32).to_be_bytes()); + } else { + v.extend(0u8.to_be_bytes()); + v.extend((entries.len() as u32).to_be_bytes()); + for entry in entries { + v.extend(entry.entry_len.to_be_bytes()); + } + } + Ok(()) +} + +fn write_full_saio( + v: &mut Vec, + auxiliary_information: &AuxiliaryInformation, +) -> Result<(), Error> { + let version = if auxiliary_information + .entries + .iter() + .any(|entry| entry.entry_offset > (u32::MAX as u64)) + { + FULL_BOX_VERSION_1 + } else { + FULL_BOX_VERSION_0 + }; + if let Some(aux_info_type) = auxiliary_information.aux_info_type { + write_full_box(v, b"saio", version, 1, |v| { + v.extend(aux_info_type); + v.extend(auxiliary_information.aux_info_type_parameter.to_be_bytes()); + if version == FULL_BOX_VERSION_0 { + write_saio_entries_v0(v, &auxiliary_information.entries) + } else { + write_saio_entries_v1(v, &auxiliary_information.entries) + } + })?; + } else { + write_full_box(v, b"saio", version, 0, |v| { + if version == FULL_BOX_VERSION_0 { + write_saio_entries_v0(v, &auxiliary_information.entries) + } else { + write_saio_entries_v1(v, &auxiliary_information.entries) + } + })?; + } + Ok(()) +} + +fn write_saio_entries_v0( + v: &mut Vec, + entries: &[AuxiliaryInformationEntry], +) -> Result<(), Error> { + v.extend((entries.len() as u32).to_be_bytes()); + for entry in entries { + v.extend((entry.entry_offset as u32).to_be_bytes()); + } + Ok(()) +} + +fn write_saio_entries_v1( + v: &mut Vec, + entries: &[AuxiliaryInformationEntry], +) -> Result<(), Error> { + v.extend((entries.len() as u32).to_be_bytes()); + for entry in entries { + v.extend(entry.entry_offset.to_be_bytes()); + } Ok(()) } diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs index c9117a3c4..7881a4cdc 100644 --- a/mux/mp4/src/mp4mux/imp.rs +++ b/mux/mp4/src/mp4mux/imp.rs @@ -20,6 +20,8 @@ use std::str::FromStr; use std::sync::Mutex; use crate::mp4mux::obu::read_seq_header_obu_bytes; +use crate::mp4mux::AuxiliaryInformation; +use crate::mp4mux::AuxiliaryInformationEntry; use crate::mp4mux::TaicClockType; use std::sync::LazyLock; @@ -41,6 +43,11 @@ static NTP_CAPS: LazyLock = static UNIX_CAPS: LazyLock = LazyLock::new(|| gst::Caps::builder("timestamp/x-unix").build()); +#[cfg(feature = "v1_28")] +/// Reference timestamp meta caps for TAI timestamps with 1958-01-01 epoch. +static TAI1958_CAPS: LazyLock = + LazyLock::new(|| gst::Caps::builder("timestamp/x-tai1958").build()); + /// Returns the UTC time of the buffer in the UNIX epoch. fn get_utc_time_from_buffer(buffer: &gst::BufferRef) -> Option { buffer @@ -127,6 +134,7 @@ struct Settings { interleave_time: Option, movie_timescale: u32, extra_brands: Vec<[u8; 4]>, + with_precision_timestamps: bool, } impl Default for Settings { @@ -136,6 +144,7 @@ impl Default for Settings { interleave_time: DEFAULT_INTERLEAVE_TIME, movie_timescale: 0, extra_brands: Vec::new(), + with_precision_timestamps: false, } } } @@ -164,7 +173,14 @@ struct PendingBuffer { composition_time_offset: Option, duration: Option, } +#[derive(Debug)] +struct PendingAuxInfoEntry { + aux_info_type: Option<[u8; 4]>, + aux_info_type_parameter: u32, + data: Vec, +} +#[derive(Debug)] struct Stream { /// Sink pad for this stream. sinkpad: super::MP4MuxPad, @@ -222,6 +238,11 @@ struct Stream { /// TAI precision clock information tai_clock_info: Option, + + /// The auxiliary information (saio/saiz) for the stream + aux_info: Vec, + /// auxiliary information to be written after the chunk is finished + pending_aux_info_data: VecDeque, } impl Stream { @@ -371,6 +392,10 @@ struct State { /// Size of the `mdat` as written so far. mdat_size: u64, + + #[cfg(feature = "v1_28")] + /// The last TAI timestamp value, in nanoseconds after epoch + last_tai_timestamp: u64, } #[derive(Default)] @@ -957,6 +982,7 @@ impl MP4Mux { &self, settings: &Settings, state: &mut State, + buffers: &mut gst::BufferListRef, ) -> Result, gst::FlowError> { if let Some(current_stream_idx) = state.current_stream_idx { // If a stream was previously selected, check if another buffer from @@ -991,6 +1017,10 @@ impl MP4Mux { ); return Ok(Some(current_stream_idx)); } + let num_bytes_added = + self.flush_aux_info(buffers, stream, state.current_offset); + state.current_offset += num_bytes_added; + state.mdat_size += num_bytes_added; state.current_stream_idx = None; gst::debug!( @@ -1012,6 +1042,10 @@ impl MP4Mux { obj = stream.sinkpad, "Stream is EOS, switching to next stream" ); + let num_bytes_added = + self.flush_aux_info(buffers, stream, state.current_offset); + state.current_offset += num_bytes_added; + state.mdat_size += num_bytes_added; state.current_stream_idx = None; } Err(err) => { @@ -1101,6 +1135,56 @@ impl MP4Mux { } } + fn flush_aux_info( + &self, + buffers: &mut gst::BufferListRef, + stream: &mut Stream, + initial_offset: u64, + ) -> u64 { + gst::debug!( + CAT, + obj = stream.sinkpad, + "Flushing {} pending entries of auxiliary information from stream {} to mdat at end of current chunk", + stream.pending_aux_info_data.len(), + stream.sinkpad.name(), + ); + let mut num_bytes_added = 0u64; + while let Some(pending_aux_info) = stream.pending_aux_info_data.pop_front() { + // We don't handle the case where the aux_info_type is None - no idea what the semantics of that would be + assert!(pending_aux_info.aux_info_type.is_some()); + let maybe_index = stream.aux_info.iter().position(|aux_info| { + (aux_info.aux_info_type == pending_aux_info.aux_info_type) + && (aux_info.aux_info_type_parameter + == pending_aux_info.aux_info_type_parameter) + }); + let index: usize = match maybe_index { + Some(index) => index, + None => { + // We did not already have a matching entry in the stream, add it + let aux_info = AuxiliaryInformation { + aux_info_type: pending_aux_info.aux_info_type, + aux_info_type_parameter: pending_aux_info.aux_info_type_parameter, + entries: Vec::new(), + }; + stream.aux_info.push(aux_info); + stream.aux_info.len() - 1 + } + }; + let pending_aux_info_data = gst::Buffer::from_slice(pending_aux_info.data); + assert!(pending_aux_info_data.size() <= u8::MAX as usize); + let entry_size = pending_aux_info_data.size() as u8; + buffers.add(pending_aux_info_data); + stream.aux_info[index] + .entries + .push(AuxiliaryInformationEntry { + entry_offset: initial_offset + num_bytes_added, + entry_len: entry_size, + }); + num_bytes_added += entry_size as u64; + } + num_bytes_added + } + fn drain_buffers( &self, settings: &Settings, @@ -1108,7 +1192,7 @@ impl MP4Mux { buffers: &mut gst::BufferListRef, ) -> Result<(), gst::FlowError> { // Now we can start handling buffers - while let Some(idx) = self.find_earliest_stream(settings, state)? { + while let Some(idx) = self.find_earliest_stream(settings, state, buffers)? { let stream = &mut state.streams[idx]; let buffer = stream.pending_buffer.take().unwrap(); @@ -1173,7 +1257,97 @@ impl MP4Mux { } let mut buffer = buffer.buffer; - + #[cfg(feature = "v1_28")] + { + if settings.with_precision_timestamps { + // Builds a TAITimestampPacket structure as defined in ISO/IEC 23001-17 Amendment 1, Section 8.1.2 + // That will be written out as `stai` aux info per Section 8.1.3. + // See ISO/IEC 14496-12 Section 8.7.8 and 8.7.9 for more on aux info. + if let Some(meta) = buffer + .iter_meta::() + .find(|m| m.reference().can_intersect(&TAI1958_CAPS) && m.info().is_some()) + { + gst::trace!( + CAT, + imp = self, + "got TAI ReferenceTimestampMeta on the buffer" + ); + let mut timestamp_packet = Vec::::with_capacity(9); + timestamp_packet.extend(meta.timestamp().nseconds().to_be_bytes()); + state.last_tai_timestamp = meta.timestamp().nseconds(); + let iso23001_17_timestamp_info = meta.info().unwrap(); // checked in filter + let mut timestamp_packet_flags = 0u8; + if let Ok(synced) = + iso23001_17_timestamp_info.get::("synchronization-state") + { + gst::trace!( + CAT, + imp = self, + "synchronized to atomic source: {:?}", + synced + ); + if synced { + timestamp_packet_flags |= 0x80u8; + } + } else { + gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected synchronisation state, assuming not synchronised"); + } + if let Ok(generation_failure) = + iso23001_17_timestamp_info.get::("timestamp-generation-failure") + { + gst::trace!( + CAT, + imp = self, + "timestamp generation failure: {:?}", + generation_failure + ); + if generation_failure { + timestamp_packet_flags |= 0x40u8; + } + } else if meta.timestamp().nseconds() > state.last_tai_timestamp { + gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected generation failure flag, timestamp looks OK, assuming OK"); + } else { + gst::warning!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected generation failure flag and unexpected timestamp value, assuming generation failure"); + timestamp_packet_flags |= 0x40u8; + } + if let Ok(timestamp_is_modified) = + iso23001_17_timestamp_info.get::("timestamp-is-modified") + { + gst::trace!( + CAT, + imp = self, + "timestamp is modified: {:?}", + timestamp_is_modified + ); + if timestamp_is_modified { + timestamp_packet_flags |= 0x20u8; + } + } else { + gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected modification state value, assuming not modified"); + } + timestamp_packet.extend(timestamp_packet_flags.to_be_bytes()); + stream.pending_aux_info_data.push_back(PendingAuxInfoEntry { + aux_info_type: Some(*b"stai"), + aux_info_type_parameter: 0, + data: timestamp_packet, + }); + } else { + // generate a failure packet, because we always need aux info for a sample + let mut timestamp_packet = Vec::::with_capacity(9); + // The timestamp must monotonically increase + let timestamp = state.last_tai_timestamp + 1; + timestamp_packet.extend(timestamp.to_be_bytes()); + state.last_tai_timestamp = timestamp; + let flags = 0x40u8; // not sync'd | generation failure | not modified, + timestamp_packet.extend(flags.to_be_bytes()); + stream.pending_aux_info_data.push_back(PendingAuxInfoEntry { + aux_info_type: Some(*b"stai"), + aux_info_type_parameter: 0, + data: timestamp_packet, + }); + } + } + } stream.queued_chunk_time += duration; stream.queued_chunk_bytes += buffer.size() as u64; @@ -1469,6 +1643,8 @@ impl MP4Mux { max_bitrate, avg_bitrate, tai_clock_info, + aux_info: Vec::new(), + pending_aux_info_data: VecDeque::new(), }); } @@ -1545,6 +1721,12 @@ impl ObjectImpl for MP4Mux { .blurb("Comma-separated list of 4-character brand codes (e.g. duke,sook)") .mutable_ready() .build(), + glib::ParamSpecBoolean::builder("tai-precision-timestamps") + .nick("Precision Timestamps") + .blurb("Whether to encode ISO/IEC 23001-17 TAI timestamps as auxiliary data") + .default_value(false) + .mutable_ready() + .build(), ] }); @@ -1600,6 +1782,11 @@ impl ObjectImpl for MP4Mux { } } + "tai-precision-timestamps" => { + let mut settings = self.settings.lock().unwrap(); + settings.with_precision_timestamps = value.get().expect("type checked upstream"); + } + _ => unimplemented!(), } } @@ -1633,6 +1820,11 @@ impl ObjectImpl for MP4Mux { Some(brands_str).to_value() } + "tai-precision-timestamps" => { + let settings = self.settings.lock().unwrap(); + settings.with_precision_timestamps.to_value() + } + _ => unimplemented!(), } } @@ -2005,6 +2197,9 @@ impl AggregatorImpl for MP4Mux { _ => {} } } + if settings.with_precision_timestamps { + compatible_brands.insert(*b"iso6"); // required for saiz/saio support + } } if have_image_sequence && have_only_image_sequence { major_brand = b"msf1"; @@ -2091,6 +2286,7 @@ impl AggregatorImpl for MP4Mux { avg_bitrate: stream.avg_bitrate, chunks: stream.chunks, tai_clock_info: stream.tai_clock_info, + auxiliary_info: stream.aux_info, }); } diff --git a/mux/mp4/src/mp4mux/mod.rs b/mux/mp4/src/mp4mux/mod.rs index 826832037..897227f7d 100644 --- a/mux/mp4/src/mp4mux/mod.rs +++ b/mux/mp4/src/mp4mux/mod.rs @@ -241,6 +241,20 @@ pub(crate) struct TaiClockInfo { clock_type: TaicClockType, } +// Data for auxiliary information, as used for per-sample timestamps and for protection schemes +#[derive(Clone, Debug, Default)] +struct AuxiliaryInformationEntry { + entry_offset: u64, + entry_len: u8, +} + +#[derive(Clone, Debug, Default)] +struct AuxiliaryInformation { + aux_info_type: Option<[u8; 4]>, + aux_info_type_parameter: u32, + entries: Vec, +} + #[derive(Debug)] pub(crate) struct Stream { /// Caps of this stream @@ -284,6 +298,9 @@ pub(crate) struct Stream { /// TAI Clock information (ISO/IEC 23001-17 Amd 1) tai_clock_info: Option, + + /// Sample auxiliary information (ISO/IEC 14496-12:2022 Section 8.7.8 and 8.7.9) + auxiliary_info: Vec, } #[derive(Debug)] diff --git a/mux/mp4/tests/tests.rs b/mux/mp4/tests/tests.rs index 1bf979688..b9e7081ee 100644 --- a/mux/mp4/tests/tests.rs +++ b/mux/mp4/tests/tests.rs @@ -8,7 +8,11 @@ // use std::{fs::File, path::Path}; +#[cfg(feature = "v1_28")] +use std::{io::Seek as _, sync::LazyLock}; +#[cfg(feature = "v1_28")] +use gst::{ClockTime, ReferenceTimestampMeta}; use gst_pbutils::prelude::*; use mp4_atom::{Atom, ReadAtom as _, ReadFrom as _}; use tempfile::tempdir; @@ -23,6 +27,10 @@ fn init() { }); } +#[cfg(feature = "v1_28")] +static TAI1958_CAPS: LazyLock = + LazyLock::new(|| gst::Caps::builder("timestamp/x-tai1958").build()); + struct Pipeline(gst::Pipeline); impl std::ops::Deref for Pipeline { type Target = gst::Pipeline; @@ -73,6 +81,7 @@ struct ExpectedConfiguration { has_taic: bool, taic_time_uncertainty: u64, taic_clock_type: u8, + num_tai_timestamps: i32, } fn test_basic_with(video_enc: &str, audio_enc: &str, cb: impl FnOnce(&Path)) { @@ -290,6 +299,7 @@ fn test_expected_uncompressed_output(location: &Path, width: u32, height: u32) { has_taic: false, taic_clock_type: 0, // only if has_taic taic_time_uncertainty: 0, // only if has_taic + num_tai_timestamps: 0, }, ); } @@ -626,6 +636,7 @@ fn test_expected_image_sequence_output(location: &Path, width: u32, height: u32) has_taic: false, taic_time_uncertainty: 0, // only if has_taic taic_clock_type: 0, // only if has_taic + num_tai_timestamps: 0, }, ); } @@ -695,6 +706,7 @@ fn test_audio_only_output(location: &Path) { has_taic: false, taic_time_uncertainty: 0, // only if has_taic taic_clock_type: 0, // only if has_taic + num_tai_timestamps: 0, }, ); } @@ -811,8 +823,8 @@ fn check_stbl_sanity(stbl: &mp4_atom::Stbl, expected_config: &ExpectedConfigurat } else { assert!(stbl.ctts.is_none()); } - assert!(stbl.saio.is_none()); - assert!(stbl.saiz.is_none()); + check_saio_sanity(&stbl.saio, expected_config); + check_saiz_sanity(&stbl.saiz, expected_config); check_stco_sanity(&stbl.stco); check_stsc_sanity(&stbl.stsc); check_stsd_sanity(&stbl.stsd, expected_config); @@ -828,6 +840,49 @@ fn check_stbl_sanity(stbl: &mp4_atom::Stbl, expected_config: &ExpectedConfigurat // TODO: check consistency between sample sizes and chunk / sample offsets } +fn check_saio_sanity(maybe_saio: &Option, expected_config: &ExpectedConfiguration) { + if expected_config.num_tai_timestamps == 0 { + assert!(maybe_saio.is_none()); + } else { + assert!(maybe_saio.is_some()); + let saio = maybe_saio.as_ref().unwrap(); + assert!(saio.aux_info.is_some()); + assert_eq!( + saio.aux_info.as_ref().unwrap().aux_info_type, + b"stai".into() + ); + assert_eq!(saio.aux_info.as_ref().unwrap().aux_info_type_parameter, 0); + assert_eq!( + saio.offsets.len(), + expected_config.num_tai_timestamps as usize + ); + let mut previous_offset = 0u64; + for offset in &saio.offsets { + // We check that the byte offsets are increasing + // This is different to checking that the timestamps are increasing + assert!(*offset > previous_offset); + previous_offset = *offset; + } + } +} + +fn check_saiz_sanity(maybe_saiz: &Option, expected_config: &ExpectedConfiguration) { + if expected_config.num_tai_timestamps == 0 { + assert!(maybe_saiz.is_none()); + } else { + assert!(maybe_saiz.is_some()); + let saiz = maybe_saiz.as_ref().unwrap(); + assert!(saiz.aux_info.is_some()); + assert_eq!( + saiz.aux_info.as_ref().unwrap().aux_info_type, + b"stai".into() + ); + assert_eq!(saiz.aux_info.as_ref().unwrap().aux_info_type_parameter, 0); + assert_eq!(saiz.default_sample_info_size, 9); + assert_eq!(saiz.sample_count, expected_config.num_tai_timestamps as u32); + } +} + fn check_stco_sanity(maybe_stco: &Option) { assert!(maybe_stco .as_ref() @@ -1062,10 +1117,179 @@ fn test_taic_encode(video_enc: &str) { has_taic: true, taic_time_uncertainty: 100_000, taic_clock_type: 2, + num_tai_timestamps: 0, }, ); } +#[cfg(feature = "v1_28")] +fn test_taic_stai_encode(video_enc: &str, enabled: bool) { + let filename = format!("taic_{video_enc}.mp4").to_string(); + let temp_dir = tempdir().unwrap(); + let temp_file_path = temp_dir.path().join(filename); + let location = temp_file_path.as_path(); + let number_of_frames = 12; + let pipeline = gst::Pipeline::builder() + .name(format!("stai-{video_enc}")) + .build(); + let videotestsrc = gst::ElementFactory::make("videotestsrc") + .property("num-buffers", number_of_frames) + .property("is-live", true) + .build() + .unwrap(); + let encoder = gst::ElementFactory::make(video_enc) + .name("video encoder") + .property("bframes", 0u32) + .build() + .unwrap(); + let taginject = gst::ElementFactory::make("taginject") + .property_from_str("tags", "precision-clock-type=can-sync-to-TAI,precision-clock-time-uncertainty-nanoseconds=100000") + .property_from_str("scope", "stream") + .build().unwrap(); + let mux = gst::ElementFactory::make("isomp4mux") + .property("tai-precision-timestamps", enabled) + .build() + .unwrap(); + let sink = gst::ElementFactory::make("filesink") + .property("location", location) + .build() + .unwrap(); + pipeline + .add_many([&videotestsrc, &encoder, &taginject, &mux, &sink]) + .unwrap(); + + gst::Element::link_many([&videotestsrc, &encoder, &taginject, &mux, &sink]).unwrap(); + + let tai_nanos_initial_offset: u64 = 100_000_000_000; + let tai_nanos_per_frame_step = 20_000_000; // 20 milliseconds. + let tai_nanos = std::sync::atomic::AtomicU64::new(tai_nanos_initial_offset); + videotestsrc.static_pad("src").unwrap().add_probe( + gst::PadProbeType::BUFFER, + move |_pad, info| { + if let Some(buffer) = info.buffer_mut() { + let timestamp: ClockTime = + ClockTime::from_nseconds(tai_nanos.load(std::sync::atomic::Ordering::Acquire)); + let mut meta = + ReferenceTimestampMeta::add(buffer.make_mut(), &TAI1958_CAPS, timestamp, None); + let s = gst::Structure::builder("iso23001-17-timestamp") + .field("synchronization-state", true) + .field("timestamp-generation-failure", false) + .field("timestamp-is-modified", false) + .build(); + meta.set_info(s); + tai_nanos.fetch_add( + tai_nanos_per_frame_step, + std::sync::atomic::Ordering::AcqRel, + ); + } + gst::PadProbeReturn::Ok + }, + ); + + pipeline + .set_state(gst::State::Playing) + .expect("Unable to set the pipeline to the `Playing` state"); + for msg in pipeline.bus().unwrap().iter_timed(gst::ClockTime::NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::Eos(..) => break, + MessageView::Error(err) => { + panic!( + "Error from {:?}: {} ({:?})", + err.src().map(|s| s.path_string()), + err.error(), + err.debug() + ); + } + _ => (), + } + } + pipeline + .set_state(gst::State::Null) + .expect("Unable to set the pipeline to the `Null` state"); + + check_generic_single_trak_file_structure( + location, + b"iso4".into(), + 0, + if enabled { + vec![ + b"isom".into(), + b"mp41".into(), + b"mp42".into(), + b"iso6".into(), + ] + } else { + vec![b"isom".into(), b"mp41".into(), b"mp42".into()] + }, + ExpectedConfiguration { + is_audio: false, + width: 320, + height: 240, + has_ctts: false, + has_stss: true, + has_taic: true, + taic_time_uncertainty: 100_000, + taic_clock_type: 2, + num_tai_timestamps: if enabled { number_of_frames } else { 0 }, + }, + ); + if enabled { + let mut input = File::open(location).unwrap(); + let mut mdat_data: Option> = None; + let mut mdat_offset: u64 = 0; + while let Ok(header) = mp4_atom::Header::read_from(&mut input) { + match header.kind { + mp4_atom::Moov::KIND => { + let moov = mp4_atom::Moov::read_atom(&header, &mut input).unwrap(); + let stbl = &moov.trak.first().unwrap().mdia.minf.stbl; + let saio = stbl.saio.as_ref().unwrap(); + let saiz = stbl.saiz.as_ref().unwrap(); + if mdat_data.is_some() { + assert_eq!( + saio.aux_info.as_ref().unwrap().aux_info_type, + b"stai".into() + ); + assert_eq!(saio.aux_info.as_ref().unwrap().aux_info_type_parameter, 0); + assert_eq!( + saiz.aux_info.as_ref().unwrap().aux_info_type, + b"stai".into() + ); + assert_eq!(saiz.aux_info.as_ref().unwrap().aux_info_type_parameter, 0); + for i in 0..saio.offsets.len() { + let offset = saio.offsets[i]; + let len = saiz.default_sample_info_size as u64; + let offset_into_mdat_start = (offset - mdat_offset) as usize; + let offset_into_mdat_end = offset_into_mdat_start + len as usize; + let vec = &mdat_data.as_ref().unwrap() + [offset_into_mdat_start..offset_into_mdat_end] + .to_vec(); + assert_eq!(vec.len(), 9); + let mut timestamp_bytes: [u8; 8] = [0; 8]; + timestamp_bytes.copy_from_slice(&vec.as_slice()[0..8]); + let timestamp = u64::from_be_bytes(timestamp_bytes); + assert_eq!( + timestamp, + tai_nanos_initial_offset + (i as u64) * tai_nanos_per_frame_step + ); + assert_eq!(vec[8], 0x80); + } + } else { + panic!("mdat should not be none"); + } + } + mp4_atom::Mdat::KIND => { + mdat_offset = input.stream_position().unwrap(); + let mdat = mp4_atom::Mdat::read_atom(&header, &mut input).unwrap(); + mdat_data = Some(mdat.data); + } + _ => {} + } + } + } +} + fn test_taic_encode_cannot_sync(video_enc: &str) { let filename = format!("taic_{video_enc}_cannot_sync.mp4").to_string(); let temp_dir = tempdir().unwrap(); @@ -1112,6 +1336,7 @@ fn test_taic_encode_cannot_sync(video_enc: &str) { has_taic: true, taic_time_uncertainty: 0xFFFF_FFFF_FFFF_FFFF, taic_clock_type: 1, + num_tai_timestamps: 0, }, ); } @@ -1122,6 +1347,20 @@ fn test_taic_x264() { test_taic_encode("x264enc"); } +#[test] +#[cfg(feature = "v1_28")] +fn test_taic_stai_x264() { + init(); + test_taic_stai_encode("x264enc", true); +} + +#[test] +#[cfg(feature = "v1_28")] +fn test_taic_stai_x264_not_enabled() { + init(); + test_taic_stai_encode("x264enc", false); +} + #[test] fn test_taic_x264_no_sync() { init();