mirror of
https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git
synced 2025-09-02 01:33:47 +00:00
mp4mux: add TAI timestamp muxing
This is an implementation of the TAI timestamp functionality described in ISO/IEC 23001-17 Amendment 1 Section 8.1.2 and 8.1.3. Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/2280>
This commit is contained in:
Brad Hards
parent
efaab53ab3
commit
06939540a1
6 changed files with 572 additions and 4 deletions
|
|
@ -5514,6 +5514,18 @@
|
|||
"readable": true,
|
||||
"type": "guint",
|
||||
"writable": true
|
||||
},
|
||||
"tai-precision-timestamps": {
|
||||
"blurb": "Whether to encode ISO/IEC 23001-17 TAI timestamps as auxiliary data",
|
||||
"conditionally-available": false,
|
||||
"construct": false,
|
||||
"construct-only": false,
|
||||
"controllable": false,
|
||||
"default": "false",
|
||||
"mutable": "ready",
|
||||
"readable": true,
|
||||
"type": "gboolean",
|
||||
"writable": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ default = []
|
|||
static = []
|
||||
capi = []
|
||||
doc = []
|
||||
v1_28 = ["gst/v1_28"]
|
||||
|
||||
[package.metadata.capi]
|
||||
min_version = "0.9.21"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ use std::str::FromStr;
|
|||
use std::sync::LazyLock;
|
||||
use std::{collections::BTreeMap, convert::TryFrom};
|
||||
|
||||
use crate::mp4mux::{AuxiliaryInformation, AuxiliaryInformationEntry};
|
||||
|
||||
fn write_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>(
|
||||
vec: &mut Vec<u8>,
|
||||
fourcc: impl std::borrow::Borrow<[u8; 4]>,
|
||||
|
|
@ -656,6 +658,107 @@ fn write_stbl(
|
|||
})?;
|
||||
}
|
||||
|
||||
for auxiliary_information in &stream.auxiliary_info {
|
||||
if !auxiliary_information.entries.is_empty() {
|
||||
write_full_saiz(v, auxiliary_information)?;
|
||||
write_full_saio(v, auxiliary_information)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_full_saiz(
|
||||
v: &mut Vec<u8>,
|
||||
auxiliary_information: &AuxiliaryInformation,
|
||||
) -> Result<(), Error> {
|
||||
if let Some(aux_info_type) = auxiliary_information.aux_info_type {
|
||||
write_full_box(v, b"saiz", FULL_BOX_VERSION_0, 1, |v| {
|
||||
v.extend(aux_info_type);
|
||||
v.extend(auxiliary_information.aux_info_type_parameter.to_be_bytes());
|
||||
write_saiz_entries(v, &auxiliary_information.entries)
|
||||
})?;
|
||||
} else {
|
||||
write_full_box(v, b"saiz", FULL_BOX_VERSION_0, 0, |v| {
|
||||
write_saiz_entries(v, &auxiliary_information.entries)
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_saiz_entries(v: &mut Vec<u8>, entries: &[AuxiliaryInformationEntry]) -> Result<(), Error> {
|
||||
assert!(!entries.is_empty());
|
||||
let first_entry_length = entries[0].entry_len;
|
||||
if entries[1..]
|
||||
.iter()
|
||||
.all(|entry| entry.entry_len == first_entry_length)
|
||||
{
|
||||
v.extend(first_entry_length.to_be_bytes());
|
||||
v.extend((entries.len() as u32).to_be_bytes());
|
||||
} else {
|
||||
v.extend(0u8.to_be_bytes());
|
||||
v.extend((entries.len() as u32).to_be_bytes());
|
||||
for entry in entries {
|
||||
v.extend(entry.entry_len.to_be_bytes());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_full_saio(
|
||||
v: &mut Vec<u8>,
|
||||
auxiliary_information: &AuxiliaryInformation,
|
||||
) -> Result<(), Error> {
|
||||
let version = if auxiliary_information
|
||||
.entries
|
||||
.iter()
|
||||
.any(|entry| entry.entry_offset > (u32::MAX as u64))
|
||||
{
|
||||
FULL_BOX_VERSION_1
|
||||
} else {
|
||||
FULL_BOX_VERSION_0
|
||||
};
|
||||
if let Some(aux_info_type) = auxiliary_information.aux_info_type {
|
||||
write_full_box(v, b"saio", version, 1, |v| {
|
||||
v.extend(aux_info_type);
|
||||
v.extend(auxiliary_information.aux_info_type_parameter.to_be_bytes());
|
||||
if version == FULL_BOX_VERSION_0 {
|
||||
write_saio_entries_v0(v, &auxiliary_information.entries)
|
||||
} else {
|
||||
write_saio_entries_v1(v, &auxiliary_information.entries)
|
||||
}
|
||||
})?;
|
||||
} else {
|
||||
write_full_box(v, b"saio", version, 0, |v| {
|
||||
if version == FULL_BOX_VERSION_0 {
|
||||
write_saio_entries_v0(v, &auxiliary_information.entries)
|
||||
} else {
|
||||
write_saio_entries_v1(v, &auxiliary_information.entries)
|
||||
}
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_saio_entries_v0(
|
||||
v: &mut Vec<u8>,
|
||||
entries: &[AuxiliaryInformationEntry],
|
||||
) -> Result<(), Error> {
|
||||
v.extend((entries.len() as u32).to_be_bytes());
|
||||
for entry in entries {
|
||||
v.extend((entry.entry_offset as u32).to_be_bytes());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_saio_entries_v1(
|
||||
v: &mut Vec<u8>,
|
||||
entries: &[AuxiliaryInformationEntry],
|
||||
) -> Result<(), Error> {
|
||||
v.extend((entries.len() as u32).to_be_bytes());
|
||||
for entry in entries {
|
||||
v.extend(entry.entry_offset.to_be_bytes());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ use std::str::FromStr;
|
|||
use std::sync::Mutex;
|
||||
|
||||
use crate::mp4mux::obu::read_seq_header_obu_bytes;
|
||||
use crate::mp4mux::AuxiliaryInformation;
|
||||
use crate::mp4mux::AuxiliaryInformationEntry;
|
||||
use crate::mp4mux::TaicClockType;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
|
|
@ -41,6 +43,11 @@ static NTP_CAPS: LazyLock<gst::Caps> =
|
|||
static UNIX_CAPS: LazyLock<gst::Caps> =
|
||||
LazyLock::new(|| gst::Caps::builder("timestamp/x-unix").build());
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
/// Reference timestamp meta caps for TAI timestamps with 1958-01-01 epoch.
|
||||
static TAI1958_CAPS: LazyLock<gst::Caps> =
|
||||
LazyLock::new(|| gst::Caps::builder("timestamp/x-tai1958").build());
|
||||
|
||||
/// Returns the UTC time of the buffer in the UNIX epoch.
|
||||
fn get_utc_time_from_buffer(buffer: &gst::BufferRef) -> Option<gst::ClockTime> {
|
||||
buffer
|
||||
|
|
@ -127,6 +134,7 @@ struct Settings {
|
|||
interleave_time: Option<gst::ClockTime>,
|
||||
movie_timescale: u32,
|
||||
extra_brands: Vec<[u8; 4]>,
|
||||
with_precision_timestamps: bool,
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
|
|
@ -136,6 +144,7 @@ impl Default for Settings {
|
|||
interleave_time: DEFAULT_INTERLEAVE_TIME,
|
||||
movie_timescale: 0,
|
||||
extra_brands: Vec::new(),
|
||||
with_precision_timestamps: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -164,7 +173,14 @@ struct PendingBuffer {
|
|||
composition_time_offset: Option<i64>,
|
||||
duration: Option<gst::ClockTime>,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
struct PendingAuxInfoEntry {
|
||||
aux_info_type: Option<[u8; 4]>,
|
||||
aux_info_type_parameter: u32,
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Stream {
|
||||
/// Sink pad for this stream.
|
||||
sinkpad: super::MP4MuxPad,
|
||||
|
|
@ -222,6 +238,11 @@ struct Stream {
|
|||
|
||||
/// TAI precision clock information
|
||||
tai_clock_info: Option<TaiClockInfo>,
|
||||
|
||||
/// The auxiliary information (saio/saiz) for the stream
|
||||
aux_info: Vec<AuxiliaryInformation>,
|
||||
/// auxiliary information to be written after the chunk is finished
|
||||
pending_aux_info_data: VecDeque<PendingAuxInfoEntry>,
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
|
|
@ -371,6 +392,10 @@ struct State {
|
|||
|
||||
/// Size of the `mdat` as written so far.
|
||||
mdat_size: u64,
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
/// The last TAI timestamp value, in nanoseconds after epoch
|
||||
last_tai_timestamp: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
|
|
@ -957,6 +982,7 @@ impl MP4Mux {
|
|||
&self,
|
||||
settings: &Settings,
|
||||
state: &mut State,
|
||||
buffers: &mut gst::BufferListRef,
|
||||
) -> Result<Option<usize>, gst::FlowError> {
|
||||
if let Some(current_stream_idx) = state.current_stream_idx {
|
||||
// If a stream was previously selected, check if another buffer from
|
||||
|
|
@ -991,6 +1017,10 @@ impl MP4Mux {
|
|||
);
|
||||
return Ok(Some(current_stream_idx));
|
||||
}
|
||||
let num_bytes_added =
|
||||
self.flush_aux_info(buffers, stream, state.current_offset);
|
||||
state.current_offset += num_bytes_added;
|
||||
state.mdat_size += num_bytes_added;
|
||||
|
||||
state.current_stream_idx = None;
|
||||
gst::debug!(
|
||||
|
|
@ -1012,6 +1042,10 @@ impl MP4Mux {
|
|||
obj = stream.sinkpad,
|
||||
"Stream is EOS, switching to next stream"
|
||||
);
|
||||
let num_bytes_added =
|
||||
self.flush_aux_info(buffers, stream, state.current_offset);
|
||||
state.current_offset += num_bytes_added;
|
||||
state.mdat_size += num_bytes_added;
|
||||
state.current_stream_idx = None;
|
||||
}
|
||||
Err(err) => {
|
||||
|
|
@ -1101,6 +1135,56 @@ impl MP4Mux {
|
|||
}
|
||||
}
|
||||
|
||||
fn flush_aux_info(
|
||||
&self,
|
||||
buffers: &mut gst::BufferListRef,
|
||||
stream: &mut Stream,
|
||||
initial_offset: u64,
|
||||
) -> u64 {
|
||||
gst::debug!(
|
||||
CAT,
|
||||
obj = stream.sinkpad,
|
||||
"Flushing {} pending entries of auxiliary information from stream {} to mdat at end of current chunk",
|
||||
stream.pending_aux_info_data.len(),
|
||||
stream.sinkpad.name(),
|
||||
);
|
||||
let mut num_bytes_added = 0u64;
|
||||
while let Some(pending_aux_info) = stream.pending_aux_info_data.pop_front() {
|
||||
// We don't handle the case where the aux_info_type is None - no idea what the semantics of that would be
|
||||
assert!(pending_aux_info.aux_info_type.is_some());
|
||||
let maybe_index = stream.aux_info.iter().position(|aux_info| {
|
||||
(aux_info.aux_info_type == pending_aux_info.aux_info_type)
|
||||
&& (aux_info.aux_info_type_parameter
|
||||
== pending_aux_info.aux_info_type_parameter)
|
||||
});
|
||||
let index: usize = match maybe_index {
|
||||
Some(index) => index,
|
||||
None => {
|
||||
// We did not already have a matching entry in the stream, add it
|
||||
let aux_info = AuxiliaryInformation {
|
||||
aux_info_type: pending_aux_info.aux_info_type,
|
||||
aux_info_type_parameter: pending_aux_info.aux_info_type_parameter,
|
||||
entries: Vec::new(),
|
||||
};
|
||||
stream.aux_info.push(aux_info);
|
||||
stream.aux_info.len() - 1
|
||||
}
|
||||
};
|
||||
let pending_aux_info_data = gst::Buffer::from_slice(pending_aux_info.data);
|
||||
assert!(pending_aux_info_data.size() <= u8::MAX as usize);
|
||||
let entry_size = pending_aux_info_data.size() as u8;
|
||||
buffers.add(pending_aux_info_data);
|
||||
stream.aux_info[index]
|
||||
.entries
|
||||
.push(AuxiliaryInformationEntry {
|
||||
entry_offset: initial_offset + num_bytes_added,
|
||||
entry_len: entry_size,
|
||||
});
|
||||
num_bytes_added += entry_size as u64;
|
||||
}
|
||||
num_bytes_added
|
||||
}
|
||||
|
||||
fn drain_buffers(
|
||||
&self,
|
||||
settings: &Settings,
|
||||
|
|
@ -1108,7 +1192,7 @@ impl MP4Mux {
|
|||
buffers: &mut gst::BufferListRef,
|
||||
) -> Result<(), gst::FlowError> {
|
||||
// Now we can start handling buffers
|
||||
while let Some(idx) = self.find_earliest_stream(settings, state)? {
|
||||
while let Some(idx) = self.find_earliest_stream(settings, state, buffers)? {
|
||||
let stream = &mut state.streams[idx];
|
||||
let buffer = stream.pending_buffer.take().unwrap();
|
||||
|
||||
|
|
@ -1173,7 +1257,97 @@ impl MP4Mux {
|
|||
}
|
||||
|
||||
let mut buffer = buffer.buffer;
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
{
|
||||
if settings.with_precision_timestamps {
|
||||
// Builds a TAITimestampPacket structure as defined in ISO/IEC 23001-17 Amendment 1, Section 8.1.2
|
||||
// That will be written out as `stai` aux info per Section 8.1.3.
|
||||
// See ISO/IEC 14496-12 Section 8.7.8 and 8.7.9 for more on aux info.
|
||||
if let Some(meta) = buffer
|
||||
.iter_meta::<gst::ReferenceTimestampMeta>()
|
||||
.find(|m| m.reference().can_intersect(&TAI1958_CAPS) && m.info().is_some())
|
||||
{
|
||||
gst::trace!(
|
||||
CAT,
|
||||
imp = self,
|
||||
"got TAI ReferenceTimestampMeta on the buffer"
|
||||
);
|
||||
let mut timestamp_packet = Vec::<u8>::with_capacity(9);
|
||||
timestamp_packet.extend(meta.timestamp().nseconds().to_be_bytes());
|
||||
state.last_tai_timestamp = meta.timestamp().nseconds();
|
||||
let iso23001_17_timestamp_info = meta.info().unwrap(); // checked in filter
|
||||
let mut timestamp_packet_flags = 0u8;
|
||||
if let Ok(synced) =
|
||||
iso23001_17_timestamp_info.get::<bool>("synchronization-state")
|
||||
{
|
||||
gst::trace!(
|
||||
CAT,
|
||||
imp = self,
|
||||
"synchronized to atomic source: {:?}",
|
||||
synced
|
||||
);
|
||||
if synced {
|
||||
timestamp_packet_flags |= 0x80u8;
|
||||
}
|
||||
} else {
|
||||
gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected synchronisation state, assuming not synchronised");
|
||||
}
|
||||
if let Ok(generation_failure) =
|
||||
iso23001_17_timestamp_info.get::<bool>("timestamp-generation-failure")
|
||||
{
|
||||
gst::trace!(
|
||||
CAT,
|
||||
imp = self,
|
||||
"timestamp generation failure: {:?}",
|
||||
generation_failure
|
||||
);
|
||||
if generation_failure {
|
||||
timestamp_packet_flags |= 0x40u8;
|
||||
}
|
||||
} else if meta.timestamp().nseconds() > state.last_tai_timestamp {
|
||||
gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected generation failure flag, timestamp looks OK, assuming OK");
|
||||
} else {
|
||||
gst::warning!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected generation failure flag and unexpected timestamp value, assuming generation failure");
|
||||
timestamp_packet_flags |= 0x40u8;
|
||||
}
|
||||
if let Ok(timestamp_is_modified) =
|
||||
iso23001_17_timestamp_info.get::<bool>("timestamp-is-modified")
|
||||
{
|
||||
gst::trace!(
|
||||
CAT,
|
||||
imp = self,
|
||||
"timestamp is modified: {:?}",
|
||||
timestamp_is_modified
|
||||
);
|
||||
if timestamp_is_modified {
|
||||
timestamp_packet_flags |= 0x20u8;
|
||||
}
|
||||
} else {
|
||||
gst::info!(CAT, imp=self, "TAI ReferenceTimestampMeta did not contain expected modification state value, assuming not modified");
|
||||
}
|
||||
timestamp_packet.extend(timestamp_packet_flags.to_be_bytes());
|
||||
stream.pending_aux_info_data.push_back(PendingAuxInfoEntry {
|
||||
aux_info_type: Some(*b"stai"),
|
||||
aux_info_type_parameter: 0,
|
||||
data: timestamp_packet,
|
||||
});
|
||||
} else {
|
||||
// generate a failure packet, because we always need aux info for a sample
|
||||
let mut timestamp_packet = Vec::<u8>::with_capacity(9);
|
||||
// The timestamp must monotonically increase
|
||||
let timestamp = state.last_tai_timestamp + 1;
|
||||
timestamp_packet.extend(timestamp.to_be_bytes());
|
||||
state.last_tai_timestamp = timestamp;
|
||||
let flags = 0x40u8; // not sync'd | generation failure | not modified,
|
||||
timestamp_packet.extend(flags.to_be_bytes());
|
||||
stream.pending_aux_info_data.push_back(PendingAuxInfoEntry {
|
||||
aux_info_type: Some(*b"stai"),
|
||||
aux_info_type_parameter: 0,
|
||||
data: timestamp_packet,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.queued_chunk_time += duration;
|
||||
stream.queued_chunk_bytes += buffer.size() as u64;
|
||||
|
||||
|
|
@ -1469,6 +1643,8 @@ impl MP4Mux {
|
|||
max_bitrate,
|
||||
avg_bitrate,
|
||||
tai_clock_info,
|
||||
aux_info: Vec::new(),
|
||||
pending_aux_info_data: VecDeque::new(),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -1545,6 +1721,12 @@ impl ObjectImpl for MP4Mux {
|
|||
.blurb("Comma-separated list of 4-character brand codes (e.g. duke,sook)")
|
||||
.mutable_ready()
|
||||
.build(),
|
||||
glib::ParamSpecBoolean::builder("tai-precision-timestamps")
|
||||
.nick("Precision Timestamps")
|
||||
.blurb("Whether to encode ISO/IEC 23001-17 TAI timestamps as auxiliary data")
|
||||
.default_value(false)
|
||||
.mutable_ready()
|
||||
.build(),
|
||||
]
|
||||
});
|
||||
|
||||
|
|
@ -1600,6 +1782,11 @@ impl ObjectImpl for MP4Mux {
|
|||
}
|
||||
}
|
||||
|
||||
"tai-precision-timestamps" => {
|
||||
let mut settings = self.settings.lock().unwrap();
|
||||
settings.with_precision_timestamps = value.get().expect("type checked upstream");
|
||||
}
|
||||
|
||||
_ => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
|
@ -1633,6 +1820,11 @@ impl ObjectImpl for MP4Mux {
|
|||
Some(brands_str).to_value()
|
||||
}
|
||||
|
||||
"tai-precision-timestamps" => {
|
||||
let settings = self.settings.lock().unwrap();
|
||||
settings.with_precision_timestamps.to_value()
|
||||
}
|
||||
|
||||
_ => unimplemented!(),
|
||||
}
|
||||
}
|
||||
|
|
@ -2005,6 +2197,9 @@ impl AggregatorImpl for MP4Mux {
|
|||
_ => {}
|
||||
}
|
||||
}
|
||||
if settings.with_precision_timestamps {
|
||||
compatible_brands.insert(*b"iso6"); // required for saiz/saio support
|
||||
}
|
||||
}
|
||||
if have_image_sequence && have_only_image_sequence {
|
||||
major_brand = b"msf1";
|
||||
|
|
@ -2091,6 +2286,7 @@ impl AggregatorImpl for MP4Mux {
|
|||
avg_bitrate: stream.avg_bitrate,
|
||||
chunks: stream.chunks,
|
||||
tai_clock_info: stream.tai_clock_info,
|
||||
auxiliary_info: stream.aux_info,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -241,6 +241,20 @@ pub(crate) struct TaiClockInfo {
|
|||
clock_type: TaicClockType,
|
||||
}
|
||||
|
||||
// Data for auxiliary information, as used for per-sample timestamps and for protection schemes
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct AuxiliaryInformationEntry {
|
||||
entry_offset: u64,
|
||||
entry_len: u8,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct AuxiliaryInformation {
|
||||
aux_info_type: Option<[u8; 4]>,
|
||||
aux_info_type_parameter: u32,
|
||||
entries: Vec<AuxiliaryInformationEntry>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Stream {
|
||||
/// Caps of this stream
|
||||
|
|
@ -284,6 +298,9 @@ pub(crate) struct Stream {
|
|||
|
||||
/// TAI Clock information (ISO/IEC 23001-17 Amd 1)
|
||||
tai_clock_info: Option<TaiClockInfo>,
|
||||
|
||||
/// Sample auxiliary information (ISO/IEC 14496-12:2022 Section 8.7.8 and 8.7.9)
|
||||
auxiliary_info: Vec<AuxiliaryInformation>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,11 @@
|
|||
//
|
||||
|
||||
use std::{fs::File, path::Path};
|
||||
#[cfg(feature = "v1_28")]
|
||||
use std::{io::Seek as _, sync::LazyLock};
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
use gst::{ClockTime, ReferenceTimestampMeta};
|
||||
use gst_pbutils::prelude::*;
|
||||
use mp4_atom::{Atom, ReadAtom as _, ReadFrom as _};
|
||||
use tempfile::tempdir;
|
||||
|
|
@ -23,6 +27,10 @@ fn init() {
|
|||
});
|
||||
}
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
static TAI1958_CAPS: LazyLock<gst::Caps> =
|
||||
LazyLock::new(|| gst::Caps::builder("timestamp/x-tai1958").build());
|
||||
|
||||
struct Pipeline(gst::Pipeline);
|
||||
impl std::ops::Deref for Pipeline {
|
||||
type Target = gst::Pipeline;
|
||||
|
|
@ -73,6 +81,7 @@ struct ExpectedConfiguration {
|
|||
has_taic: bool,
|
||||
taic_time_uncertainty: u64,
|
||||
taic_clock_type: u8,
|
||||
num_tai_timestamps: i32,
|
||||
}
|
||||
|
||||
fn test_basic_with(video_enc: &str, audio_enc: &str, cb: impl FnOnce(&Path)) {
|
||||
|
|
@ -290,6 +299,7 @@ fn test_expected_uncompressed_output(location: &Path, width: u32, height: u32) {
|
|||
has_taic: false,
|
||||
taic_clock_type: 0, // only if has_taic
|
||||
taic_time_uncertainty: 0, // only if has_taic
|
||||
num_tai_timestamps: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -626,6 +636,7 @@ fn test_expected_image_sequence_output(location: &Path, width: u32, height: u32)
|
|||
has_taic: false,
|
||||
taic_time_uncertainty: 0, // only if has_taic
|
||||
taic_clock_type: 0, // only if has_taic
|
||||
num_tai_timestamps: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -695,6 +706,7 @@ fn test_audio_only_output(location: &Path) {
|
|||
has_taic: false,
|
||||
taic_time_uncertainty: 0, // only if has_taic
|
||||
taic_clock_type: 0, // only if has_taic
|
||||
num_tai_timestamps: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -811,8 +823,8 @@ fn check_stbl_sanity(stbl: &mp4_atom::Stbl, expected_config: &ExpectedConfigurat
|
|||
} else {
|
||||
assert!(stbl.ctts.is_none());
|
||||
}
|
||||
assert!(stbl.saio.is_none());
|
||||
assert!(stbl.saiz.is_none());
|
||||
check_saio_sanity(&stbl.saio, expected_config);
|
||||
check_saiz_sanity(&stbl.saiz, expected_config);
|
||||
check_stco_sanity(&stbl.stco);
|
||||
check_stsc_sanity(&stbl.stsc);
|
||||
check_stsd_sanity(&stbl.stsd, expected_config);
|
||||
|
|
@ -828,6 +840,49 @@ fn check_stbl_sanity(stbl: &mp4_atom::Stbl, expected_config: &ExpectedConfigurat
|
|||
// TODO: check consistency between sample sizes and chunk / sample offsets
|
||||
}
|
||||
|
||||
fn check_saio_sanity(maybe_saio: &Option<mp4_atom::Saio>, expected_config: &ExpectedConfiguration) {
|
||||
if expected_config.num_tai_timestamps == 0 {
|
||||
assert!(maybe_saio.is_none());
|
||||
} else {
|
||||
assert!(maybe_saio.is_some());
|
||||
let saio = maybe_saio.as_ref().unwrap();
|
||||
assert!(saio.aux_info.is_some());
|
||||
assert_eq!(
|
||||
saio.aux_info.as_ref().unwrap().aux_info_type,
|
||||
b"stai".into()
|
||||
);
|
||||
assert_eq!(saio.aux_info.as_ref().unwrap().aux_info_type_parameter, 0);
|
||||
assert_eq!(
|
||||
saio.offsets.len(),
|
||||
expected_config.num_tai_timestamps as usize
|
||||
);
|
||||
let mut previous_offset = 0u64;
|
||||
for offset in &saio.offsets {
|
||||
// We check that the byte offsets are increasing
|
||||
// This is different to checking that the timestamps are increasing
|
||||
assert!(*offset > previous_offset);
|
||||
previous_offset = *offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_saiz_sanity(maybe_saiz: &Option<mp4_atom::Saiz>, expected_config: &ExpectedConfiguration) {
|
||||
if expected_config.num_tai_timestamps == 0 {
|
||||
assert!(maybe_saiz.is_none());
|
||||
} else {
|
||||
assert!(maybe_saiz.is_some());
|
||||
let saiz = maybe_saiz.as_ref().unwrap();
|
||||
assert!(saiz.aux_info.is_some());
|
||||
assert_eq!(
|
||||
saiz.aux_info.as_ref().unwrap().aux_info_type,
|
||||
b"stai".into()
|
||||
);
|
||||
assert_eq!(saiz.aux_info.as_ref().unwrap().aux_info_type_parameter, 0);
|
||||
assert_eq!(saiz.default_sample_info_size, 9);
|
||||
assert_eq!(saiz.sample_count, expected_config.num_tai_timestamps as u32);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_stco_sanity(maybe_stco: &Option<mp4_atom::Stco>) {
|
||||
assert!(maybe_stco
|
||||
.as_ref()
|
||||
|
|
@ -1062,10 +1117,179 @@ fn test_taic_encode(video_enc: &str) {
|
|||
has_taic: true,
|
||||
taic_time_uncertainty: 100_000,
|
||||
taic_clock_type: 2,
|
||||
num_tai_timestamps: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(feature = "v1_28")]
|
||||
fn test_taic_stai_encode(video_enc: &str, enabled: bool) {
|
||||
let filename = format!("taic_{video_enc}.mp4").to_string();
|
||||
let temp_dir = tempdir().unwrap();
|
||||
let temp_file_path = temp_dir.path().join(filename);
|
||||
let location = temp_file_path.as_path();
|
||||
let number_of_frames = 12;
|
||||
let pipeline = gst::Pipeline::builder()
|
||||
.name(format!("stai-{video_enc}"))
|
||||
.build();
|
||||
let videotestsrc = gst::ElementFactory::make("videotestsrc")
|
||||
.property("num-buffers", number_of_frames)
|
||||
.property("is-live", true)
|
||||
.build()
|
||||
.unwrap();
|
||||
let encoder = gst::ElementFactory::make(video_enc)
|
||||
.name("video encoder")
|
||||
.property("bframes", 0u32)
|
||||
.build()
|
||||
.unwrap();
|
||||
let taginject = gst::ElementFactory::make("taginject")
|
||||
.property_from_str("tags", "precision-clock-type=can-sync-to-TAI,precision-clock-time-uncertainty-nanoseconds=100000")
|
||||
.property_from_str("scope", "stream")
|
||||
.build().unwrap();
|
||||
let mux = gst::ElementFactory::make("isomp4mux")
|
||||
.property("tai-precision-timestamps", enabled)
|
||||
.build()
|
||||
.unwrap();
|
||||
let sink = gst::ElementFactory::make("filesink")
|
||||
.property("location", location)
|
||||
.build()
|
||||
.unwrap();
|
||||
pipeline
|
||||
.add_many([&videotestsrc, &encoder, &taginject, &mux, &sink])
|
||||
.unwrap();
|
||||
|
||||
gst::Element::link_many([&videotestsrc, &encoder, &taginject, &mux, &sink]).unwrap();
|
||||
|
||||
let tai_nanos_initial_offset: u64 = 100_000_000_000;
|
||||
let tai_nanos_per_frame_step = 20_000_000; // 20 milliseconds.
|
||||
let tai_nanos = std::sync::atomic::AtomicU64::new(tai_nanos_initial_offset);
|
||||
videotestsrc.static_pad("src").unwrap().add_probe(
|
||||
gst::PadProbeType::BUFFER,
|
||||
move |_pad, info| {
|
||||
if let Some(buffer) = info.buffer_mut() {
|
||||
let timestamp: ClockTime =
|
||||
ClockTime::from_nseconds(tai_nanos.load(std::sync::atomic::Ordering::Acquire));
|
||||
let mut meta =
|
||||
ReferenceTimestampMeta::add(buffer.make_mut(), &TAI1958_CAPS, timestamp, None);
|
||||
let s = gst::Structure::builder("iso23001-17-timestamp")
|
||||
.field("synchronization-state", true)
|
||||
.field("timestamp-generation-failure", false)
|
||||
.field("timestamp-is-modified", false)
|
||||
.build();
|
||||
meta.set_info(s);
|
||||
tai_nanos.fetch_add(
|
||||
tai_nanos_per_frame_step,
|
||||
std::sync::atomic::Ordering::AcqRel,
|
||||
);
|
||||
}
|
||||
gst::PadProbeReturn::Ok
|
||||
},
|
||||
);
|
||||
|
||||
pipeline
|
||||
.set_state(gst::State::Playing)
|
||||
.expect("Unable to set the pipeline to the `Playing` state");
|
||||
for msg in pipeline.bus().unwrap().iter_timed(gst::ClockTime::NONE) {
|
||||
use gst::MessageView;
|
||||
|
||||
match msg.view() {
|
||||
MessageView::Eos(..) => break,
|
||||
MessageView::Error(err) => {
|
||||
panic!(
|
||||
"Error from {:?}: {} ({:?})",
|
||||
err.src().map(|s| s.path_string()),
|
||||
err.error(),
|
||||
err.debug()
|
||||
);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
pipeline
|
||||
.set_state(gst::State::Null)
|
||||
.expect("Unable to set the pipeline to the `Null` state");
|
||||
|
||||
check_generic_single_trak_file_structure(
|
||||
location,
|
||||
b"iso4".into(),
|
||||
0,
|
||||
if enabled {
|
||||
vec![
|
||||
b"isom".into(),
|
||||
b"mp41".into(),
|
||||
b"mp42".into(),
|
||||
b"iso6".into(),
|
||||
]
|
||||
} else {
|
||||
vec![b"isom".into(), b"mp41".into(), b"mp42".into()]
|
||||
},
|
||||
ExpectedConfiguration {
|
||||
is_audio: false,
|
||||
width: 320,
|
||||
height: 240,
|
||||
has_ctts: false,
|
||||
has_stss: true,
|
||||
has_taic: true,
|
||||
taic_time_uncertainty: 100_000,
|
||||
taic_clock_type: 2,
|
||||
num_tai_timestamps: if enabled { number_of_frames } else { 0 },
|
||||
},
|
||||
);
|
||||
if enabled {
|
||||
let mut input = File::open(location).unwrap();
|
||||
let mut mdat_data: Option<Vec<u8>> = None;
|
||||
let mut mdat_offset: u64 = 0;
|
||||
while let Ok(header) = mp4_atom::Header::read_from(&mut input) {
|
||||
match header.kind {
|
||||
mp4_atom::Moov::KIND => {
|
||||
let moov = mp4_atom::Moov::read_atom(&header, &mut input).unwrap();
|
||||
let stbl = &moov.trak.first().unwrap().mdia.minf.stbl;
|
||||
let saio = stbl.saio.as_ref().unwrap();
|
||||
let saiz = stbl.saiz.as_ref().unwrap();
|
||||
if mdat_data.is_some() {
|
||||
assert_eq!(
|
||||
saio.aux_info.as_ref().unwrap().aux_info_type,
|
||||
b"stai".into()
|
||||
);
|
||||
assert_eq!(saio.aux_info.as_ref().unwrap().aux_info_type_parameter, 0);
|
||||
assert_eq!(
|
||||
saiz.aux_info.as_ref().unwrap().aux_info_type,
|
||||
b"stai".into()
|
||||
);
|
||||
assert_eq!(saiz.aux_info.as_ref().unwrap().aux_info_type_parameter, 0);
|
||||
for i in 0..saio.offsets.len() {
|
||||
let offset = saio.offsets[i];
|
||||
let len = saiz.default_sample_info_size as u64;
|
||||
let offset_into_mdat_start = (offset - mdat_offset) as usize;
|
||||
let offset_into_mdat_end = offset_into_mdat_start + len as usize;
|
||||
let vec = &mdat_data.as_ref().unwrap()
|
||||
[offset_into_mdat_start..offset_into_mdat_end]
|
||||
.to_vec();
|
||||
assert_eq!(vec.len(), 9);
|
||||
let mut timestamp_bytes: [u8; 8] = [0; 8];
|
||||
timestamp_bytes.copy_from_slice(&vec.as_slice()[0..8]);
|
||||
let timestamp = u64::from_be_bytes(timestamp_bytes);
|
||||
assert_eq!(
|
||||
timestamp,
|
||||
tai_nanos_initial_offset + (i as u64) * tai_nanos_per_frame_step
|
||||
);
|
||||
assert_eq!(vec[8], 0x80);
|
||||
}
|
||||
} else {
|
||||
panic!("mdat should not be none");
|
||||
}
|
||||
}
|
||||
mp4_atom::Mdat::KIND => {
|
||||
mdat_offset = input.stream_position().unwrap();
|
||||
let mdat = mp4_atom::Mdat::read_atom(&header, &mut input).unwrap();
|
||||
mdat_data = Some(mdat.data);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn test_taic_encode_cannot_sync(video_enc: &str) {
|
||||
let filename = format!("taic_{video_enc}_cannot_sync.mp4").to_string();
|
||||
let temp_dir = tempdir().unwrap();
|
||||
|
|
@ -1112,6 +1336,7 @@ fn test_taic_encode_cannot_sync(video_enc: &str) {
|
|||
has_taic: true,
|
||||
taic_time_uncertainty: 0xFFFF_FFFF_FFFF_FFFF,
|
||||
taic_clock_type: 1,
|
||||
num_tai_timestamps: 0,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -1122,6 +1347,20 @@ fn test_taic_x264() {
|
|||
test_taic_encode("x264enc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "v1_28")]
|
||||
fn test_taic_stai_x264() {
|
||||
init();
|
||||
test_taic_stai_encode("x264enc", true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "v1_28")]
|
||||
fn test_taic_stai_x264_not_enabled() {
|
||||
init();
|
||||
test_taic_stai_encode("x264enc", false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_taic_x264_no_sync() {
|
||||
init();
|
||||
|
|
|
|||
Loading…
Reference in a new issue