fmp4mux: Add support to write edts to handle audio priming

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1434>
This commit is contained in:
Thibault Saunier 2024-01-09 08:15:02 -03:00 committed by GStreamer Marge Bot
parent 54bc7a898e
commit e630aab769
7 changed files with 516 additions and 138 deletions

View file

@ -2425,6 +2425,18 @@
"type": "guint", "type": "guint",
"writable": true "writable": true
}, },
"write-edts-mode": {
"blurb": "Mode for writing EDTS, when in auto mode, edts written only for non-live streams.",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "auto (0)",
"mutable": "ready",
"readable": true,
"type": "GstFMP4MuxWriteEdtsMode",
"writable": true
},
"write-mehd": { "write-mehd": {
"blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)", "blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)",
"conditionally-available": false, "conditionally-available": false,
@ -2516,6 +2528,26 @@
"writable": true "writable": true
} }
} }
},
"GstFMP4MuxWriteEdtsMode": {
"kind": "enum",
"values": [
{
"desc": "Auto",
"name": "auto",
"value": "0"
},
{
"desc": "Always",
"name": "always",
"value": "1"
},
{
"desc": "Never",
"name": "never",
"value": "2"
}
]
} }
}, },
"package": "gst-plugin-fmp4", "package": "gst-plugin-fmp4",

View file

@ -6,6 +6,7 @@
// //
// SPDX-License-Identifier: MPL-2.0 // SPDX-License-Identifier: MPL-2.0
use crate::fmp4mux::imp::CAT;
use gst::prelude::*; use gst::prelude::*;
use anyhow::{anyhow, bail, Context, Error}; use anyhow::{anyhow, bail, Context, Error};
@ -555,6 +556,35 @@ struct TrackReference {
track_ids: Vec<u32>, track_ids: Vec<u32>,
} }
fn write_edts(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
Ok(())
}
fn write_elst(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
// Entry count
v.extend((stream.elst_infos.len() as u32).to_be_bytes());
for elst_info in &stream.elst_infos {
v.extend(
elst_info
.duration
.expect("Should have been set by `get_elst_infos`")
.to_be_bytes(),
);
// Media time
v.extend(elst_info.start.to_be_bytes());
// Media rate
v.extend(1u16.to_be_bytes());
v.extend(0u16.to_be_bytes());
}
Ok(())
}
fn write_trak( fn write_trak(
v: &mut Vec<u8>, v: &mut Vec<u8>,
cfg: &super::HeaderConfiguration, cfg: &super::HeaderConfiguration,
@ -571,10 +601,13 @@ fn write_trak(
|v| write_tkhd(v, cfg, idx, stream, creation_time), |v| write_tkhd(v, cfg, idx, stream, creation_time),
)?; )?;
// TODO: write edts if necessary: for audio tracks to remove initialization samples
// TODO: write edts optionally for negative DTS instead of offsetting the DTS // TODO: write edts optionally for negative DTS instead of offsetting the DTS
write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?; write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
if !stream.elst_infos.is_empty() {
if let Err(e) = write_edts(v, stream) {
gst::warning!(CAT, "Failed to write edts: {e}");
}
}
if !references.is_empty() { if !references.is_empty() {
write_box(v, b"tref", |v| write_tref(v, cfg, references))?; write_box(v, b"tref", |v| write_tref(v, cfg, references))?;

View file

@ -12,6 +12,7 @@ use gst::subclass::prelude::*;
use gst_base::prelude::*; use gst_base::prelude::*;
use gst_base::subclass::prelude::*; use gst_base::subclass::prelude::*;
use anyhow::Context;
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::mem; use std::mem;
@ -90,7 +91,7 @@ fn utc_time_to_running_time(
.and_then(|res| res.positive()) .and_then(|res| res.positive())
} }
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| { pub static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new( gst::DebugCategory::new(
"fmp4mux", "fmp4mux",
gst::DebugColorFlags::empty(), gst::DebugColorFlags::empty(),
@ -234,6 +235,88 @@ struct Stream {
running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>, running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>,
extra_header_data: Option<Vec<u8>>, extra_header_data: Option<Vec<u8>>,
/// Earliest PTS of the whole stream
earliest_pts: Option<gst::ClockTime>,
/// Current end PTS of the whole stream
end_pts: Option<gst::ClockTime>,
/// Edit list entries for this stream.
elst_infos: Vec<super::ElstInfo>,
}
impl Stream {
fn get_elst_infos(&self) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
let mut elst_infos = self.elst_infos.clone();
let timescale = self.timescale();
let earliest_pts = self.earliest_pts.unwrap_or(gst::ClockTime::ZERO);
let end_pts = self
.end_pts
.unwrap_or(gst::ClockTime::from_nseconds(u64::MAX - 1));
let mut iter = elst_infos.iter_mut().peekable();
while let Some(&mut ref mut elst_info) = iter.next() {
if elst_info.duration.unwrap_or(0) == 0 {
elst_info.duration = if let Some(next) = iter.peek_mut() {
let duration = next.start - elst_info.start;
if duration < 0 {
anyhow::bail!("Invalid edit list entry, negative duration");
}
Some(duration as u64)
} else {
Some(
(end_pts.checked_sub(earliest_pts))
.expect("stream end < stream start?")
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?,
)
}
}
}
Ok(elst_infos)
}
fn timescale(&self) -> u32 {
let trak_timescale = { self.sinkpad.imp().state.lock().unwrap().trak_timescale };
if trak_timescale > 0 {
return trak_timescale;
}
let s = self.caps.structure(0).unwrap();
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
if fps.numer() == 0 {
return 10_000;
}
if fps.denom() != 1 && fps.denom() != 1001 {
if let Some(fps) = (fps.denom() as u64)
.nseconds()
.mul_div_round(1_000_000_000, fps.numer() as u64)
.and_then(gst_video::guess_framerate)
{
return (fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000);
}
}
if fps.denom() == 1001 {
fps.numer() as u32
} else {
(fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000)
}
} else if let Ok(rate) = s.get::<i32>("rate") {
rate as u32
} else {
10_000
}
}
} }
#[derive(Default)] #[derive(Default)]
@ -295,6 +378,68 @@ pub(crate) struct FMP4Mux {
} }
impl FMP4Mux { impl FMP4Mux {
fn add_elst_info(
&self,
buffer: &PreQueuedBuffer,
stream: &mut Stream,
) -> Result<(), anyhow::Error> {
if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
let timescale = stream
.caps
.structure(0)
.unwrap()
.get::<i32>("rate")
.unwrap_or_else(|_| stream.timescale() as i32);
let gstclocktime_to_samples = move |v: gst::ClockTime| {
v.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("Invalid start in the AudioClipMeta")
};
let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
if let gst::GenericFormattedValue::Default(Some(v)) = t {
let v = v.into();
Ok(Some(v).filter(|x| x != &0u64))
} else if let gst::GenericFormattedValue::Time(Some(v)) = t {
Ok(Some(gstclocktime_to_samples(v)?))
} else {
Ok(None)
}
};
let start = generic_to_samples(cmeta.start())?;
let end = generic_to_samples(cmeta.end())?;
if end.is_none() && start.is_none() {
return Err(anyhow::anyhow!(
"No start or end time in `default` format in the AudioClipingMeta"
));
}
let start = if let Some(start) = generic_to_samples(cmeta.start())? {
start + gstclocktime_to_samples(buffer.pts)?
} else {
0
};
let duration = if let Some(e) = end {
Some(
gstclocktime_to_samples(buffer.end_pts)?
.checked_sub(e)
.context("Invalid end in the AudioClipMeta")?,
)
} else {
None
};
stream.elst_infos.push(super::ElstInfo {
start: start as i64,
duration,
});
}
Ok(())
}
/// Checks if a buffer is valid according to the stream configuration. /// Checks if a buffer is valid according to the stream configuration.
fn check_buffer(buffer: &gst::BufferRef, stream: &Stream) -> Result<(), gst::FlowError> { fn check_buffer(buffer: &gst::BufferRef, stream: &Stream) -> Result<(), gst::FlowError> {
let Stream { let Stream {
@ -400,6 +545,17 @@ impl FMP4Mux {
gst::ClockTime::ZERO gst::ClockTime::ZERO
}); });
if stream
.earliest_pts
.map_or(true, |earliest_pts| earliest_pts > pts)
{
stream.end_pts = Some(pts);
}
if stream.end_pts.opt_lt(end_pts).unwrap_or(true) {
stream.end_pts = Some(end_pts);
}
let (dts, end_dts) = if !stream.delta_frames.requires_dts() { let (dts, end_dts) = if !stream.delta_frames.requires_dts() {
(None, None) (None, None)
} else { } else {
@ -665,7 +821,13 @@ impl FMP4Mux {
assert!(stream.running_time_utc_time_mapping.is_some()); assert!(stream.running_time_utc_time_mapping.is_some());
} }
stream.pre_queue.pop_front().unwrap() let buffer = stream.pre_queue.pop_front().unwrap();
if let Err(err) = self.add_elst_info(&buffer, stream) {
gst::error!(CAT, "Failed to add elst info: {err:#}");
}
buffer
} }
// Caps/tag changes are allowed only in case that the // Caps/tag changes are allowed only in case that the
@ -2083,8 +2245,6 @@ impl FMP4Mux {
); );
for (idx, stream) in state.streams.iter_mut().enumerate() { for (idx, stream) in state.streams.iter_mut().enumerate() {
let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
let gops = self.drain_buffers_one_stream( let gops = self.drain_buffers_one_stream(
settings, settings,
stream, stream,
@ -2163,6 +2323,7 @@ impl FMP4Mux {
} }
} }
let trak_timescale = stream.sinkpad.imp().state.lock().unwrap().trak_timescale;
if gops.is_empty() { if gops.is_empty() {
gst::info!(CAT, obj = stream.sinkpad, "Draining no buffers",); gst::info!(CAT, obj = stream.sinkpad, "Draining no buffers",);
@ -2171,7 +2332,7 @@ impl FMP4Mux {
caps: stream.caps.clone(), caps: stream.caps.clone(),
start_time: None, start_time: None,
delta_frames: stream.delta_frames, delta_frames: stream.delta_frames,
trak_timescale: stream_settings.trak_timescale, trak_timescale,
}, },
VecDeque::new(), VecDeque::new(),
)); ));
@ -2222,7 +2383,7 @@ impl FMP4Mux {
caps: stream.caps.clone(), caps: stream.caps.clone(),
start_time: None, start_time: None,
delta_frames: stream.delta_frames, delta_frames: stream.delta_frames,
trak_timescale: stream_settings.trak_timescale, trak_timescale,
}, },
VecDeque::new(), VecDeque::new(),
)); ));
@ -2270,7 +2431,7 @@ impl FMP4Mux {
caps: stream.caps.clone(), caps: stream.caps.clone(),
start_time: Some(start_time), start_time: Some(start_time),
delta_frames: stream.delta_frames, delta_frames: stream.delta_frames,
trak_timescale: stream_settings.trak_timescale, trak_timescale,
}, },
buffers, buffers,
)); ));
@ -2860,6 +3021,9 @@ impl FMP4Mux {
current_position: gst::ClockTime::ZERO, current_position: gst::ClockTime::ZERO,
running_time_utc_time_mapping: None, running_time_utc_time_mapping: None,
extra_header_data: None, extra_header_data: None,
earliest_pts: None,
end_pts: None,
elst_infos: Vec::new(),
}); });
} }
@ -2923,11 +3087,19 @@ impl FMP4Mux {
let streams = state let streams = state
.streams .streams
.iter() .iter()
.map(|s| super::HeaderStream { .map(|s| {
trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale, let trak_timescale = { s.sinkpad.imp().settings.lock().unwrap().trak_timescale };
super::HeaderStream {
trak_timescale,
delta_frames: s.delta_frames, delta_frames: s.delta_frames,
caps: s.caps.clone(), caps: s.caps.clone(),
extra_header_data: s.extra_header_data.clone(), extra_header_data: s.extra_header_data.clone(),
elst_infos: s.get_elst_infos().unwrap_or_else(|e| {
gst::error!(CAT, "Could not prepare edit lists: {e:?}");
Vec::new()
}),
}
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
@ -3598,6 +3770,12 @@ impl AggregatorImpl for FMP4Mux {
fn start(&self) -> Result<(), gst::ErrorMessage> { fn start(&self) -> Result<(), gst::ErrorMessage> {
gst::trace!(CAT, imp = self, "Starting"); gst::trace!(CAT, imp = self, "Starting");
for pad in self.obj().sink_pads() {
let pad = pad.downcast_ref::<super::FMP4MuxPad>().unwrap().imp();
pad.state.lock().unwrap().trak_timescale = pad.settings.lock().unwrap().trak_timescale;
}
self.parent_start()?; self.parent_start()?;
// For non-single-stream variants configure a default segment that allows for negative // For non-single-stream variants configure a default segment that allows for negative
@ -4239,9 +4417,16 @@ struct PadSettings {
trak_timescale: u32, trak_timescale: u32,
} }
#[derive(Default, Clone)]
struct PadState {
// the selected trak_timescale
trak_timescale: u32,
}
#[derive(Default)] #[derive(Default)]
pub(crate) struct FMP4MuxPad { pub(crate) struct FMP4MuxPad {
settings: Mutex<PadSettings>, settings: Mutex<PadSettings>,
state: Mutex<PadState>,
} }
#[glib::object_subclass] #[glib::object_subclass]

View file

@ -169,6 +169,12 @@ pub(crate) struct HeaderConfiguration {
start_utc_time: Option<u64>, start_utc_time: Option<u64>,
} }
#[derive(Debug, Clone)]
pub(crate) struct ElstInfo {
start: i64,
duration: Option<u64>,
}
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct HeaderStream { pub(crate) struct HeaderStream {
/// Caps of this stream /// Caps of this stream
@ -182,6 +188,9 @@ pub(crate) struct HeaderStream {
// More data to be included in the fragmented stream header // More data to be included in the fragmented stream header
extra_header_data: Option<Vec<u8>>, extra_header_data: Option<Vec<u8>>,
/// Edit list clipping information
elst_infos: Vec<ElstInfo>,
} }
#[derive(Debug)] #[derive(Debug)]

View file

@ -225,50 +225,12 @@ fn write_moov(v: &mut Vec<u8>, header: &super::Header) -> Result<(), Error> {
Ok(()) Ok(())
} }
fn stream_to_timescale(stream: &super::Stream) -> u32 {
if stream.trak_timescale > 0 {
stream.trak_timescale
} else {
let s = stream.caps.structure(0).unwrap();
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
if fps.numer() == 0 {
return 10_000;
}
if fps.denom() != 1 && fps.denom() != 1001 {
if let Some(fps) = (fps.denom() as u64)
.nseconds()
.mul_div_round(1_000_000_000, fps.numer() as u64)
.and_then(gst_video::guess_framerate)
{
return (fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000);
}
}
if fps.denom() == 1001 {
fps.numer() as u32
} else {
(fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000)
}
} else if let Ok(rate) = s.get::<i32>("rate") {
rate as u32
} else {
10_000
}
}
}
fn header_to_timescale(header: &super::Header) -> u32 { fn header_to_timescale(header: &super::Header) -> u32 {
if header.movie_timescale > 0 { if header.movie_timescale > 0 {
header.movie_timescale header.movie_timescale
} else { } else {
// Use the reference track timescale // Use the reference track timescale
stream_to_timescale(&header.streams[0]) header.streams[0].timescale
} }
} }
@ -352,7 +314,7 @@ fn write_trak(
if !references.is_empty() { if !references.is_empty() {
write_box(v, b"tref", |v| write_tref(v, header, references))?; write_box(v, b"tref", |v| write_tref(v, header, references))?;
} }
write_box(v, b"edts", |v| write_edts(v, header, stream))?; write_box(v, b"edts", |v| write_edts(v, stream))?;
Ok(()) Ok(())
} }
@ -480,7 +442,7 @@ fn write_mdhd(
stream: &super::Stream, stream: &super::Stream,
creation_time: u64, creation_time: u64,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timescale = stream_to_timescale(stream); let timescale = stream.timescale;
// Creation time // Creation time
v.extend(creation_time.to_be_bytes()); v.extend(creation_time.to_be_bytes());
@ -1454,7 +1416,7 @@ fn write_stts(
_header: &super::Header, _header: &super::Header,
stream: &super::Stream, stream: &super::Stream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timescale = stream_to_timescale(stream); let timescale = stream.timescale;
let entry_count_position = v.len(); let entry_count_position = v.len();
// Entry count, rewritten in the end // Entry count, rewritten in the end
@ -1508,7 +1470,7 @@ fn write_ctts(
stream: &super::Stream, stream: &super::Stream,
version: u8, version: u8,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timescale = stream_to_timescale(stream); let timescale = stream.timescale;
let entry_count_position = v.len(); let entry_count_position = v.len();
// Entry count, rewritten in the end // Entry count, rewritten in the end
@ -1578,7 +1540,7 @@ fn write_cslg(
_header: &super::Header, _header: &super::Header,
stream: &super::Stream, stream: &super::Stream,
) -> Result<(), Error> { ) -> Result<(), Error> {
let timescale = stream_to_timescale(stream); let timescale = stream.timescale;
let (min_ctts, max_ctts) = stream let (min_ctts, max_ctts) = stream
.chunks .chunks
@ -1794,82 +1756,31 @@ fn write_tref(
Ok(()) Ok(())
} }
fn write_edts( fn write_edts(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
v: &mut Vec<u8>, write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
header: &super::Header,
stream: &super::Stream,
) -> Result<(), Error> {
write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| {
write_elst(v, header, stream)
})?;
Ok(()) Ok(())
} }
fn write_elst( fn write_elst(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
v: &mut Vec<u8>,
header: &super::Header,
stream: &super::Stream,
) -> Result<(), Error> {
// In movie header timescale
let timescale = header_to_timescale(header);
let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap();
if min_earliest_pts != stream.earliest_pts {
let gap = (stream.earliest_pts - min_earliest_pts)
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big gap")?;
if gap > 0 {
// Entry count // Entry count
v.extend(2u32.to_be_bytes()); v.extend((stream.elst_infos.len() as u32).to_be_bytes());
// First entry for the gap for elst_info in &stream.elst_infos {
v.extend(
// Edit duration elst_info
v.extend(gap.to_be_bytes()); .duration
.expect("Should have been set by `get_elst_infos`")
.to_be_bytes(),
);
// Media time // Media time
v.extend((-1i64).to_be_bytes()); v.extend(elst_info.start.to_be_bytes());
// Media rate // Media rate
v.extend(1u16.to_be_bytes()); v.extend(1u16.to_be_bytes());
v.extend(0u16.to_be_bytes()); v.extend(0u16.to_be_bytes());
} else {
// Entry count
v.extend(1u32.to_be_bytes());
} }
} else {
// Entry count
v.extend(1u32.to_be_bytes());
}
// Edit duration
let duration = (stream.end_pts - stream.earliest_pts)
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?;
v.extend(duration.to_be_bytes());
// Media time
if let Some(start_dts) = stream.start_dts {
let shift = (gst::Signed::Positive(stream.earliest_pts) - start_dts)
.nseconds()
.positive()
.unwrap_or(0)
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?;
v.extend(shift.to_be_bytes());
} else {
v.extend(0u64.to_be_bytes());
}
// Media rate
v.extend(1u16.to_be_bytes());
v.extend(0u16.to_be_bytes());
Ok(()) Ok(())
} }

View file

@ -6,6 +6,7 @@
// //
// SPDX-License-Identifier: MPL-2.0 // SPDX-License-Identifier: MPL-2.0
use anyhow::{anyhow, Context};
use gst::glib; use gst::glib;
use gst::prelude::*; use gst::prelude::*;
use gst::subclass::prelude::*; use gst::subclass::prelude::*;
@ -131,6 +132,10 @@ struct Stream {
/// Earliest PTS. /// Earliest PTS.
earliest_pts: Option<gst::ClockTime>, earliest_pts: Option<gst::ClockTime>,
/// Edit list entries for this stream.
elst_infos: Vec<super::ElstInfo>,
/// Current end PTS. /// Current end PTS.
end_pts: Option<gst::ClockTime>, end_pts: Option<gst::ClockTime>,
@ -143,6 +148,122 @@ struct Stream {
orientation: Option<ImageOrientation>, orientation: Option<ImageOrientation>,
} }
impl Stream {
fn get_elst_infos(
&self,
min_earliest_pts: gst::ClockTime,
) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
let mut elst_infos = self.elst_infos.clone();
let timescale = self.timescale();
let earliest_pts = self
.earliest_pts
.expect("Streams without earliest_pts should have been skipped");
let end_pts = self
.end_pts
.expect("Streams without end_pts should have been skipped");
// If no elst info were set, use the whole track
if self.elst_infos.is_empty() {
let start = if let Some(start_dts) = self.start_dts {
((gst::Signed::Positive(earliest_pts) - start_dts)
.nseconds()
.positive()
.unwrap_or(0)
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?) as i64
} else {
0i64
};
elst_infos.push(super::ElstInfo {
start,
duration: Some(
(end_pts - earliest_pts)
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?,
),
});
}
// Add a gap at the beginning if needed
if min_earliest_pts != earliest_pts {
let gap_duration = (earliest_pts - min_earliest_pts)
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big gap")?;
if gap_duration > 0 {
elst_infos.insert(
0,
super::ElstInfo {
start: -1,
duration: Some(gap_duration),
},
);
}
}
let mut iter = elst_infos.iter_mut().peekable();
while let Some(&mut ref mut elst_info) = iter.next() {
if elst_info.duration.unwrap_or(0u64) == 0u64 {
elst_info.duration = if let Some(next) = iter.peek_mut() {
Some((next.start - elst_info.start) as u64)
} else {
Some(
(end_pts - earliest_pts)
.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("too big track duration")?,
)
}
}
}
Ok(elst_infos)
}
fn timescale(&self) -> u32 {
let trak_timescale = { self.sinkpad.imp().settings.lock().unwrap().trak_timescale };
if trak_timescale > 0 {
return trak_timescale;
}
let s = self.caps.structure(0).unwrap();
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
if fps.numer() == 0 {
return 10_000;
}
if fps.denom() != 1 && fps.denom() != 1001 {
if let Some(fps) = (fps.denom() as u64)
.nseconds()
.mul_div_round(1_000_000_000, fps.numer() as u64)
.and_then(gst_video::guess_framerate)
{
return (fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000);
}
}
if fps.denom() == 1001 {
fps.numer() as u32
} else {
(fps.numer() as u32)
.mul_div_round(100, fps.denom() as u32)
.unwrap_or(10_000)
}
} else if let Ok(rate) = s.get::<i32>("rate") {
rate as u32
} else {
10_000
}
}
}
#[derive(Default)] #[derive(Default)]
struct State { struct State {
/// List of streams when the muxer was started. /// List of streams when the muxer was started.
@ -200,6 +321,77 @@ impl MP4Mux {
Ok(()) Ok(())
} }
fn add_elst_info(
&self,
buffer: &PendingBuffer,
stream: &mut Stream,
) -> Result<(), anyhow::Error> {
let cmeta = if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
cmeta
} else {
return Ok(());
};
let timescale = stream
.caps
.structure(0)
.unwrap()
.get::<i32>("rate")
.unwrap_or_else(|_| stream.timescale() as i32);
let gstclocktime_to_samples = move |v: gst::ClockTime| {
v.nseconds()
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
.context("Invalid start in the AudioClipMeta")
};
let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
if let gst::GenericFormattedValue::Default(Some(v)) = t {
let v: u64 = v.into();
Ok(Some(v).filter(|x| x != &0u64))
} else if let gst::GenericFormattedValue::Time(Some(v)) = t {
Ok(Some(gstclocktime_to_samples(v)?))
} else {
Ok(None)
}
};
let start: Option<u64> = generic_to_samples(cmeta.start())?;
let end: Option<u64> = generic_to_samples(cmeta.end())?;
if end.is_none() && start.is_none() {
return Err(anyhow!(
"No start or end time in `default` format in the AudioClipingMeta"
));
}
let start = if let Some(start) = generic_to_samples(cmeta.start())? {
start + gstclocktime_to_samples(buffer.pts)?
} else {
0
};
let duration = if let Some(e) = end {
Some(
gstclocktime_to_samples(buffer.pts)?
+ gstclocktime_to_samples(
buffer
.duration
.context("No duration on buffer, we can't add edit list")?,
)?
- e,
)
} else {
None
};
stream.elst_infos.push(super::ElstInfo {
start: start as i64,
duration,
});
Ok(())
}
fn peek_buffer( fn peek_buffer(
&self, &self,
sinkpad: &super::MP4MuxPad, sinkpad: &super::MP4MuxPad,
@ -889,6 +1081,11 @@ impl MP4Mux {
let duration = buffer.duration.unwrap(); let duration = buffer.duration.unwrap();
let composition_time_offset = buffer.composition_time_offset; let composition_time_offset = buffer.composition_time_offset;
if let Err(err) = self.add_elst_info(&buffer, stream) {
gst::error!(CAT, "Failed to add elst info: {:#}", err);
}
let mut buffer = buffer.buffer; let mut buffer = buffer.buffer;
stream.queued_chunk_time += duration; stream.queued_chunk_time += duration;
@ -1018,6 +1215,7 @@ impl MP4Mux {
queued_chunk_bytes: 0, queued_chunk_bytes: 0,
start_dts: None, start_dts: None,
earliest_pts: None, earliest_pts: None,
elst_infos: Default::default(),
end_pts: None, end_pts: None,
running_time_utc_time_mapping: None, running_time_utc_time_mapping: None,
extra_header_data: None, extra_header_data: None,
@ -1468,9 +1666,14 @@ impl AggregatorImpl for MP4Mux {
state.mdat_size state.mdat_size
); );
let min_earliest_pts = state
.streams
.iter()
.filter_map(|s| s.earliest_pts)
.min()
.unwrap();
let mut streams = Vec::with_capacity(state.streams.len()); let mut streams = Vec::with_capacity(state.streams.len());
for stream in state.streams.drain(..) { for stream in state.streams.drain(..) {
let pad_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts) let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts)
{ {
Some(res) => res, Some(res) => res,
@ -1480,10 +1683,14 @@ impl AggregatorImpl for MP4Mux {
streams.push(super::Stream { streams.push(super::Stream {
caps: stream.caps.clone(), caps: stream.caps.clone(),
delta_frames: stream.delta_frames, delta_frames: stream.delta_frames,
trak_timescale: pad_settings.trak_timescale, timescale: stream.timescale(),
start_dts: stream.start_dts,
earliest_pts, earliest_pts,
end_pts, end_pts,
elst_infos: stream.get_elst_infos(min_earliest_pts).unwrap_or_else(|e| {
gst::error!(CAT, "Could not prepare edit lists: {e:?}");
Vec::new()
}),
chunks: stream.chunks, chunks: stream.chunks,
extra_header_data: stream.extra_header_data.clone(), extra_header_data: stream.extra_header_data.clone(),
orientation: stream.orientation, orientation: stream.orientation,

View file

@ -172,6 +172,12 @@ pub(crate) struct Chunk {
samples: Vec<Sample>, samples: Vec<Sample>,
} }
#[derive(Debug, Clone)]
pub(crate) struct ElstInfo {
start: i64,
duration: Option<u64>,
}
#[derive(Debug)] #[derive(Debug)]
pub(crate) struct Stream { pub(crate) struct Stream {
/// Caps of this stream /// Caps of this stream
@ -181,15 +187,7 @@ pub(crate) struct Stream {
delta_frames: DeltaFrames, delta_frames: DeltaFrames,
/// Pre-defined trak timescale if not 0. /// Pre-defined trak timescale if not 0.
trak_timescale: u32, timescale: u32,
/// Start DTS
///
/// If this is negative then an edit list entry is needed to
/// make all sample times positive.
///
/// This is `None` for streams that have no concept of DTS.
start_dts: Option<gst::Signed<gst::ClockTime>>,
/// Earliest PTS /// Earliest PTS
/// ///
@ -207,6 +205,9 @@ pub(crate) struct Stream {
/// Orientation from tags /// Orientation from tags
orientation: Option<ImageOrientation>, orientation: Option<ImageOrientation>,
/// Edit list clipping information
elst_infos: Vec<ElstInfo>,
} }
#[derive(Debug)] #[derive(Debug)]