mirror of
https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git
synced 2024-11-25 13:01:07 +00:00
fmp4mux: Add support to write edts to handle audio priming
Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1434>
This commit is contained in:
Thibault Saunier
GStreamer Marge Bot
parent
54bc7a898e
commit
e630aab769
7 changed files with 516 additions and 138 deletions
|
|
@ -2425,6 +2425,18 @@
|
|||
"type": "guint",
|
||||
"writable": true
|
||||
},
|
||||
"write-edts-mode": {
|
||||
"blurb": "Mode for writing EDTS, when in auto mode, edts written only for non-live streams.",
|
||||
"conditionally-available": false,
|
||||
"construct": false,
|
||||
"construct-only": false,
|
||||
"controllable": false,
|
||||
"default": "auto (0)",
|
||||
"mutable": "ready",
|
||||
"readable": true,
|
||||
"type": "GstFMP4MuxWriteEdtsMode",
|
||||
"writable": true
|
||||
},
|
||||
"write-mehd": {
|
||||
"blurb": "Write movie extends header box with the duration at the end of the stream (needs a header-update-mode enabled)",
|
||||
"conditionally-available": false,
|
||||
|
|
@ -2516,6 +2528,26 @@
|
|||
"writable": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"GstFMP4MuxWriteEdtsMode": {
|
||||
"kind": "enum",
|
||||
"values": [
|
||||
{
|
||||
"desc": "Auto",
|
||||
"name": "auto",
|
||||
"value": "0"
|
||||
},
|
||||
{
|
||||
"desc": "Always",
|
||||
"name": "always",
|
||||
"value": "1"
|
||||
},
|
||||
{
|
||||
"desc": "Never",
|
||||
"name": "never",
|
||||
"value": "2"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"package": "gst-plugin-fmp4",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
//
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
use crate::fmp4mux::imp::CAT;
|
||||
use gst::prelude::*;
|
||||
|
||||
use anyhow::{anyhow, bail, Context, Error};
|
||||
|
|
@ -555,6 +556,35 @@ struct TrackReference {
|
|||
track_ids: Vec<u32>,
|
||||
}
|
||||
|
||||
fn write_edts(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
|
||||
write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_elst(v: &mut Vec<u8>, stream: &super::HeaderStream) -> Result<(), Error> {
|
||||
// Entry count
|
||||
v.extend((stream.elst_infos.len() as u32).to_be_bytes());
|
||||
|
||||
for elst_info in &stream.elst_infos {
|
||||
v.extend(
|
||||
elst_info
|
||||
.duration
|
||||
.expect("Should have been set by `get_elst_infos`")
|
||||
.to_be_bytes(),
|
||||
);
|
||||
|
||||
// Media time
|
||||
v.extend(elst_info.start.to_be_bytes());
|
||||
|
||||
// Media rate
|
||||
v.extend(1u16.to_be_bytes());
|
||||
v.extend(0u16.to_be_bytes());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_trak(
|
||||
v: &mut Vec<u8>,
|
||||
cfg: &super::HeaderConfiguration,
|
||||
|
|
@ -571,10 +601,13 @@ fn write_trak(
|
|||
|v| write_tkhd(v, cfg, idx, stream, creation_time),
|
||||
)?;
|
||||
|
||||
// TODO: write edts if necessary: for audio tracks to remove initialization samples
|
||||
// TODO: write edts optionally for negative DTS instead of offsetting the DTS
|
||||
|
||||
write_box(v, b"mdia", |v| write_mdia(v, cfg, stream, creation_time))?;
|
||||
if !stream.elst_infos.is_empty() {
|
||||
if let Err(e) = write_edts(v, stream) {
|
||||
gst::warning!(CAT, "Failed to write edts: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
if !references.is_empty() {
|
||||
write_box(v, b"tref", |v| write_tref(v, cfg, references))?;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ use gst::subclass::prelude::*;
|
|||
use gst_base::prelude::*;
|
||||
use gst_base::subclass::prelude::*;
|
||||
|
||||
use anyhow::Context;
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::VecDeque;
|
||||
use std::mem;
|
||||
|
|
@ -90,7 +91,7 @@ fn utc_time_to_running_time(
|
|||
.and_then(|res| res.positive())
|
||||
}
|
||||
|
||||
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
|
||||
pub static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
|
||||
gst::DebugCategory::new(
|
||||
"fmp4mux",
|
||||
gst::DebugColorFlags::empty(),
|
||||
|
|
@ -234,6 +235,88 @@ struct Stream {
|
|||
running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>,
|
||||
|
||||
extra_header_data: Option<Vec<u8>>,
|
||||
|
||||
/// Earliest PTS of the whole stream
|
||||
earliest_pts: Option<gst::ClockTime>,
|
||||
/// Current end PTS of the whole stream
|
||||
end_pts: Option<gst::ClockTime>,
|
||||
|
||||
/// Edit list entries for this stream.
|
||||
elst_infos: Vec<super::ElstInfo>,
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
fn get_elst_infos(&self) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
|
||||
let mut elst_infos = self.elst_infos.clone();
|
||||
let timescale = self.timescale();
|
||||
let earliest_pts = self.earliest_pts.unwrap_or(gst::ClockTime::ZERO);
|
||||
let end_pts = self
|
||||
.end_pts
|
||||
.unwrap_or(gst::ClockTime::from_nseconds(u64::MAX - 1));
|
||||
|
||||
let mut iter = elst_infos.iter_mut().peekable();
|
||||
while let Some(&mut ref mut elst_info) = iter.next() {
|
||||
if elst_info.duration.unwrap_or(0) == 0 {
|
||||
elst_info.duration = if let Some(next) = iter.peek_mut() {
|
||||
let duration = next.start - elst_info.start;
|
||||
if duration < 0 {
|
||||
anyhow::bail!("Invalid edit list entry, negative duration");
|
||||
}
|
||||
Some(duration as u64)
|
||||
} else {
|
||||
Some(
|
||||
(end_pts.checked_sub(earliest_pts))
|
||||
.expect("stream end < stream start?")
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(elst_infos)
|
||||
}
|
||||
|
||||
fn timescale(&self) -> u32 {
|
||||
let trak_timescale = { self.sinkpad.imp().state.lock().unwrap().trak_timescale };
|
||||
|
||||
if trak_timescale > 0 {
|
||||
return trak_timescale;
|
||||
}
|
||||
|
||||
let s = self.caps.structure(0).unwrap();
|
||||
|
||||
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
|
||||
if fps.numer() == 0 {
|
||||
return 10_000;
|
||||
}
|
||||
|
||||
if fps.denom() != 1 && fps.denom() != 1001 {
|
||||
if let Some(fps) = (fps.denom() as u64)
|
||||
.nseconds()
|
||||
.mul_div_round(1_000_000_000, fps.numer() as u64)
|
||||
.and_then(gst_video::guess_framerate)
|
||||
{
|
||||
return (fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000);
|
||||
}
|
||||
}
|
||||
|
||||
if fps.denom() == 1001 {
|
||||
fps.numer() as u32
|
||||
} else {
|
||||
(fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000)
|
||||
}
|
||||
} else if let Ok(rate) = s.get::<i32>("rate") {
|
||||
rate as u32
|
||||
} else {
|
||||
10_000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
|
|
@ -295,6 +378,68 @@ pub(crate) struct FMP4Mux {
|
|||
}
|
||||
|
||||
impl FMP4Mux {
|
||||
fn add_elst_info(
|
||||
&self,
|
||||
buffer: &PreQueuedBuffer,
|
||||
stream: &mut Stream,
|
||||
) -> Result<(), anyhow::Error> {
|
||||
if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
|
||||
let timescale = stream
|
||||
.caps
|
||||
.structure(0)
|
||||
.unwrap()
|
||||
.get::<i32>("rate")
|
||||
.unwrap_or_else(|_| stream.timescale() as i32);
|
||||
|
||||
let gstclocktime_to_samples = move |v: gst::ClockTime| {
|
||||
v.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("Invalid start in the AudioClipMeta")
|
||||
};
|
||||
|
||||
let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
|
||||
if let gst::GenericFormattedValue::Default(Some(v)) = t {
|
||||
let v = v.into();
|
||||
Ok(Some(v).filter(|x| x != &0u64))
|
||||
} else if let gst::GenericFormattedValue::Time(Some(v)) = t {
|
||||
Ok(Some(gstclocktime_to_samples(v)?))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
};
|
||||
|
||||
let start = generic_to_samples(cmeta.start())?;
|
||||
let end = generic_to_samples(cmeta.end())?;
|
||||
if end.is_none() && start.is_none() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"No start or end time in `default` format in the AudioClipingMeta"
|
||||
));
|
||||
}
|
||||
|
||||
let start = if let Some(start) = generic_to_samples(cmeta.start())? {
|
||||
start + gstclocktime_to_samples(buffer.pts)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let duration = if let Some(e) = end {
|
||||
Some(
|
||||
gstclocktime_to_samples(buffer.end_pts)?
|
||||
.checked_sub(e)
|
||||
.context("Invalid end in the AudioClipMeta")?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
stream.elst_infos.push(super::ElstInfo {
|
||||
start: start as i64,
|
||||
duration,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks if a buffer is valid according to the stream configuration.
|
||||
fn check_buffer(buffer: &gst::BufferRef, stream: &Stream) -> Result<(), gst::FlowError> {
|
||||
let Stream {
|
||||
|
|
@ -400,6 +545,17 @@ impl FMP4Mux {
|
|||
gst::ClockTime::ZERO
|
||||
});
|
||||
|
||||
if stream
|
||||
.earliest_pts
|
||||
.map_or(true, |earliest_pts| earliest_pts > pts)
|
||||
{
|
||||
stream.end_pts = Some(pts);
|
||||
}
|
||||
|
||||
if stream.end_pts.opt_lt(end_pts).unwrap_or(true) {
|
||||
stream.end_pts = Some(end_pts);
|
||||
}
|
||||
|
||||
let (dts, end_dts) = if !stream.delta_frames.requires_dts() {
|
||||
(None, None)
|
||||
} else {
|
||||
|
|
@ -665,7 +821,13 @@ impl FMP4Mux {
|
|||
assert!(stream.running_time_utc_time_mapping.is_some());
|
||||
}
|
||||
|
||||
stream.pre_queue.pop_front().unwrap()
|
||||
let buffer = stream.pre_queue.pop_front().unwrap();
|
||||
|
||||
if let Err(err) = self.add_elst_info(&buffer, stream) {
|
||||
gst::error!(CAT, "Failed to add elst info: {err:#}");
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
|
||||
// Caps/tag changes are allowed only in case that the
|
||||
|
|
@ -2083,8 +2245,6 @@ impl FMP4Mux {
|
|||
);
|
||||
|
||||
for (idx, stream) in state.streams.iter_mut().enumerate() {
|
||||
let stream_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
|
||||
|
||||
let gops = self.drain_buffers_one_stream(
|
||||
settings,
|
||||
stream,
|
||||
|
|
@ -2163,6 +2323,7 @@ impl FMP4Mux {
|
|||
}
|
||||
}
|
||||
|
||||
let trak_timescale = stream.sinkpad.imp().state.lock().unwrap().trak_timescale;
|
||||
if gops.is_empty() {
|
||||
gst::info!(CAT, obj = stream.sinkpad, "Draining no buffers",);
|
||||
|
||||
|
|
@ -2171,7 +2332,7 @@ impl FMP4Mux {
|
|||
caps: stream.caps.clone(),
|
||||
start_time: None,
|
||||
delta_frames: stream.delta_frames,
|
||||
trak_timescale: stream_settings.trak_timescale,
|
||||
trak_timescale,
|
||||
},
|
||||
VecDeque::new(),
|
||||
));
|
||||
|
|
@ -2222,7 +2383,7 @@ impl FMP4Mux {
|
|||
caps: stream.caps.clone(),
|
||||
start_time: None,
|
||||
delta_frames: stream.delta_frames,
|
||||
trak_timescale: stream_settings.trak_timescale,
|
||||
trak_timescale,
|
||||
},
|
||||
VecDeque::new(),
|
||||
));
|
||||
|
|
@ -2270,7 +2431,7 @@ impl FMP4Mux {
|
|||
caps: stream.caps.clone(),
|
||||
start_time: Some(start_time),
|
||||
delta_frames: stream.delta_frames,
|
||||
trak_timescale: stream_settings.trak_timescale,
|
||||
trak_timescale,
|
||||
},
|
||||
buffers,
|
||||
));
|
||||
|
|
@ -2860,6 +3021,9 @@ impl FMP4Mux {
|
|||
current_position: gst::ClockTime::ZERO,
|
||||
running_time_utc_time_mapping: None,
|
||||
extra_header_data: None,
|
||||
earliest_pts: None,
|
||||
end_pts: None,
|
||||
elst_infos: Vec::new(),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -2923,11 +3087,19 @@ impl FMP4Mux {
|
|||
let streams = state
|
||||
.streams
|
||||
.iter()
|
||||
.map(|s| super::HeaderStream {
|
||||
trak_timescale: s.sinkpad.imp().settings.lock().unwrap().trak_timescale,
|
||||
.map(|s| {
|
||||
let trak_timescale = { s.sinkpad.imp().settings.lock().unwrap().trak_timescale };
|
||||
super::HeaderStream {
|
||||
trak_timescale,
|
||||
delta_frames: s.delta_frames,
|
||||
caps: s.caps.clone(),
|
||||
extra_header_data: s.extra_header_data.clone(),
|
||||
elst_infos: s.get_elst_infos().unwrap_or_else(|e| {
|
||||
gst::error!(CAT, "Could not prepare edit lists: {e:?}");
|
||||
|
||||
Vec::new()
|
||||
}),
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
|
|
@ -3598,6 +3770,12 @@ impl AggregatorImpl for FMP4Mux {
|
|||
fn start(&self) -> Result<(), gst::ErrorMessage> {
|
||||
gst::trace!(CAT, imp = self, "Starting");
|
||||
|
||||
for pad in self.obj().sink_pads() {
|
||||
let pad = pad.downcast_ref::<super::FMP4MuxPad>().unwrap().imp();
|
||||
|
||||
pad.state.lock().unwrap().trak_timescale = pad.settings.lock().unwrap().trak_timescale;
|
||||
}
|
||||
|
||||
self.parent_start()?;
|
||||
|
||||
// For non-single-stream variants configure a default segment that allows for negative
|
||||
|
|
@ -4239,9 +4417,16 @@ struct PadSettings {
|
|||
trak_timescale: u32,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct PadState {
|
||||
// the selected trak_timescale
|
||||
trak_timescale: u32,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct FMP4MuxPad {
|
||||
settings: Mutex<PadSettings>,
|
||||
state: Mutex<PadState>,
|
||||
}
|
||||
|
||||
#[glib::object_subclass]
|
||||
|
|
|
|||
|
|
@ -169,6 +169,12 @@ pub(crate) struct HeaderConfiguration {
|
|||
start_utc_time: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ElstInfo {
|
||||
start: i64,
|
||||
duration: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct HeaderStream {
|
||||
/// Caps of this stream
|
||||
|
|
@ -182,6 +188,9 @@ pub(crate) struct HeaderStream {
|
|||
|
||||
// More data to be included in the fragmented stream header
|
||||
extra_header_data: Option<Vec<u8>>,
|
||||
|
||||
/// Edit list clipping information
|
||||
elst_infos: Vec<ElstInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
|
|
@ -225,50 +225,12 @@ fn write_moov(v: &mut Vec<u8>, header: &super::Header) -> Result<(), Error> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn stream_to_timescale(stream: &super::Stream) -> u32 {
|
||||
if stream.trak_timescale > 0 {
|
||||
stream.trak_timescale
|
||||
} else {
|
||||
let s = stream.caps.structure(0).unwrap();
|
||||
|
||||
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
|
||||
if fps.numer() == 0 {
|
||||
return 10_000;
|
||||
}
|
||||
|
||||
if fps.denom() != 1 && fps.denom() != 1001 {
|
||||
if let Some(fps) = (fps.denom() as u64)
|
||||
.nseconds()
|
||||
.mul_div_round(1_000_000_000, fps.numer() as u64)
|
||||
.and_then(gst_video::guess_framerate)
|
||||
{
|
||||
return (fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000);
|
||||
}
|
||||
}
|
||||
|
||||
if fps.denom() == 1001 {
|
||||
fps.numer() as u32
|
||||
} else {
|
||||
(fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000)
|
||||
}
|
||||
} else if let Ok(rate) = s.get::<i32>("rate") {
|
||||
rate as u32
|
||||
} else {
|
||||
10_000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn header_to_timescale(header: &super::Header) -> u32 {
|
||||
if header.movie_timescale > 0 {
|
||||
header.movie_timescale
|
||||
} else {
|
||||
// Use the reference track timescale
|
||||
stream_to_timescale(&header.streams[0])
|
||||
header.streams[0].timescale
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -352,7 +314,7 @@ fn write_trak(
|
|||
if !references.is_empty() {
|
||||
write_box(v, b"tref", |v| write_tref(v, header, references))?;
|
||||
}
|
||||
write_box(v, b"edts", |v| write_edts(v, header, stream))?;
|
||||
write_box(v, b"edts", |v| write_edts(v, stream))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -480,7 +442,7 @@ fn write_mdhd(
|
|||
stream: &super::Stream,
|
||||
creation_time: u64,
|
||||
) -> Result<(), Error> {
|
||||
let timescale = stream_to_timescale(stream);
|
||||
let timescale = stream.timescale;
|
||||
|
||||
// Creation time
|
||||
v.extend(creation_time.to_be_bytes());
|
||||
|
|
@ -1454,7 +1416,7 @@ fn write_stts(
|
|||
_header: &super::Header,
|
||||
stream: &super::Stream,
|
||||
) -> Result<(), Error> {
|
||||
let timescale = stream_to_timescale(stream);
|
||||
let timescale = stream.timescale;
|
||||
|
||||
let entry_count_position = v.len();
|
||||
// Entry count, rewritten in the end
|
||||
|
|
@ -1508,7 +1470,7 @@ fn write_ctts(
|
|||
stream: &super::Stream,
|
||||
version: u8,
|
||||
) -> Result<(), Error> {
|
||||
let timescale = stream_to_timescale(stream);
|
||||
let timescale = stream.timescale;
|
||||
|
||||
let entry_count_position = v.len();
|
||||
// Entry count, rewritten in the end
|
||||
|
|
@ -1578,7 +1540,7 @@ fn write_cslg(
|
|||
_header: &super::Header,
|
||||
stream: &super::Stream,
|
||||
) -> Result<(), Error> {
|
||||
let timescale = stream_to_timescale(stream);
|
||||
let timescale = stream.timescale;
|
||||
|
||||
let (min_ctts, max_ctts) = stream
|
||||
.chunks
|
||||
|
|
@ -1794,82 +1756,31 @@ fn write_tref(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn write_edts(
|
||||
v: &mut Vec<u8>,
|
||||
header: &super::Header,
|
||||
stream: &super::Stream,
|
||||
) -> Result<(), Error> {
|
||||
write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| {
|
||||
write_elst(v, header, stream)
|
||||
})?;
|
||||
fn write_edts(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
|
||||
write_full_box(v, b"elst", FULL_BOX_VERSION_1, 0, |v| write_elst(v, stream))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_elst(
|
||||
v: &mut Vec<u8>,
|
||||
header: &super::Header,
|
||||
stream: &super::Stream,
|
||||
) -> Result<(), Error> {
|
||||
// In movie header timescale
|
||||
let timescale = header_to_timescale(header);
|
||||
|
||||
let min_earliest_pts = header.streams.iter().map(|s| s.earliest_pts).min().unwrap();
|
||||
|
||||
if min_earliest_pts != stream.earliest_pts {
|
||||
let gap = (stream.earliest_pts - min_earliest_pts)
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big gap")?;
|
||||
|
||||
if gap > 0 {
|
||||
fn write_elst(v: &mut Vec<u8>, stream: &super::Stream) -> Result<(), Error> {
|
||||
// Entry count
|
||||
v.extend(2u32.to_be_bytes());
|
||||
v.extend((stream.elst_infos.len() as u32).to_be_bytes());
|
||||
|
||||
// First entry for the gap
|
||||
|
||||
// Edit duration
|
||||
v.extend(gap.to_be_bytes());
|
||||
for elst_info in &stream.elst_infos {
|
||||
v.extend(
|
||||
elst_info
|
||||
.duration
|
||||
.expect("Should have been set by `get_elst_infos`")
|
||||
.to_be_bytes(),
|
||||
);
|
||||
|
||||
// Media time
|
||||
v.extend((-1i64).to_be_bytes());
|
||||
v.extend(elst_info.start.to_be_bytes());
|
||||
|
||||
// Media rate
|
||||
v.extend(1u16.to_be_bytes());
|
||||
v.extend(0u16.to_be_bytes());
|
||||
} else {
|
||||
// Entry count
|
||||
v.extend(1u32.to_be_bytes());
|
||||
}
|
||||
} else {
|
||||
// Entry count
|
||||
v.extend(1u32.to_be_bytes());
|
||||
}
|
||||
|
||||
// Edit duration
|
||||
let duration = (stream.end_pts - stream.earliest_pts)
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?;
|
||||
v.extend(duration.to_be_bytes());
|
||||
|
||||
// Media time
|
||||
if let Some(start_dts) = stream.start_dts {
|
||||
let shift = (gst::Signed::Positive(stream.earliest_pts) - start_dts)
|
||||
.nseconds()
|
||||
.positive()
|
||||
.unwrap_or(0)
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?;
|
||||
|
||||
v.extend(shift.to_be_bytes());
|
||||
} else {
|
||||
v.extend(0u64.to_be_bytes());
|
||||
}
|
||||
|
||||
// Media rate
|
||||
v.extend(1u16.to_be_bytes());
|
||||
v.extend(0u16.to_be_bytes());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
//
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
use anyhow::{anyhow, Context};
|
||||
use gst::glib;
|
||||
use gst::prelude::*;
|
||||
use gst::subclass::prelude::*;
|
||||
|
|
@ -131,6 +132,10 @@ struct Stream {
|
|||
|
||||
/// Earliest PTS.
|
||||
earliest_pts: Option<gst::ClockTime>,
|
||||
|
||||
/// Edit list entries for this stream.
|
||||
elst_infos: Vec<super::ElstInfo>,
|
||||
|
||||
/// Current end PTS.
|
||||
end_pts: Option<gst::ClockTime>,
|
||||
|
||||
|
|
@ -143,6 +148,122 @@ struct Stream {
|
|||
orientation: Option<ImageOrientation>,
|
||||
}
|
||||
|
||||
impl Stream {
|
||||
fn get_elst_infos(
|
||||
&self,
|
||||
min_earliest_pts: gst::ClockTime,
|
||||
) -> Result<Vec<super::ElstInfo>, anyhow::Error> {
|
||||
let mut elst_infos = self.elst_infos.clone();
|
||||
let timescale = self.timescale();
|
||||
let earliest_pts = self
|
||||
.earliest_pts
|
||||
.expect("Streams without earliest_pts should have been skipped");
|
||||
let end_pts = self
|
||||
.end_pts
|
||||
.expect("Streams without end_pts should have been skipped");
|
||||
|
||||
// If no elst info were set, use the whole track
|
||||
if self.elst_infos.is_empty() {
|
||||
let start = if let Some(start_dts) = self.start_dts {
|
||||
((gst::Signed::Positive(earliest_pts) - start_dts)
|
||||
.nseconds()
|
||||
.positive()
|
||||
.unwrap_or(0)
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?) as i64
|
||||
} else {
|
||||
0i64
|
||||
};
|
||||
|
||||
elst_infos.push(super::ElstInfo {
|
||||
start,
|
||||
duration: Some(
|
||||
(end_pts - earliest_pts)
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?,
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
// Add a gap at the beginning if needed
|
||||
if min_earliest_pts != earliest_pts {
|
||||
let gap_duration = (earliest_pts - min_earliest_pts)
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big gap")?;
|
||||
|
||||
if gap_duration > 0 {
|
||||
elst_infos.insert(
|
||||
0,
|
||||
super::ElstInfo {
|
||||
start: -1,
|
||||
duration: Some(gap_duration),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut iter = elst_infos.iter_mut().peekable();
|
||||
while let Some(&mut ref mut elst_info) = iter.next() {
|
||||
if elst_info.duration.unwrap_or(0u64) == 0u64 {
|
||||
elst_info.duration = if let Some(next) = iter.peek_mut() {
|
||||
Some((next.start - elst_info.start) as u64)
|
||||
} else {
|
||||
Some(
|
||||
(end_pts - earliest_pts)
|
||||
.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("too big track duration")?,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(elst_infos)
|
||||
}
|
||||
|
||||
fn timescale(&self) -> u32 {
|
||||
let trak_timescale = { self.sinkpad.imp().settings.lock().unwrap().trak_timescale };
|
||||
|
||||
if trak_timescale > 0 {
|
||||
return trak_timescale;
|
||||
}
|
||||
|
||||
let s = self.caps.structure(0).unwrap();
|
||||
|
||||
if let Ok(fps) = s.get::<gst::Fraction>("framerate") {
|
||||
if fps.numer() == 0 {
|
||||
return 10_000;
|
||||
}
|
||||
|
||||
if fps.denom() != 1 && fps.denom() != 1001 {
|
||||
if let Some(fps) = (fps.denom() as u64)
|
||||
.nseconds()
|
||||
.mul_div_round(1_000_000_000, fps.numer() as u64)
|
||||
.and_then(gst_video::guess_framerate)
|
||||
{
|
||||
return (fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000);
|
||||
}
|
||||
}
|
||||
|
||||
if fps.denom() == 1001 {
|
||||
fps.numer() as u32
|
||||
} else {
|
||||
(fps.numer() as u32)
|
||||
.mul_div_round(100, fps.denom() as u32)
|
||||
.unwrap_or(10_000)
|
||||
}
|
||||
} else if let Ok(rate) = s.get::<i32>("rate") {
|
||||
rate as u32
|
||||
} else {
|
||||
10_000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
/// List of streams when the muxer was started.
|
||||
|
|
@ -200,6 +321,77 @@ impl MP4Mux {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn add_elst_info(
|
||||
&self,
|
||||
buffer: &PendingBuffer,
|
||||
stream: &mut Stream,
|
||||
) -> Result<(), anyhow::Error> {
|
||||
let cmeta = if let Some(cmeta) = buffer.buffer.meta::<gst_audio::AudioClippingMeta>() {
|
||||
cmeta
|
||||
} else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let timescale = stream
|
||||
.caps
|
||||
.structure(0)
|
||||
.unwrap()
|
||||
.get::<i32>("rate")
|
||||
.unwrap_or_else(|_| stream.timescale() as i32);
|
||||
|
||||
let gstclocktime_to_samples = move |v: gst::ClockTime| {
|
||||
v.nseconds()
|
||||
.mul_div_round(timescale as u64, gst::ClockTime::SECOND.nseconds())
|
||||
.context("Invalid start in the AudioClipMeta")
|
||||
};
|
||||
|
||||
let generic_to_samples = move |t| -> Result<Option<u64>, anyhow::Error> {
|
||||
if let gst::GenericFormattedValue::Default(Some(v)) = t {
|
||||
let v: u64 = v.into();
|
||||
Ok(Some(v).filter(|x| x != &0u64))
|
||||
} else if let gst::GenericFormattedValue::Time(Some(v)) = t {
|
||||
Ok(Some(gstclocktime_to_samples(v)?))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
};
|
||||
|
||||
let start: Option<u64> = generic_to_samples(cmeta.start())?;
|
||||
let end: Option<u64> = generic_to_samples(cmeta.end())?;
|
||||
|
||||
if end.is_none() && start.is_none() {
|
||||
return Err(anyhow!(
|
||||
"No start or end time in `default` format in the AudioClipingMeta"
|
||||
));
|
||||
}
|
||||
|
||||
let start = if let Some(start) = generic_to_samples(cmeta.start())? {
|
||||
start + gstclocktime_to_samples(buffer.pts)?
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let duration = if let Some(e) = end {
|
||||
Some(
|
||||
gstclocktime_to_samples(buffer.pts)?
|
||||
+ gstclocktime_to_samples(
|
||||
buffer
|
||||
.duration
|
||||
.context("No duration on buffer, we can't add edit list")?,
|
||||
)?
|
||||
- e,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
stream.elst_infos.push(super::ElstInfo {
|
||||
start: start as i64,
|
||||
duration,
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn peek_buffer(
|
||||
&self,
|
||||
sinkpad: &super::MP4MuxPad,
|
||||
|
|
@ -889,6 +1081,11 @@ impl MP4Mux {
|
|||
|
||||
let duration = buffer.duration.unwrap();
|
||||
let composition_time_offset = buffer.composition_time_offset;
|
||||
|
||||
if let Err(err) = self.add_elst_info(&buffer, stream) {
|
||||
gst::error!(CAT, "Failed to add elst info: {:#}", err);
|
||||
}
|
||||
|
||||
let mut buffer = buffer.buffer;
|
||||
|
||||
stream.queued_chunk_time += duration;
|
||||
|
|
@ -1018,6 +1215,7 @@ impl MP4Mux {
|
|||
queued_chunk_bytes: 0,
|
||||
start_dts: None,
|
||||
earliest_pts: None,
|
||||
elst_infos: Default::default(),
|
||||
end_pts: None,
|
||||
running_time_utc_time_mapping: None,
|
||||
extra_header_data: None,
|
||||
|
|
@ -1468,9 +1666,14 @@ impl AggregatorImpl for MP4Mux {
|
|||
state.mdat_size
|
||||
);
|
||||
|
||||
let min_earliest_pts = state
|
||||
.streams
|
||||
.iter()
|
||||
.filter_map(|s| s.earliest_pts)
|
||||
.min()
|
||||
.unwrap();
|
||||
let mut streams = Vec::with_capacity(state.streams.len());
|
||||
for stream in state.streams.drain(..) {
|
||||
let pad_settings = stream.sinkpad.imp().settings.lock().unwrap().clone();
|
||||
let (earliest_pts, end_pts) = match Option::zip(stream.earliest_pts, stream.end_pts)
|
||||
{
|
||||
Some(res) => res,
|
||||
|
|
@ -1480,10 +1683,14 @@ impl AggregatorImpl for MP4Mux {
|
|||
streams.push(super::Stream {
|
||||
caps: stream.caps.clone(),
|
||||
delta_frames: stream.delta_frames,
|
||||
trak_timescale: pad_settings.trak_timescale,
|
||||
start_dts: stream.start_dts,
|
||||
timescale: stream.timescale(),
|
||||
earliest_pts,
|
||||
end_pts,
|
||||
elst_infos: stream.get_elst_infos(min_earliest_pts).unwrap_or_else(|e| {
|
||||
gst::error!(CAT, "Could not prepare edit lists: {e:?}");
|
||||
|
||||
Vec::new()
|
||||
}),
|
||||
chunks: stream.chunks,
|
||||
extra_header_data: stream.extra_header_data.clone(),
|
||||
orientation: stream.orientation,
|
||||
|
|
|
|||
|
|
@ -172,6 +172,12 @@ pub(crate) struct Chunk {
|
|||
samples: Vec<Sample>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct ElstInfo {
|
||||
start: i64,
|
||||
duration: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct Stream {
|
||||
/// Caps of this stream
|
||||
|
|
@ -181,15 +187,7 @@ pub(crate) struct Stream {
|
|||
delta_frames: DeltaFrames,
|
||||
|
||||
/// Pre-defined trak timescale if not 0.
|
||||
trak_timescale: u32,
|
||||
|
||||
/// Start DTS
|
||||
///
|
||||
/// If this is negative then an edit list entry is needed to
|
||||
/// make all sample times positive.
|
||||
///
|
||||
/// This is `None` for streams that have no concept of DTS.
|
||||
start_dts: Option<gst::Signed<gst::ClockTime>>,
|
||||
timescale: u32,
|
||||
|
||||
/// Earliest PTS
|
||||
///
|
||||
|
|
@ -207,6 +205,9 @@ pub(crate) struct Stream {
|
|||
|
||||
/// Orientation from tags
|
||||
orientation: Option<ImageOrientation>,
|
||||
|
||||
/// Edit list clipping information
|
||||
elst_infos: Vec<ElstInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
Loading…
Reference in a new issue