diff --git a/Cargo.lock b/Cargo.lock index fb42c823..c77ae4e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2706,16 +2706,22 @@ dependencies = [ "anyhow", "atomic_refcell", "bitstream-io", + "byte-slice-cast", + "chrono", "gst-plugin-version-helper", "gstreamer", "gstreamer-app", + "gstreamer-audio", "gstreamer-check", "gstreamer-rtp", "gstreamer-video", + "hex", "once_cell", "rand", "rtp-types", + "slab", "smallvec", + "thiserror", "time", ] diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 1c08c051..3c57c068 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -6925,6 +6925,138 @@ }, "rank": "marginal" }, + "rtpmp4adepay2": { + "author": "François Laignel ", + "description": "Depayload an MPEG-4 Audio bitstream (e.g. AAC) from RTP packets (RFC 3016)", + "hierarchy": [ + "GstRtpMpeg4AudioDepay", + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Depayloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "application/x-rtp:\n media: audio\n clock-rate: [ 1, 2147483647 ]\n encoding-name: MP4A-LATM\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "audio/mpeg:\n mpegversion: 4\n framed: true\n stream-format: raw\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtpmp4apay2": { + "author": "François Laignel ", + "description": "Payload an MPEG-4 Audio bitstream (e.g. AAC) into RTP packets (RFC 3016)", + "hierarchy": [ + "GstRtpMpeg4AudioPay", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Payloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "audio/mpeg:\n mpegversion: 4\n framed: true\n stream-format: raw\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "application/x-rtp:\n media: audio\n payload: [ 96, 127 ]\n clock-rate: [ 1, 2147483647 ]\n encoding-name: MP4A-LATM\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtpmp4gdepay2": { + "author": "François Laignel ", + "description": "Depayload MPEG-4 Generic elementary streams from RTP packets (RFC 3640)", + "hierarchy": [ + "GstRtpMpeg4GenericDepay", + "GstRtpBaseDepay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Depayloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "application/x-rtp:\n media: { (string)audio, (string)video }\n clock-rate: [ 1, 2147483647 ]\n encoding-name: MPEG4-GENERIC\n mode: { (string)generic, (string)AAC-lbr, (string)AAC-hbr, (string)aac-hbr }\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "video/mpeg:\n mpegversion: 4\n systemstream: false\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n", + "direction": "src", + "presence": "always" + } + }, + "rank": "marginal" + }, + "rtpmp4gpay2": { + "author": "François Laignel ", + "description": "Payload an MPEG-4 Generic elementary stream into RTP packets (RFC 3640)", + "hierarchy": [ + "GstRtpMpeg4GenericPay", + "GstRtpBasePay2", + "GstElement", + "GstObject", + "GInitiallyUnowned", + "GObject" + ], + "klass": "Codec/Payloader/Network/RTP", + "pad-templates": { + "sink": { + "caps": "video/mpeg:\n mpegversion: 4\n systemstream: false\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n", + "direction": "sink", + "presence": "always" + }, + "src": { + "caps": "application/x-rtp:\n media: { (string)audio, (string)video }\n clock-rate: [ 1, 2147483647 ]\n encoding-name: MPEG4-GENERIC\n streamtype: { (string)4, (string)5 }\n mode: { (string)generic, (string)AAC-lbr, (string)AAC-hbr, (string)aac-hbr }\n", + "direction": "src", + "presence": "always" + } + }, + "properties": { + "aggregate-mode": { + "blurb": "Whether to send out AUs immediately or aggregate them until a packet is full.", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "auto (-1)", + "mutable": "null", + "readable": true, + "type": "GstRtpMpeg4GenericPayAggregateMode", + "writable": true + }, + "max-ptime": { + "blurb": "Maximum duration of the packet data in ns (-1 = unlimited up to MTU)", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "18446744073709551615", + "max": "9223372036854775807", + "min": "-1", + "mutable": "playing", + "readable": true, + "type": "gint64", + "writable": true + } + }, + "rank": "marginal" + }, "rtppcmadepay2": { "author": "Sebastian Dröge ", "description": "Depayload A-law from RTP packets (RFC 3551)", @@ -7694,6 +7826,26 @@ } } }, + "GstRtpMpeg4GenericPayAggregateMode": { + "kind": "enum", + "values": [ + { + "desc": "Automatic: zero-latency if upstream is live, otherwise aggregate elementary streams until packet is full.", + "name": "auto", + "value": "-1" + }, + { + "desc": "Zero Latency: always send out elementary streams right away, do not wait for more elementary streams to fill a packet.", + "name": "zero-latency", + "value": "0" + }, + { + "desc": "Aggregate: collect elementary streams until we have a full packet or the max-ptime limit is hit (if set).", + "name": "aggregate", + "value": "1" + } + ] + }, "GstRtpPcmauDepay2": { "hierarchy": [ "GstRtpPcmauDepay2", diff --git a/net/rtp/Cargo.toml b/net/rtp/Cargo.toml index 893c8ddb..86988f88 100644 --- a/net/rtp/Cargo.toml +++ b/net/rtp/Cargo.toml @@ -12,13 +12,19 @@ rust-version.workspace = true anyhow = "1" atomic_refcell = "0.1" bitstream-io = "2.1" +byte-slice-cast = "1.2" +chrono = { version = "0.4", default-features = false } gst = { workspace = true, features = ["v1_20"] } +gst-audio = { workspace = true, features = ["v1_20"] } gst-rtp = { workspace = true, features = ["v1_20"] } gst-video = { workspace = true, features = ["v1_20"] } +hex = "0.4.3" once_cell.workspace = true rand = { version = "0.8", default-features = false, features = ["std", "std_rng" ] } rtp-types = { version = "0.1" } +slab = "0.4.9" smallvec = { version = "1.11", features = ["union", "write", "const_generics", "const_new"] } +thiserror = "1" time = { version = "0.3", default-features = false, features = ["std"] } [dev-dependencies] diff --git a/net/rtp/src/lib.rs b/net/rtp/src/lib.rs index f68dcb41..c57d484d 100644 --- a/net/rtp/src/lib.rs +++ b/net/rtp/src/lib.rs @@ -16,6 +16,9 @@ */ use gst::glib; +#[macro_use] +mod utils; + mod gcc; mod audio_discont; @@ -25,6 +28,8 @@ mod basepay; mod av1; mod mp2t; +mod mp4a; +mod mp4g; mod pcmau; mod vp8; mod vp9; @@ -53,6 +58,12 @@ fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { mp2t::depay::register(plugin)?; mp2t::pay::register(plugin)?; + mp4a::depay::register(plugin)?; + mp4a::pay::register(plugin)?; + + mp4g::depay::register(plugin)?; + mp4g::pay::register(plugin)?; + pcmau::depay::register(plugin)?; pcmau::pay::register(plugin)?; diff --git a/net/rtp/src/mp4a/depay/imp.rs b/net/rtp/src/mp4a/depay/imp.rs new file mode 100644 index 00000000..d123afce --- /dev/null +++ b/net/rtp/src/mp4a/depay/imp.rs @@ -0,0 +1,836 @@ +// GStreamer RTP MPEG-4 Audio Depayloader +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-rtpmp4adepay2 + * @see_also: rtpmp4apay2, rtpmp4adepay, rtpmp4apay + * + * Depayload an MPEG-4 Audio bitstream from RTP packets as per [RFC 3016][rfc-3016]. + * + * [rfc-3016]: https://www.rfc-editor.org/rfc/rfc3016.html#section-4 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 udpsrc caps='application/x-rtp,media=audio,clock-rate=90000,encoding-name=MP4A-LATM,payload=96,config=(string)40002410' ! rtpjitterbuffer ! rtpmp4adepay2 ! decodebin3 ! audioconvert ! audioresample ! autoaudiosink + * ]| This will depayload an incoming RTP MPEG-4 Audio bitstream (AAC) with + * 1 channel @ 44100 sampling rate (default `audiotestsrc ! fdkaacenc` negotiation). + * You can use the #rtpmp4apay2 or #rtpmp4apay elements to create such an RTP stream. + * + * Since: plugins-rs-0.13.0 + */ +use atomic_refcell::AtomicRefCell; +use bitstream_io::{BigEndian, BitRead, BitReader}; +use once_cell::sync::Lazy; + +use gst::{glib, prelude::*, subclass::prelude::*}; + +use std::ops::ControlFlow; + +use crate::basedepay::{Packet, PacketToBufferRelation, RtpBaseDepay2Ext, TimestampOffset}; +use crate::mp4a::parsers::{StreamMuxConfig, Subframes}; +use crate::mp4a::{DEFAULT_CLOCK_RATE, ENCODING_NAME}; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpmp4adepay2", + gst::DebugColorFlags::empty(), + Some("RTP MPEG-4 Audio Depayloader"), + ) +}); + +#[derive(Default)] +pub struct RtpMpeg4AudioDepay { + state: AtomicRefCell, +} + +#[glib::object_subclass] +impl ObjectSubclass for RtpMpeg4AudioDepay { + const NAME: &'static str = "GstRtpMpeg4AudioDepay"; + type Type = super::RtpMpeg4AudioDepay; + type ParentType = crate::basedepay::RtpBaseDepay2; +} + +impl ObjectImpl for RtpMpeg4AudioDepay {} + +impl GstObjectImpl for RtpMpeg4AudioDepay {} + +impl ElementImpl for RtpMpeg4AudioDepay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP MPEG-4 Audio Depayloader", + "Codec/Depayloader/Network/RTP", + "Depayload an MPEG-4 Audio bitstream (e.g. AAC) from RTP packets (RFC 3016)", + "François Laignel ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", ENCODING_NAME) + /* All optional parameters + * + * "profile-level-id=[1,MAX]" + * "config=" + */ + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("framed", true) + .field("stream-format", "raw") + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +#[derive(Debug, Default)] +struct State { + config: Option, + frame_acc: Option, + seqnum_base: Option, + can_parse: bool, +} + +impl State { + fn flush(&mut self) { + self.frame_acc = None; + self.can_parse = false; + } +} + +#[derive(Debug)] +pub struct FrameAccumulator { + buf: Option>, + start_ext_seqnum: u64, +} + +impl FrameAccumulator { + pub fn new(packet: &Packet) -> Self { + FrameAccumulator { + buf: Some(packet.payload().to_owned()), + start_ext_seqnum: packet.ext_seqnum(), + } + } + + /// Extends this `FrameAccumulator` with the provided `Packet` payload. + /// + /// # Panic + /// + /// Panics if the subframes have already been taken. + #[track_caller] + pub fn extend(&mut self, packet: &Packet) { + self.buf + .as_mut() + .expect("subframes already taken") + .extend_from_slice(packet.payload()); + } + + /// Takes the `Subframes` out of this `FrameAccumulator`. + /// + /// # Panic + /// + /// Panics if the subframes have already been taken. + #[track_caller] + pub fn take_subframes<'a>(&'a mut self, config: &'a StreamMuxConfig) -> Subframes<'a> { + let buf = self.buf.take().expect("subframes already taken"); + Subframes::new(buf, config) + } +} + +#[derive(Debug)] +struct ConfigWithCodecData { + config: StreamMuxConfig, + codec_data: gst::Buffer, +} + +impl ConfigWithCodecData { + fn from_caps_structure(s: &gst::StructureRef) -> anyhow::Result> { + use anyhow::Context; + + let conf_str = s.get_optional::<&str>("config").context("config field")?; + let Some(conf_str) = conf_str else { + return Ok(None); + }; + + let mut data = hex::decode(conf_str).context("decoding config")?; + + let mut reader = BitReader::endian(data.as_slice(), BigEndian); + let config = reader.parse::()?; + + // Shift buffer for codec_data + for i in 0..(data.len() - 2) { + data[i] = ((data[i + 1] & 1) << 7) | ((data[i + 2] & 0xfe) >> 1); + } + + let codec_data = gst::Buffer::from_mut_slice(data); + + Ok(Some(ConfigWithCodecData { config, codec_data })) + } +} + +impl crate::basedepay::RtpBaseDepay2Impl for RtpMpeg4AudioDepay { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio"]; + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + + Ok(()) + } + + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let mut caps_builder = gst::Caps::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("framed", true) + .field("stream-format", "raw"); + + let mut config = match ConfigWithCodecData::from_caps_structure(s) { + Ok(Some(c)) => { + gst::log!(CAT, imp: self, "{:?}", c.config); + + caps_builder = caps_builder + .field("channels", c.config.prog.channel_conf as i32) + .field("rate", c.config.prog.sampling_freq as i32) + .field("codec_data", c.codec_data); + + c.config + } + Ok(None) => { + // In-band StreamMuxConfig not supported yet + gst::log!(CAT, imp: self, "config field not found"); + return false; + } + Err(err) => { + gst::error!(CAT, imp: self, "Error parsing StreamMuxConfig: {err}"); + return false; + } + }; + + let clock_rate = s.get::("clock-rate").expect("Required by Caps"); + debug_assert!(clock_rate.is_positive()); // constrained by Caps + let clock_rate = clock_rate as u32; + + let audio = &config.prog; + if clock_rate != DEFAULT_CLOCK_RATE && clock_rate != audio.sampling_freq { + if (audio.audio_object_type == 5 || audio.audio_object_type == 29) + && clock_rate == 2 * audio.sampling_freq + { + // FIXME this is a workaround for forward compatibility with AAC SBR & HE + // see also comment in the parsers module. + gst::warning!(CAT, imp: self, concat!( + "Found audio object type {}, which uses a specific extension for samplingFrequency. ", + "This extension is not supported yet. ", + "Will use 'clock-rate' {} as a workaround.", + ), + audio.audio_object_type, + clock_rate, + ); + } else { + gst::error!(CAT, imp: self, concat!( + "Caps 'clock-rate' {} and 'codec-data' sample rate {} mismatch. ", + "Will use 'clock-rate'", + ), + clock_rate, + audio.sampling_freq, + ); + } + + config.prog.sampling_freq = clock_rate; + } + + { + let mut state = self.state.borrow_mut(); + state.seqnum_base = s.get_optional::("seqnum-base").unwrap(); + state.config = Some(config); + } + + self.obj().set_src_caps(&caps_builder.build()); + + true + } + + // Can't push incomplete frames, so draining is the same as flushing. + fn flush(&self) { + gst::debug!(CAT, imp: self, "Flushing"); + self.state.borrow_mut().flush(); + } + + /// Packetization of MPEG-4 audio bitstreams: + /// https://www.rfc-editor.org/rfc/rfc3016.html#section-4 + fn handle_packet( + &self, + packet: &crate::basedepay::Packet, + ) -> Result { + let mut state = self.state.borrow_mut(); + + if !state.can_parse && self.check_initial_packet(&mut state, packet).is_break() { + self.obj().drop_packets(..=packet.ext_seqnum()); + return Ok(gst::FlowSuccess::Ok); + } + + if let Some(ref mut frame_acc) = state.frame_acc { + frame_acc.extend(packet); + } else { + state.frame_acc = Some(FrameAccumulator::new(packet)); + } + + // RTP marker bit indicates the last packet of the AudioMuxElement + if !packet.marker_bit() { + return Ok(gst::FlowSuccess::Ok); + } + + let mut frame = state.frame_acc.take().expect("frame_acc "); + + // Extract and push subframes from the accumulated buffers. + + // Payload is AudioMuxElement - ISO/IEC 14496-3 sub 1 table 1.20 + + // FIXME StreamMuxConfig may be present in the payload if muxConfigPresent is set + // in which case the audioMuxElement SHALL include an indication bit useSameStreamMux + // Current implementation is on par with rtpmp4adepay + // See also: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1173 + + let Some(config) = state.config.as_ref() else { + gst::error!(CAT, imp: self, "In-band StreamMuxConfig not supported"); + return Err(gst::FlowError::NotSupported); + }; + + let range = frame.start_ext_seqnum..=packet.ext_seqnum(); + + let mut accumulated_duration = gst::ClockTime::ZERO; + + for (idx, subframe) in frame.take_subframes(config).enumerate() { + match subframe { + Ok(subframe) => { + gst::log!(CAT, imp: self, "subframe {idx}: len {}", subframe.size()); + // The duration is always set by the subframes iterator + let duration = subframe.duration().expect("no duration set"); + + self.obj().queue_buffer( + PacketToBufferRelation::SeqnumsWithOffset { + seqnums: range.clone(), + timestamp_offset: TimestampOffset::Pts( + accumulated_duration.into_positive(), + ), + }, + subframe, + )?; + + accumulated_duration.opt_add_assign(duration); + } + Err(err) if err.is_zero_length_subframe() => { + gst::warning!(CAT, imp: self, "{err}"); + continue; + } + Err(err) => { + gst::warning!(CAT, imp: self, "{err}"); + self.obj().drop_packets(..=packet.ext_seqnum()); + break; + } + } + } + + Ok(gst::FlowSuccess::Ok) + } +} + +impl RtpMpeg4AudioDepay { + #[inline] + fn check_initial_packet(&self, state: &mut State, packet: &Packet) -> ControlFlow<()> { + let seqnum = (packet.ext_seqnum() & 0xffff) as u16; + + if let Some(seqnum_base) = state.seqnum_base { + let seqnum_base = (seqnum_base & 0xffff) as u16; + + // Assume seqnum_base and the initial ext_seqnum are in the same cycle + // This should be guaranteed by the JitterBuffer + let delta = crate::utils::seqnum_distance(seqnum, seqnum_base); + + if delta == 0 { + gst::debug!(CAT, imp: self, + "Got initial packet {seqnum_base} @ ext seqnum {}", packet.ext_seqnum(), + ); + state.can_parse = true; + + return ControlFlow::Continue(()); + } + + if delta < 0 { + gst::log!(CAT, imp: self, + "Waiting for initial packet {seqnum_base}, got {seqnum} (ext seqnum {})", + packet.ext_seqnum(), + ); + + return ControlFlow::Break(()); + } + + gst::debug!(CAT, imp: self, + "Packet {seqnum} (ext seqnum {}) passed expected initial packet {seqnum_base}, will sync on next marker", + packet.ext_seqnum(), + ); + + state.seqnum_base = None; + } + + // AudioMuxElement doesn't come with a frame start marker + // so wait until a marked packet is found and start parsing from the next packet + if packet.marker_bit() { + gst::debug!(CAT, imp: self, + "Found first marked packet {seqnum} (ext seqnum {}). Will start parsing from next packet", + packet.ext_seqnum(), + ); + + assert!(state.frame_acc.is_none()); + state.can_parse = true; + } else { + gst::log!(CAT, imp: self, + "First marked packet not found yet, skipping packet {seqnum} (ext seqnum {})", + packet.ext_seqnum(), + ); + } + + ControlFlow::Break(()) + } +} + +#[cfg(test)] +mod tests { + const RATE: u64 = 44_100; + const FRAME_LEN: u64 = 1024; + + struct HarnessBuilder { + subframes: u64, + seqnum_base: Option, + } + + impl HarnessBuilder { + fn subframes(mut self, subframes: u64) -> Self { + assert!(subframes > 0 && subframes <= 0b100_0000); + self.subframes = subframes; + self + } + + fn seqnum_base(mut self, seqnum_base: u32) -> Self { + self.seqnum_base = Some(seqnum_base); + self + } + + fn build_and_prepare(self) -> Harness { + use gst::prelude::MulDiv; + + gst::init().unwrap(); + crate::plugin_register_static().expect("failed to register plugin"); + + let depay = gst::ElementFactory::make("rtpmp4adepay2").build().unwrap(); + + let mut h = gst_check::Harness::with_element(&depay, Some("sink"), Some("src")); + h.play(); + + let caps = gst::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("clock-rate", RATE as i32) + .field("encoding-name", "MP4A-LATM") + .field( + "config", + format!("{:02x}002410", 0x40 | (self.subframes - 1)), + ) + .field_if_some("seqnum-base", self.seqnum_base) + .build(); + + assert!(h.push_event(gst::event::Caps::new(&caps))); + + let segment = gst::FormattedSegment::::new(); + assert!(h.push_event(gst::event::Segment::new(&segment))); + + let frame_duration = FRAME_LEN + .mul_div_floor(*gst::ClockTime::SECOND, RATE) + .map(gst::ClockTime::from_nseconds) + .unwrap(); + + Harness { + h, + frame_duration, + pts: gst::ClockTime::ZERO, + } + } + } + + struct Harness { + h: gst_check::Harness, + frame_duration: gst::ClockTime, + pts: gst::ClockTime, + } + + impl Harness { + fn builder() -> HarnessBuilder { + HarnessBuilder { + subframes: 1, + seqnum_base: None, + } + } + + /// Prepares a Harness with defaults. + fn prepare() -> Harness { + Self::builder().build_and_prepare() + } + + fn crank_pts(&mut self) { + self.pts += self.frame_duration; + } + + #[track_caller] + fn check_pts(&self, frame: &gst::Buffer) { + assert_eq!(frame.pts().unwrap(), self.pts); + } + + #[track_caller] + fn push(&mut self, packet: &'static [u8]) { + let mut buf = gst::Buffer::from_slice(packet); + buf.get_mut().unwrap().set_pts(self.pts); + + self.h.push(buf).expect("Couldn't push buffer"); + } + + #[track_caller] + fn push_and_ensure_no_frames(&mut self, packet: &'static [u8]) { + self.push(packet); + assert!(self.h.try_pull().is_none(), "Expecting no frames, got one"); + } + + #[track_caller] + fn push_and_check_single_packet_frame(&mut self, packet: &'static [u8]) { + self.push(packet); + let frame = self.h.pull().unwrap(); + self.check_pts(&frame); + assert_eq!(frame.map_readable().unwrap().as_slice(), &packet[13..]); + self.crank_pts(); + } + + #[track_caller] + fn flush_and_push_segment(&mut self) { + self.h.push_event(gst::event::FlushStart::new()); + self.h.push_event(gst::event::FlushStop::new(false)); + + let segment = gst::FormattedSegment::::new(); + assert!(self.h.push_event(gst::event::Segment::new(&segment))); + } + } + + impl std::ops::Deref for Harness { + type Target = gst_check::Harness; + fn deref(&self) -> &Self::Target { + &self.h + } + } + + impl std::ops::DerefMut for Harness { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.h + } + } + + #[test] + fn two_frames_two_packets_skipping_first() { + let mut h = Harness::prepare(); + + let p0 = &[ + 0x80, 0xe0, 0x73, 0x02, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Skipping first packet, but it comes with a marker, + // so will start parsing from next packet. + h.push_and_ensure_no_frames(p0); + + let p1 = &[ + 0x80, 0xe0, 0x73, 0x03, 0xb3, 0x1f, 0x7e, 0x9a, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p1); + } + + #[test] + fn two_frames_three_packets_skipping_first() { + let mut h = Harness::prepare(); + + let p0 = &[ + 0x80, 0x60, 0x04, 0x16, 0x76, 0xe8, 0x29, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0xff, 0x33, + 0x01, 0x3a, 0x99, 0x98, 0x3d, 0xbe, 0x2a, 0x29, 0xbe, 0x29, 0x42, 0x73, 0x7a, 0x9b, + 0x20, 0x2e, 0xbe, 0xb8, 0xd7, 0xb7, 0x9d, 0xba, 0xac, 0xff, 0xfa, 0xbf, 0xe7, 0xf1, + 0xd7, 0x1a, 0xf6, 0xa9, 0x4d, 0xff, 0xfd, 0x6f, 0xf1, 0xf8, 0xeb, 0x5e, 0x6e, 0xa5, + 0x52, 0x29, 0xa5, 0x20, 0x1a, 0x68, 0x80, 0x1e, 0x9a, 0x04, 0x49, 0xa6, 0x01, 0x03, + 0x4d, 0x02, 0x24, 0xbf, 0x7f, 0x16, 0xfd, 0xa5, 0x91, 0xfd, 0x0e, 0xa8, 0xfc, 0x07, + 0x60, 0x7d, 0xb3, 0xb0, 0x38, 0x41, 0xa9, 0x64, 0x68, 0x85, 0xd8, 0x1c, 0xa1, 0xf7, + 0x89, 0xb3, 0xa0, 0x30, 0xca, 0x18, 0x62, 0x0c, 0x58, 0x04, 0x9c, 0x13, 0x8c, 0x30, + 0xca, 0x0c, 0x62, 0x0c, 0x4e, 0x09, 0x18, 0x23, 0x40, 0x3e, 0x3f, 0xf1, 0x8e, 0x40, + 0xdf, 0x96, 0xc5, 0x70, 0xf1, 0xa3, 0x92, 0x55, 0x16, 0x17, 0x1e, 0xfd, 0xb6, 0x9e, + 0x95, 0x0d, 0x49, 0xea, 0x68, 0xf3, 0xfb, 0xbc, 0xc5, 0xe3, 0x9a, 0x9f, 0x92, 0x9a, + 0x3f, 0xbb, 0xf5, 0xee, 0x4c, 0xf7, 0xbf, 0x8a, 0x8e, 0xb2, 0x68, 0x3f, 0x05, 0xd1, + 0xba, 0x8a, 0x87, 0x06, 0x29, 0x16, 0x6e, 0x7d, 0x36, 0x63, 0xc2, 0xe2, 0xdc, 0xaa, + 0xf9, 0x55, 0x56, 0xa9, 0x81, 0xef, 0xbe, 0x5a, 0xfa, 0xf6, 0x1e, 0x6a, 0xd9, 0xba, + 0x3a, 0x35, 0x7f, 0x3f, 0x5c, 0x5f, 0x2d, 0x9b, 0x7b, 0x96, 0x1b, 0x6d, 0xca, 0xb1, + 0xb6, 0x27, 0xd5, 0x4a, 0x57, 0x7b, 0x96, 0x65, 0xe7, 0xd9, 0x2f, 0x3e, 0xc9, 0x63, + 0x6c, 0x56, 0x1a, 0xd4, 0xec, 0x71, 0x95, 0xc6, 0x4a, 0x24, 0xaf, 0xdd, 0xb2, 0xfd, + 0xdc, 0x2f, 0x6b, 0x85, 0x36, 0x75, 0x18, 0xcc, 0xb4, 0x11, 0x3c, 0x20, 0x9f, 0xda, + 0xe1, 0x7b, 0x5c, 0x2e, + ]; + // Skipping first markerless packet + h.push_and_ensure_no_frames(p0); + + let p1 = &[ + 0x80, 0xe0, 0x04, 0x17, 0x76, 0xe8, 0x29, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0xeb, 0x6d, + 0x79, 0x22, 0x4a, 0x25, 0x22, 0x54, 0x65, 0x18, 0x5e, 0xfb, 0xd8, 0x65, 0xce, 0x11, + 0xb2, 0xe4, 0x22, 0x20, 0x17, 0xd7, 0xee, 0xea, 0x60, 0x53, 0x3f, 0xc6, 0xee, 0x9f, + 0xe3, 0x7a, 0xef, 0xeb, 0xbd, 0x76, 0x82, 0x23, 0xf3, 0x08, 0xb2, 0x76, 0xed, 0x77, + 0x1d, 0x8d, 0x8e, 0x8d, 0x7e, 0x52, 0xfc, 0xa5, 0x52, 0x95, 0x4a, 0x66, 0x92, 0x69, + 0x1a, 0x4a, 0x1d, 0x9d, 0x9f, 0x07, + ]; + // Skipping p1, but it comes with a marker, + // so will start parsing from next packet. + h.push_and_ensure_no_frames(p1); + + let p2 = &[ + 0x80, 0xe0, 0x04, 0x18, 0x76, 0xe8, 0x2d, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0x41, 0x01, + 0x38, 0xf4, 0x2d, 0x22, 0xd0, 0x91, 0x5d, 0xfe, 0x79, 0xff, 0x12, 0x9e, 0x5c, 0x4d, + 0x4b, 0x96, 0xe2, 0x35, 0xa2, 0x8c, 0x1c, 0x3e, 0x78, 0x84, 0x10, 0xc9, 0x9a, 0x96, + 0x8b, 0x61, 0x76, 0xdc, 0xae, 0x5f, 0xfe, 0xcc, 0xc0, 0x5a, 0xfe, 0xb7, 0x75, 0x71, + 0x76, 0x2c, 0xdb, 0x19, 0xe6, 0xfe, 0x1e, 0x25, 0x3f, 0x8f, 0x84, 0xfe, 0x18, 0x0c, + 0x5e, 0x13, 0xe9, 0x80, 0x0b, 0x7f, 0x01, 0xc0, + ]; + // Packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p2); + } + + #[test] + fn seqnum_base_first_packet() { + let mut h = Harness::builder().seqnum_base(0x7302).build_and_prepare(); + + // ext_seqnum in packet: 0x1_7302 + let p0 = &[ + 0x80, 0xe0, 0x73, 0x02, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // First packet matches `seqnum-base` => parsing starts from here + // & packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p0); + } + + #[test] + fn two_frames_three_packets_seqnum_base_first_packet() { + let mut h = Harness::builder().seqnum_base(0x0416).build_and_prepare(); + + // ext_seqnum in packet: 0x1_0416 + let p0 = &[ + 0x80, 0x60, 0x04, 0x16, 0x76, 0xe8, 0x29, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0xff, 0x33, + 0x01, 0x3a, 0x99, 0x98, 0x3d, 0xbe, 0x2a, 0x29, 0xbe, 0x29, 0x42, 0x73, 0x7a, 0x9b, + 0x20, 0x2e, 0xbe, 0xb8, 0xd7, 0xb7, 0x9d, 0xba, 0xac, 0xff, 0xfa, 0xbf, 0xe7, 0xf1, + 0xd7, 0x1a, 0xf6, 0xa9, 0x4d, 0xff, 0xfd, 0x6f, 0xf1, 0xf8, 0xeb, 0x5e, 0x6e, 0xa5, + 0x52, 0x29, 0xa5, 0x20, 0x1a, 0x68, 0x80, 0x1e, 0x9a, 0x04, 0x49, 0xa6, 0x01, 0x03, + 0x4d, 0x02, 0x24, 0xbf, 0x7f, 0x16, 0xfd, 0xa5, 0x91, 0xfd, 0x0e, 0xa8, 0xfc, 0x07, + 0x60, 0x7d, 0xb3, 0xb0, 0x38, 0x41, 0xa9, 0x64, 0x68, 0x85, 0xd8, 0x1c, 0xa1, 0xf7, + 0x89, 0xb3, 0xa0, 0x30, 0xca, 0x18, 0x62, 0x0c, 0x58, 0x04, 0x9c, 0x13, 0x8c, 0x30, + 0xca, 0x0c, 0x62, 0x0c, 0x4e, 0x09, 0x18, 0x23, 0x40, 0x3e, 0x3f, 0xf1, 0x8e, 0x40, + 0xdf, 0x96, 0xc5, 0x70, 0xf1, 0xa3, 0x92, 0x55, 0x16, 0x17, 0x1e, 0xfd, 0xb6, 0x9e, + 0x95, 0x0d, 0x49, 0xea, 0x68, 0xf3, 0xfb, 0xbc, 0xc5, 0xe3, 0x9a, 0x9f, 0x92, 0x9a, + 0x3f, 0xbb, 0xf5, 0xee, 0x4c, 0xf7, 0xbf, 0x8a, 0x8e, 0xb2, 0x68, 0x3f, 0x05, 0xd1, + 0xba, 0x8a, 0x87, 0x06, 0x29, 0x16, 0x6e, 0x7d, 0x36, 0x63, 0xc2, 0xe2, 0xdc, 0xaa, + 0xf9, 0x55, 0x56, 0xa9, 0x81, 0xef, 0xbe, 0x5a, 0xfa, 0xf6, 0x1e, 0x6a, 0xd9, 0xba, + 0x3a, 0x35, 0x7f, 0x3f, 0x5c, 0x5f, 0x2d, 0x9b, 0x7b, 0x96, 0x1b, 0x6d, 0xca, 0xb1, + 0xb6, 0x27, 0xd5, 0x4a, 0x57, 0x7b, 0x96, 0x65, 0xe7, 0xd9, 0x2f, 0x3e, 0xc9, 0x63, + 0x6c, 0x56, 0x1a, 0xd4, 0xec, 0x71, 0x95, 0xc6, 0x4a, 0x24, 0xaf, 0xdd, 0xb2, 0xfd, + 0xdc, 0x2f, 0x6b, 0x85, 0x36, 0x75, 0x18, 0xcc, 0xb4, 0x11, 0x3c, 0x20, 0x9f, 0xda, + 0xe1, 0x7b, 0x5c, 0x2e, + ]; + // First packet matches `seqnum-base` => parsing starts from here + // But packet is not marked => accumulating + h.push_and_ensure_no_frames(p0); + + let p1 = &[ + 0x80, 0xe0, 0x04, 0x17, 0x76, 0xe8, 0x29, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0xeb, 0x6d, + 0x79, 0x22, 0x4a, 0x25, 0x22, 0x54, 0x65, 0x18, 0x5e, 0xfb, 0xd8, 0x65, 0xce, 0x11, + 0xb2, 0xe4, 0x22, 0x20, 0x17, 0xd7, 0xee, 0xea, 0x60, 0x53, 0x3f, 0xc6, 0xee, 0x9f, + 0xe3, 0x7a, 0xef, 0xeb, 0xbd, 0x76, 0x82, 0x23, 0xf3, 0x08, 0xb2, 0x76, 0xed, 0x77, + 0x1d, 0x8d, 0x8e, 0x8d, 0x7e, 0x52, 0xfc, 0xa5, 0x52, 0x95, 0x4a, 0x66, 0x92, 0x69, + 0x1a, 0x4a, 0x1d, 0x9d, 0x9f, 0x07, + ]; + // Packet is marked => will push frame to the src pad + h.push(p1); + + let frame = h.pull().unwrap(); + h.check_pts(&frame); + let frame = frame.map_readable().unwrap(); + assert_eq!(frame[..p0.len() - 14], p0[14..]); + assert_eq!(frame[p0.len() - 14..], p1[12..]); + + let p2 = &[ + 0x80, 0xe0, 0x04, 0x18, 0x76, 0xe8, 0x2d, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0x41, 0x01, + 0x38, 0xf4, 0x2d, 0x22, 0xd0, 0x91, 0x5d, 0xfe, 0x79, 0xff, 0x12, 0x9e, 0x5c, 0x4d, + 0x4b, 0x96, 0xe2, 0x35, 0xa2, 0x8c, 0x1c, 0x3e, 0x78, 0x84, 0x10, 0xc9, 0x9a, 0x96, + 0x8b, 0x61, 0x76, 0xdc, 0xae, 0x5f, 0xfe, 0xcc, 0xc0, 0x5a, 0xfe, 0xb7, 0x75, 0x71, + 0x76, 0x2c, 0xdb, 0x19, 0xe6, 0xfe, 0x1e, 0x25, 0x3f, 0x8f, 0x84, 0xfe, 0x18, 0x0c, + 0x5e, 0x13, 0xe9, 0x80, 0x0b, 0x7f, 0x01, 0xc0, + ]; + // Packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p2); + } + + #[test] + fn one_frame_two_subframes() { + let mut h = Harness::builder() + .subframes(2) + .seqnum_base(0x7302) + .build_and_prepare(); + + // ext_seqnum in packet: 0x1_7302 + let p0 = &[ + 0x80, 0xe0, 0x73, 0x02, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, 0x06, 0x01, 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // First packet matches `seqnum-base` => parsing starts from here + // & packet is marked => will push 2 subframes to the src pad + h.push(p0); + + let subframe = h.pull().unwrap(); + h.check_pts(&subframe); + let mut offset = 13usize; + let mut len = p0[offset - 1] as usize; + assert_eq!( + subframe.map_readable().unwrap().as_slice(), + &p0[offset..][..len] + ); + + // 2 subframes in one packet => reflect this on the actual pts + h.crank_pts(); + + let subframe = h.pull().unwrap(); + h.check_pts(&subframe); + offset += len + 1; + len = p0[offset - 1] as usize; + assert_eq!( + subframe.map_readable().unwrap().as_slice(), + &p0[offset..][..len] + ); + } + + #[test] + fn seqnum_base_second_packet() { + let mut h = Harness::builder().seqnum_base(0x7303).build_and_prepare(); + + // ext_seqnum in packet: 0x1_7302 + let p0 = &[ + 0x80, 0xe0, 0x73, 0x02, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Skipping first packet with seqnum 94978, + h.push_and_ensure_no_frames(p0); + + let p1 = &[ + 0x80, 0xe0, 0x73, 0x03, 0xb3, 0x1f, 0x7e, 0x9a, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // p1 matches `seqnum-base` => parsing starts from here + // & packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p1); + } + + #[test] + fn seqnum_base_passed_first_packet() { + let mut h = Harness::builder().seqnum_base(0x7300).build_and_prepare(); + + // ext_seqnum in packet: 0x1_7302 + let p0 = &[ + 0x80, 0xe0, 0x73, 0x02, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // First packet with seqnum 94978 passed `seqnum-base`, + // but it comes with a marker, so will start parsing from next packet + h.push_and_ensure_no_frames(p0); + + let p1 = &[ + 0x80, 0xe0, 0x73, 0x03, 0xb3, 0x1f, 0x7e, 0x9a, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p1); + } + + #[test] + fn two_packets_frame_flush_more_packets() { + let mut h = Harness::builder().seqnum_base(0x0416).build_and_prepare(); + + // ext_seqnum in packet: 0x1_0416 + let p0 = &[ + 0x80, 0x60, 0x04, 0x16, 0x76, 0xe8, 0x29, 0xc2, 0x16, 0xd8, 0x37, 0x68, 0xff, 0x33, + 0x01, 0x3a, 0x99, 0x98, 0x3d, 0xbe, 0x2a, 0x29, 0xbe, 0x29, 0x42, 0x73, 0x7a, 0x9b, + 0x20, 0x2e, 0xbe, 0xb8, 0xd7, 0xb7, 0x9d, 0xba, 0xac, 0xff, 0xfa, 0xbf, 0xe7, 0xf1, + 0xd7, 0x1a, 0xf6, 0xa9, 0x4d, 0xff, 0xfd, 0x6f, 0xf1, 0xf8, 0xeb, 0x5e, 0x6e, 0xa5, + 0x52, 0x29, 0xa5, 0x20, 0x1a, 0x68, 0x80, 0x1e, 0x9a, 0x04, 0x49, 0xa6, 0x01, 0x03, + 0x4d, 0x02, 0x24, 0xbf, 0x7f, 0x16, 0xfd, 0xa5, 0x91, 0xfd, 0x0e, 0xa8, 0xfc, 0x07, + 0x60, 0x7d, 0xb3, 0xb0, 0x38, 0x41, 0xa9, 0x64, 0x68, 0x85, 0xd8, 0x1c, 0xa1, 0xf7, + 0x89, 0xb3, 0xa0, 0x30, 0xca, 0x18, 0x62, 0x0c, 0x58, 0x04, 0x9c, 0x13, 0x8c, 0x30, + 0xca, 0x0c, 0x62, 0x0c, 0x4e, 0x09, 0x18, 0x23, 0x40, 0x3e, 0x3f, 0xf1, 0x8e, 0x40, + 0xdf, 0x96, 0xc5, 0x70, 0xf1, 0xa3, 0x92, 0x55, 0x16, 0x17, 0x1e, 0xfd, 0xb6, 0x9e, + 0x95, 0x0d, 0x49, 0xea, 0x68, 0xf3, 0xfb, 0xbc, 0xc5, 0xe3, 0x9a, 0x9f, 0x92, 0x9a, + 0x3f, 0xbb, 0xf5, 0xee, 0x4c, 0xf7, 0xbf, 0x8a, 0x8e, 0xb2, 0x68, 0x3f, 0x05, 0xd1, + 0xba, 0x8a, 0x87, 0x06, 0x29, 0x16, 0x6e, 0x7d, 0x36, 0x63, 0xc2, 0xe2, 0xdc, 0xaa, + 0xf9, 0x55, 0x56, 0xa9, 0x81, 0xef, 0xbe, 0x5a, 0xfa, 0xf6, 0x1e, 0x6a, 0xd9, 0xba, + 0x3a, 0x35, 0x7f, 0x3f, 0x5c, 0x5f, 0x2d, 0x9b, 0x7b, 0x96, 0x1b, 0x6d, 0xca, 0xb1, + 0xb6, 0x27, 0xd5, 0x4a, 0x57, 0x7b, 0x96, 0x65, 0xe7, 0xd9, 0x2f, 0x3e, 0xc9, 0x63, + 0x6c, 0x56, 0x1a, 0xd4, 0xec, 0x71, 0x95, 0xc6, 0x4a, 0x24, 0xaf, 0xdd, 0xb2, 0xfd, + 0xdc, 0x2f, 0x6b, 0x85, 0x36, 0x75, 0x18, 0xcc, 0xb4, 0x11, 0x3c, 0x20, 0x9f, 0xda, + 0xe1, 0x7b, 0x5c, 0x2e, + ]; + // First packet matches `seqnum-base` => parsing starts from here + // But packet is not marked => accumulating + h.push_and_ensure_no_frames(p0); + + h.flush_and_push_segment(); + + let p1 = &[ + 0x80, 0xe0, 0x05, 0x00, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Skipping first packet after flush, but it comes with a marker, + // so will start parsing from next packet. + h.push_and_ensure_no_frames(p1); + + let p2 = &[ + 0x80, 0xe0, 0x05, 0x01, 0xb3, 0x1f, 0x7a, 0x9b, 0x05, 0xd9, 0x9c, 0x33, 0x06, 0x01, + 0x40, 0x22, 0x80, 0xa3, 0x07, + ]; + // Packet is marked => will push frame to the src pad + h.push_and_check_single_packet_frame(p2); + } +} diff --git a/net/rtp/src/mp4a/depay/mod.rs b/net/rtp/src/mp4a/depay/mod.rs new file mode 100644 index 00000000..611be431 --- /dev/null +++ b/net/rtp/src/mp4a/depay/mod.rs @@ -0,0 +1,28 @@ +// GStreamer RTP MPEG-4 Audio Depayloader +// +// Copyright (C) 2023 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +pub mod imp; + +glib::wrapper! { + pub struct RtpMpeg4AudioDepay(ObjectSubclass) + @extends crate::basedepay::RtpBaseDepay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "rtpmp4adepay2", + gst::Rank::MARGINAL, + RtpMpeg4AudioDepay::static_type(), + ) +} diff --git a/net/rtp/src/mp4a/mod.rs b/net/rtp/src/mp4a/mod.rs new file mode 100644 index 00000000..db213c6b --- /dev/null +++ b/net/rtp/src/mp4a/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MPL-2.0 + +const DEFAULT_CLOCK_RATE: u32 = 90000; +const ENCODING_NAME: &str = "MP4A-LATM"; + +pub mod depay; +pub mod parsers; +pub mod pay; + +#[cfg(test)] +mod tests; diff --git a/net/rtp/src/mp4a/parsers.rs b/net/rtp/src/mp4a/parsers.rs new file mode 100644 index 00000000..3b2246fe --- /dev/null +++ b/net/rtp/src/mp4a/parsers.rs @@ -0,0 +1,333 @@ +// GStreamer MPEG-4 Audio bitstream parsers. +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use anyhow::Context; +use bitstream_io::{BitRead, FromBitStream}; +use gst::prelude::*; + +const ACC_SAMPLING_FREQS: [u32; 13] = [ + 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350, +]; + +/// Errors that can be produced when parsing a `StreamMuxConfig` & `AudioSpecificConfig`. +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum MPEG4AudioParserError { + #[error("Unknown audioMuxVersion 1. Expected 0.")] + UnknownVersion, + + #[error("Unsupported: num_progs {num_progs}, num_layers {num_layers}")] + UnsupportedProgsLayer { num_progs: u8, num_layers: u8 }, + + #[error("Invalid audio object type 0")] + InvalidAudioObjectType0, + + #[error("Invalid sampling frequency idx {}", 0)] + InvalidSamplingFreqIdx(u8), + + #[error("Invalid channels {}", .0)] + InvalidChannels(u8), + + #[error("subframe {} with len 0", .0)] + ZeroLengthSubframe(u8), + + #[error("Wrong frame size. Required {required}, available {available}")] + WrongFrameSize { required: usize, available: usize }, + + #[error("Unsupported Profile {profile}")] + UnsupportedProfile { profile: String }, + + #[error("Unsupported Level {level} for Profile {profile}")] + UnsupportedLevel { level: String, profile: String }, +} + +impl MPEG4AudioParserError { + pub fn is_zero_length_subframe(&self) -> bool { + matches!(self, MPEG4AudioParserError::ZeroLengthSubframe(_)) + } +} + +/// StreamMuxConfig (partial) - ISO/IEC 14496-3 sub 1 table 1.21 +/// Support for: +/// +/// * allStreamsSameTimeFraming == true +/// * 1 prog & 1 layer only => flatten +#[derive(Debug)] +pub struct StreamMuxConfig { + pub num_sub_frames: u8, + pub prog: AudioSpecificConfig, +} + +impl FromBitStream for StreamMuxConfig { + type Error = anyhow::Error; + + fn from_reader(r: &mut R) -> anyhow::Result { + use MPEG4AudioParserError::*; + + // StreamMuxConfig - ISO/IEC 14496-3 sub 1 table 1.21 + // audioMuxVersion == 0 (1 bit) + // allStreamsSameTimeFraming == 1 (1 bit) + // numSubFrames == 0 means 1 subframe (6 bits) + // numProgram == 0 means 1 program (4 bits) + // numLayer == 0 means 1 layer (3 bits) + + if r.read::(1).context("audioMuxVersion")? != 0 { + Err(UnknownVersion)?; + } + + let _ = r.read_bit().context("allStreamsSameTimeFraming")?; + let num_sub_frames = r.read::(6).context("numSubFrames")? + 1; + + let num_progs = r.read::(4).context("numProgram")? + 1; + let num_layers = r.read::(3).context("numLayer")? + 1; + if !(num_progs == 1 && num_layers == 1) { + // Same as for rtpmp4adepay + Err(UnsupportedProgsLayer { + num_progs, + num_layers, + })?; + } + + // AudioSpecificConfig - ISO/IEC 14496-3 sub 1 table 1.8 + let prog = r.parse::().context("prog 1 layer 1")?; + + // Ignore remaining bits for now + + Ok(StreamMuxConfig { + num_sub_frames, + prog, + }) + } +} + +/// AudioSpecificConfig - ISO/IEC 14496-3 sub 1 table 1.8 +/// +/// Support for: +/// +/// * allStreamsSameTimeFraming == true +/// * 1 prog & 1 layer only => flatten +#[derive(Debug)] +pub struct AudioSpecificConfig { + pub audio_object_type: u8, + pub sampling_freq: u32, + pub channel_conf: u8, + /// GASpecificConfig (partial) - ISO/IEC 14496-3 sub 4 table 4.1 + pub frame_len: usize, +} + +impl FromBitStream for AudioSpecificConfig { + type Error = anyhow::Error; + + fn from_reader(r: &mut R) -> anyhow::Result { + use MPEG4AudioParserError::*; + + let audio_object_type = r.read(5).context("audioObjectType")?; + if audio_object_type == 0 { + Err(InvalidAudioObjectType0)?; + } + + let sampling_freq_idx = r.read::(4).context("samplingFrequencyIndex")?; + if sampling_freq_idx as usize >= ACC_SAMPLING_FREQS.len() && sampling_freq_idx != 0xf { + Err(InvalidSamplingFreqIdx(sampling_freq_idx))?; + } + + // RTP rate depends on sampling freq of the audio + let sampling_freq = if sampling_freq_idx == 0xf { + r.read(24).context("samplingFrequency")? + } else { + ACC_SAMPLING_FREQS[sampling_freq_idx as usize] + }; + + let channel_conf = r.read(4).context("channelConfiguration")?; + if channel_conf > 7 { + Err(InvalidChannels(channel_conf))?; + } + + // GASpecificConfig - ISO/IEC 14496-3 sub 4 table 4.1 + + // TODO this is based on ISO/IEC 14496-3:2001 as implemented in rtpmp4adepay + // and should be updated with enhancements from ISO/IEC 14496-3:2009 + // for AAC SBR & HE support. + + let frame_len = if [1, 2, 3, 4, 6, 7].contains(&audio_object_type) + && r.read_bit().context("frame_len_flag")? + { + 960 + } else { + 1024 + }; + + // Ignore remaining bits for now + + Ok(AudioSpecificConfig { + audio_object_type, + sampling_freq, + channel_conf, + frame_len, + }) + } +} + +/// audioProfileLevelIndication - ISO/IEC 14496-3 (2009) table 1.14 +pub struct ProfileLevel { + pub profile: String, + pub level: String, + pub id: u8, +} + +impl ProfileLevel { + pub fn from_caps(s: &gst::StructureRef) -> anyhow::Result { + // Note: could use an AudioSpecificConfig based approach + // similar to what is done in gst_codec_utils_aac_get_level + // from gst-plugins-base/gst-libs/gst/pbutils/codec-utils.c + + use MPEG4AudioParserError::*; + + let profile = s.get::("profile").context("profile")?; + let level = s.get::("level").context("level")?; + + let id = match profile.to_lowercase().as_str() { + "lc" => { + // Assumed to be AAC Profile in table 1.14 + match level.as_str() { + "1" => 0x28, + "2" => 0x29, + "4" => 0x2a, + "5" => 0x2b, + _ => Err(UnsupportedLevel { + level: level.clone(), + profile: profile.clone(), + })?, + } + } + "he-aac" | "he-aac-v1" => { + // High Efficiency AAC Profile in table 1.14 + match level.as_str() { + "2" => 0x2c, + "3" => 0x2d, + "4" => 0x2e, + "5" => 0x2f, + _ => Err(UnsupportedLevel { + level: level.clone(), + profile: profile.clone(), + })?, + } + } + "he-aac-v2" => { + // High Efficiency AAC v2 Profile in table 1.14 + match level.as_str() { + "2" => 0x30, + "3" => 0x31, + "4" => 0x32, + "5" => 0x33, + _ => Err(UnsupportedLevel { + level: level.clone(), + profile: profile.clone(), + })?, + } + } + _ => Err(UnsupportedProfile { + profile: profile.clone(), + })?, + }; + + Ok(ProfileLevel { profile, level, id }) + } +} + +#[derive(Debug)] +pub struct Subframes<'a> { + frame: gst::MappedBuffer, + pos: usize, + subframe_idx: u8, + config: &'a StreamMuxConfig, +} + +impl<'a> Subframes<'a> { + pub fn new(frame: F, config: &'a StreamMuxConfig) -> Self + where + F: AsRef<[u8]> + Send + 'static, + { + Subframes { + frame: gst::Buffer::from_slice(frame) + .into_mapped_buffer_readable() + .unwrap(), + pos: 0, + subframe_idx: 0, + config, + } + } +} + +impl<'a> Iterator for Subframes<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + use MPEG4AudioParserError::*; + + if self.subframe_idx >= self.config.num_sub_frames { + return None; + } + + self.subframe_idx += 1; + + let mut data_len: usize; + let buf = &self.frame[self.pos..]; + + // PayloadLengthInfo - ISO/IEC 14496-3 sub 1 table 1.22 + // Assuming: + // * allStreamsSameTimeFraming == true + // * 1 prog & 1 layer + // * frameLengthType == 0 + + data_len = 0; + for byte in buf.iter() { + data_len += *byte as usize; + self.pos += 1; + if *byte != 0xff { + break; + } + } + + if data_len == 0 { + return Some(Err(ZeroLengthSubframe(self.subframe_idx))); + } + + if data_len > buf.len() { + return Some(Err(WrongFrameSize { + required: self.pos + data_len, + available: self.pos + buf.len(), + })); + } + + let mut subframe = self + .frame + .buffer() + .copy_region( + gst::BufferCopyFlags::MEMORY, + self.pos..(self.pos + data_len), + ) + .expect("Failed to create subbuffer"); + + let duration = (self.config.prog.frame_len as u64) + .mul_div_floor( + *gst::ClockTime::SECOND, + self.config.prog.sampling_freq as u64, + ) + .map(gst::ClockTime::from_nseconds); + + if let Some(duration) = duration { + subframe.get_mut().unwrap().set_duration(duration); + } + + self.pos += data_len; + + Some(Ok(subframe)) + } +} diff --git a/net/rtp/src/mp4a/pay/imp.rs b/net/rtp/src/mp4a/pay/imp.rs new file mode 100644 index 00000000..bc332b54 --- /dev/null +++ b/net/rtp/src/mp4a/pay/imp.rs @@ -0,0 +1,288 @@ +// GStreamer RTP MPEG-4 Audio Payloader +// +// Copyright (C) 2023 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-rtpmp4apay2 + * @see_also: rtpmp4apay2, rtpmp4apay, fdkaacenc + * + * Payload an MPEG-4 Audio bitstream into RTP packets as per [RFC 3016][rfc-3016]. + * Also see the [IANA media-type page for MPEG-4 Advanced Audio Coding][iana-aac]. + * + * [rfc-3016]: https://www.rfc-editor.org/rfc/rfc3016.html#section-4 + * [iana-aac]: https://www.iana.org/assignments/media-types/audio/aac + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 audiotestsrc ! fdkaacenc ! rtpmp4apay2 ! udpsink host=127.0.0.1 port=5004 + * ]| This will encode an audio test signal to AAC and then payload the encoded audio + * into RTP packets and send them out via UDP to localhost (IPv4) port 5004. + * You can use the #rtpmp4adepay2 or #rtpmp4adepay elements to depayload such a stream, and + * the #fdkaacdec element to decode the depayloaded stream. + * + * Since: plugins-rs-0.13.0 + */ +use bitstream_io::{BigEndian, BitRead, BitReader, BitWrite, BitWriter}; +use once_cell::sync::Lazy; +use smallvec::SmallVec; + +use gst::{glib, subclass::prelude::*}; + +use crate::basepay::{RtpBasePay2Ext, RtpBasePay2Impl}; + +use crate::mp4a::parsers::AudioSpecificConfig; +use crate::mp4a::ENCODING_NAME; + +#[derive(Default)] +pub struct RtpMpeg4AudioPay; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpmp4apay2", + gst::DebugColorFlags::empty(), + Some("RTP MPEG-4 Audio Payloader"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for RtpMpeg4AudioPay { + const NAME: &'static str = "GstRtpMpeg4AudioPay"; + type Type = super::RtpMpeg4AudioPay; + type ParentType = crate::basepay::RtpBasePay2; +} + +impl ObjectImpl for RtpMpeg4AudioPay {} +impl GstObjectImpl for RtpMpeg4AudioPay {} + +impl ElementImpl for RtpMpeg4AudioPay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP MPEG-4 Audio Payloader", + "Codec/Payloader/Network/RTP", + "Payload an MPEG-4 Audio bitstream (e.g. AAC) into RTP packets (RFC 3016)", + "François Laignel ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("framed", true) + .field("stream-format", "raw") + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("payload", gst::IntRange::new(96i32, 127i32)) + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", ENCODING_NAME) + /* All optional parameters + * + * "profile-level-id=[1,MAX]" + * "cpresent=" + * "config=" + */ + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +#[derive(Debug)] +struct ConfigWithCodecData { + audio_config: AudioSpecificConfig, + config_data: SmallVec<[u8; 4]>, +} + +impl ConfigWithCodecData { + fn from_codec_data(s: &gst::StructureRef) -> anyhow::Result { + use anyhow::Context; + + let codec_data = s + .get::("codec_data") + .context("codec_data field")?; + let codec_data_ref = codec_data.map_readable().context("mapping codec_data")?; + + if codec_data_ref.size() != 2 { + anyhow::bail!("Unsupported size {} for codec_data", codec_data_ref.size()); + } + + let mut r = BitReader::endian(codec_data_ref.as_slice(), BigEndian); + let audio_config = r.parse::()?; + + let mut config_data = SmallVec::new(); + let mut w = BitWriter::endian(&mut config_data, BigEndian); + + // StreamMuxConfig - ISO/IEC 14496-3 sub 1 table 1.21 + // audioMuxVersion == 0 (1 bit) + // allStreamsSameTimeFraming == 1 (1 bit) + // numSubFrames == 0 means 1 subframe (6 bits) + // numProgram == 0 means 1 program (4 bits) + // numLayer == 0 means 1 layer (3 bits) + + w.write(1, 0).unwrap(); + w.write_bit(true).unwrap(); + w.write(13, 0).unwrap(); + // 1 bit missing for byte alignment + + // Append AudioSpecificConfig for prog 1 layer 1 (from codec_data) + for byte in codec_data_ref.as_slice() { + w.write(8, *byte).context("appending codec_data")? + } + + // Padding + w.write(7, 0).unwrap(); + + Ok(ConfigWithCodecData { + audio_config, + config_data, + }) + } +} + +impl RtpBasePay2Impl for RtpMpeg4AudioPay { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio"]; + + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let (config, config_data) = match ConfigWithCodecData::from_codec_data(s) { + Ok(c) => (c.audio_config, c.config_data), + Err(err) => { + gst::error!(CAT, imp: self, "Unusable codec_data: {err:#}"); + return false; + } + }; + + let rate = if let Ok(rate) = s.get::("rate") { + rate + } else { + config.sampling_freq as i32 + }; + + self.obj().set_src_caps( + &gst::Caps::builder("application/x-rtp") + .field("media", "audio") + .field("encoding-name", ENCODING_NAME) + .field("clock-rate", rate) + .field("profile-level-id", config.audio_object_type) + .field("cpresent", 0) + .field("config", hex::encode(config_data)) + .build(), + ); + + true + } + + // Encapsulation of MPEG-4 Audio bitstream: + // https://www.rfc-editor.org/rfc/rfc3016.html#section-4 + // + // We either put 1 whole AAC frame into a single RTP packet, + // or fragment a single AAC frame over multiple RTP packets. + // + fn handle_buffer( + &self, + buffer: &gst::Buffer, + id: u64, + ) -> Result { + if buffer.size() == 0 { + gst::info!(CAT, imp: self, "Dropping empty buffer {id}"); + self.obj().drop_buffers(..=id); + + return Ok(gst::FlowSuccess::Ok); + } + + let Ok(buffer_ref) = buffer.map_readable() else { + gst::error!(CAT, imp: self, "Failed to map buffer {id} readable"); + + return Err(gst::FlowError::Error); + }; + + let max_payload_size = self.obj().max_payload_size() as usize; + + let mut size_prefix = SmallVec::<[u8; 3]>::new(); + let mut rem_size = buffer_ref.size(); + while rem_size > 0xff { + size_prefix.push(0xff); + rem_size >>= 8; + } + size_prefix.push(rem_size as u8); + + if max_payload_size < size_prefix.len() { + gst::error!( + CAT, + imp: self, + "Insufficient max-payload-size {} for buffer {id} at least {} bytes needed", + self.obj().max_payload_size(), + size_prefix.len() + 1, + ); + self.obj().drop_buffers(..=id); + + return Err(gst::FlowError::Error); + } + + let mut rem_data = buffer_ref.as_slice(); + let mut is_first = true; + while !rem_data.is_empty() { + let mut packet = rtp_types::RtpPacketBuilder::new(); + + let chunk_size = if is_first { + packet = packet.payload(size_prefix.as_slice()); + + std::cmp::min(rem_data.len(), max_payload_size - size_prefix.len()) + } else { + std::cmp::min(rem_data.len(), max_payload_size) + }; + + let payload = &rem_data[..chunk_size]; + rem_data = &rem_data[chunk_size..]; + + // The marker bit indicates audioMuxElement boundaries. + // It is set to one to indicate that the RTP packet contains a complete + // audioMuxElement or the last fragment of an audioMuxElement. + let marker = rem_data.is_empty(); + + gst::log!(CAT, imp: self, "Queuing {}packet with size {} for {}buffer {id}", + if marker { "marked " } else { "" }, + payload.len(), + if !marker || !is_first { "fragmented " } else { "" }, + ); + + self.obj() + .queue_packet(id.into(), packet.payload(payload).marker_bit(marker))?; + + is_first = false; + } + + self.obj().finish_pending_packets()?; + + Ok(gst::FlowSuccess::Ok) + } +} diff --git a/net/rtp/src/mp4a/pay/mod.rs b/net/rtp/src/mp4a/pay/mod.rs new file mode 100644 index 00000000..ee3b512c --- /dev/null +++ b/net/rtp/src/mp4a/pay/mod.rs @@ -0,0 +1,28 @@ +// GStreamer RTP MPEG-4 Audio Payloader +// +// Copyright (C) 2023 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +pub mod imp; + +glib::wrapper! { + pub struct RtpMpeg4AudioPay(ObjectSubclass) + @extends crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "rtpmp4apay2", + gst::Rank::MARGINAL, + RtpMpeg4AudioPay::static_type(), + ) +} diff --git a/net/rtp/src/mp4a/tests.rs b/net/rtp/src/mp4a/tests.rs new file mode 100644 index 00000000..cc24e078 --- /dev/null +++ b/net/rtp/src/mp4a/tests.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: MPL-2.0 + +use crate::tests::{run_test_pipeline, ExpectedBuffer, ExpectedPacket, Source}; +use gst::prelude::*; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + crate::plugin_register_static().expect("rtpmp4a test"); + }); +} + +#[test] +fn mp4a_one_frame_per_packet() { + init(); + + let src = "audiotestsrc num-buffers=100 ! audio/x-raw,rate=48000,channels=2 ! fdkaacenc"; + let pay = "rtpmp4apay2"; + let depay = "rtpmp4adepay2"; + + let mut expected_pay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::MARKER + }) + .rtp_time((position & 0xffff_ffff) as u32) + .build()]); + } + + let mut expected_depay = Vec::with_capacity(101); + for i in 0..101 { + let position = (i + 1) * 1024; + + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} + +#[test] +fn mp4a_fragmented() { + init(); + + let src = "audiotestsrc num-buffers=100 ! audio/x-raw,rate=48000,channels=1 ! fdkaacenc"; + let pay = "rtpmp4apay2 mtu=288"; + let depay = "rtpmp4adepay2"; + + let mut expected_pay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + let pts = gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + ); + let rtp_time = (position & 0xffff_ffff) as u32; + + expected_pay.push(vec![ + ExpectedPacket::builder() + .pts(pts) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .rtp_time(rtp_time) + .marker_bit(false) + .build(), + ExpectedPacket::builder() + .pts(pts) + .flags(gst::BufferFlags::MARKER) + .rtp_time(rtp_time) + .build(), + ]); + } + + let mut expected_depay = Vec::with_capacity(101); + for i in 0..101 { + let position = (i + 1) * 1024; + + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} diff --git a/net/rtp/src/mp4g/depay/deint_buf.rs b/net/rtp/src/mp4g/depay/deint_buf.rs new file mode 100644 index 00000000..b93f1454 --- /dev/null +++ b/net/rtp/src/mp4g/depay/deint_buf.rs @@ -0,0 +1,748 @@ +//! Access Unit Deinterleaving Buffer +use slab::Slab; + +use super::{AccessUnit, AccessUnitIndex, MaybeSingleAuOrList, Mpeg4GenericDepayError}; + +#[derive(Debug)] +struct AuNode { + au: AccessUnit, + /// Index of the next AuNode in the early_aus buffer + next: Option, +} + +/// Access Unit Deinterleaving Buffer. +/// +/// In some packet modes, non-consecutive AUs might be grouped together, +/// which can limit the gap between to AUs in case of packet loss. +/// +/// The Deinterleaving Buffer collects AUs as they arrive and outputs +/// them in the expected order whenever possible. +/// +/// See [Interleaving in RFC 3640](rfc-interleaving). +/// +/// [rfc-interleaving]: https://www.rfc-editor.org/rfc/rfc3640.html#section-3.2.3.2 +#[derive(Debug, Default)] +pub struct DeinterleaveAuBuffer { + /// Linked list of the early AUs + early_aus: Slab, + /// Index of the head in early_aus buffer + head: Option, + expected_index: Option, +} + +impl DeinterleaveAuBuffer { + pub fn new(max_displacement: u32) -> Self { + DeinterleaveAuBuffer { + early_aus: Slab::with_capacity(max_displacement as usize), + ..Default::default() + } + } + + pub fn drain(&mut self) -> MaybeSingleAuOrList { + self.expected_index = None; + + let mut cur_opt = self.head.take(); + + let len = self.early_aus.len(); + match len { + 0 => return MaybeSingleAuOrList::default(), + 1 => { + let node = self.early_aus.remove(cur_opt.unwrap()); + return MaybeSingleAuOrList::from(node.au); + } + _ => (), + } + + let mut list = MaybeSingleAuOrList::new_list(len); + + while let Some(cur) = cur_opt { + let cur_node = self.early_aus.remove(cur); + list.push(cur_node.au); + + cur_opt = cur_node.next; + } + + list + } + + pub fn flush(&mut self) { + self.early_aus.clear(); + self.head = None; + self.expected_index = None; + } + + #[track_caller] + pub fn push_and_pop( + &mut self, + au: AccessUnit, + outbuf: &mut MaybeSingleAuOrList, + ) -> Result<(), Mpeg4GenericDepayError> { + use std::cmp::Ordering::*; + + let mut expected_index = match self.expected_index { + Some(expected_index) => match au.index.try_cmp(expected_index)? { + Equal => expected_index, + Greater => return self.insert_au(au), + Less => { + // Dropping too early Au + return Err(Mpeg4GenericDepayError::TooEarlyAU { + index: au.index, + expected_index, + }); + } + }, + None => au.index, // first AU + }; + + outbuf.push(au); + + expected_index += 1; + self.expected_index = Some(expected_index); + + // Pop other ready AUs if any + let mut head; + let mut head_node_ref; + let mut head_node; + while !self.early_aus.is_empty() { + head = self.head.expect("!early_aus.is_empty"); + + head_node_ref = self.early_aus.get(head).unwrap(); + if head_node_ref.au.index.try_cmp(expected_index)?.is_ne() { + break; + } + + head_node = self.early_aus.remove(head); + outbuf.push(head_node.au); + + expected_index += 1; + self.expected_index = Some(expected_index); + + self.head = head_node.next; + } + + Ok(()) + } + + fn insert_au(&mut self, au: AccessUnit) -> Result<(), Mpeg4GenericDepayError> { + use std::cmp::Ordering::*; + + if self.early_aus.is_empty() { + self.head = Some(self.early_aus.insert(AuNode { au, next: None })); + + // Nothing to pop + return Ok(()); + } + + let mut cur = self.head.expect("!early_aus.is_empty"); + let mut cur_node = self.early_aus.get(cur).unwrap(); + + // cur & cur_node refer to current head here + match au.index.try_cmp(cur_node.au.index)? { + Greater => (), + Less => { + // New head + self.head = Some(self.early_aus.insert(AuNode { + au, + next: Some(cur), + })); + + return Ok(()); + } + Equal => { + // Duplicate + // RFC, §2.3: + // > In addition, an AU MUST NOT be repeated in other RTP packets; hence + // > repetition of an AU is only possible when using a duplicate RTP packet. + // + // But: we can't received duplicates because they would have been rejected + // by the base class or the jitterbuffer. + unreachable!(); + } + } + + // Upcoming AU is not then new head + + loop { + let Some(next) = cur_node.next else { + let new = Some(self.early_aus.insert(AuNode { au, next: None })); + self.early_aus.get_mut(cur).unwrap().next = new; + + return Ok(()); + }; + + let next_node = self.early_aus.get(next).unwrap(); + + match au.index.try_cmp(next_node.au.index)? { + Greater => (), // try next node + Less => { + let new = self.early_aus.insert(AuNode { + au, + next: Some(next), + }); + self.early_aus.get_mut(cur).unwrap().next = Some(new); + + return Ok(()); + } + Equal => { + // Duplicate + // RFC, §2.3: + // > In addition, an AU MUST NOT be repeated in other RTP packets; hence + // > repetition of an AU is only possible when using a duplicate RTP packet. + // + // But: we can't received duplicates because they would have been rejected + // by the base class or the jitterbuffer. + unreachable!(); + } + } + + cur = next; + cur_node = next_node; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::mp4g::depay::SingleAuOrList; + + impl From for AccessUnit { + fn from(index: u32) -> Self { + AccessUnit { + index: index.into(), + ..Default::default() + } + } + } + + #[test] + fn simple_group_interleave() { + // Tests the pattern illustrated in: + // https://www.rfc-editor.org/rfc/rfc3640.html#appendix-A.3 + + gst::init().unwrap(); + + let mut deint_buf = DeinterleaveAuBuffer::default(); + assert!(deint_buf.early_aus.is_empty()); + assert!(deint_buf.expected_index.is_none()); + + let mut outbuf = MaybeSingleAuOrList::default(); + assert!(outbuf.0.is_none()); + + // **** + // * P0. AUs with indices: 0, 3 & 6 + + // Expected AU 0 so it is pushed to outbuf + deint_buf.push_and_pop(0.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 1 is missing when pushing AU 3 so it is buffered + deint_buf.push_and_pop(3.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 1 is missing when pushing AU 6 so it is buffered + deint_buf.push_and_pop(6.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + + // **** + // * P1. AUs with indices: 1, 4 & 7 + + // Expected AU 1 so it is pushed to outbuf + deint_buf.push_and_pop(1.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 2); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 2 is missing when pushing AU 4 so it is buffered + deint_buf.push_and_pop(4.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 2); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 2 is missing when pushing AU 7 so it is buffered + deint_buf.push_and_pop(7.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 4); + assert_eq!(deint_buf.expected_index.unwrap(), 2); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + + // **** + // * P2. AUs with indices: 2, 5 & 8 + + // Expected AU 2 so it is pushed to outbuf + // and this also pops AUs 3 & 4 + // Remaining: 6 & 7 + deint_buf.push_and_pop(2.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 5); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 3); + + // Expected AU 5 so it is pushed to outbuf + // and this also pops AUs 6 & 7 + deint_buf.push_and_pop(5.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 8); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 6); + + // Expected AU 8 so it is pushed to outbuf + deint_buf.push_and_pop(8.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 9); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 7); + assert!(outbuf.0.is_none()); + + // **** + // * P3. AUs with indices: 9, 12 & 15 + + // Expected AU 9 so it is pushed to outbuf + deint_buf.push_and_pop(9.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 10); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 10 is missing when pushing AU 12 so it is buffered + deint_buf.push_and_pop(12.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 10); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 10 is missing when pushing AU 15 so it is buffered + deint_buf.push_and_pop(15.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 10); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + } + + #[test] + fn more_subtle_group_interleave() { + // Tests the pattern illustrated in: + // https://www.rfc-editor.org/rfc/rfc3640.html#appendix-A.4 + + gst::init().unwrap(); + + let mut deint_buf = DeinterleaveAuBuffer::default(); + let mut outbuf = MaybeSingleAuOrList::default(); + + // **** + // * P0. AUs with indices: 0 & 5 + + // Expected AU 0 so it is pushed to outbuf + deint_buf.push_and_pop(0.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 1 is missing when pushing AU 5 so it is buffered + deint_buf.push_and_pop(5.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + + // **** + // * P1. AUs with indices: 2 & 7 + + // Expected AU 1 is missing when pushing AU 2 so it is buffered + deint_buf.push_and_pop(2.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + assert!(outbuf.0.is_none()); + + // Expected AU 1 is missing when pushing AU 7 so it is buffered + deint_buf.push_and_pop(7.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + + // End of the RTP packet + assert!(outbuf.take().is_none()); + + // **** + // * P2. AUs with indices: 4 & 9 + + // Expected AU 1 is missing when pushing AU 4 so it is buffered + deint_buf.push_and_pop(4.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 4); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + assert!(outbuf.0.is_none()); + + // Expected AU 1 is missing when pushing AU 9 so it is buffered + deint_buf.push_and_pop(9.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 5); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + + // End of the RTP packet + assert!(outbuf.take().is_none()); + + // **** + // * P3. AUs with indices: 1 & 6 + + // Expected AU 1 so it is pushed to outbuf + // and this also pops AU 2 + // Remaining: 4, 5, 7 & 9 + deint_buf.push_and_pop(1.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 4); + assert_eq!(deint_buf.expected_index.unwrap(), 3); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 2); + + // Expected AU 3 is missing when pushing AU 6 so it is buffered + deint_buf.push_and_pop(6.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 5); + assert_eq!(deint_buf.expected_index.unwrap(), 3); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 2); + assert!(outbuf.0.is_none()); + + // **** + // * P4. AUs with indices: 3 & 8 + + // Expected AU 3 so it is pushed to outbuf + // and this also pops AU 4, 5, 6 & 7 + // Remaining: 9 + deint_buf.push_and_pop(3.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 8); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 5); + + // Expected AU 8 so it is pushed to outbuf + // and this also pops AU 9 + deint_buf.push_and_pop(8.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 10); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 7); + assert!(outbuf.0.is_none()); + + // **** + // * P5. AUs with indices: 10 & 15 + + // Expected AU 10 so it is pushed to outbuf + deint_buf.push_and_pop(10.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 11); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 11 is missing when pushing AU 15 so it is buffered + deint_buf.push_and_pop(15.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 11); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + } + + #[test] + fn continuous_interleave() { + // Tests the pattern illustrated in: + // https://www.rfc-editor.org/rfc/rfc3640.html#appendix-A.5 + + gst::init().unwrap(); + + let mut deint_buf = DeinterleaveAuBuffer::default(); + let mut outbuf = MaybeSingleAuOrList::default(); + + // **** + // * P0. AUs with index: 0 + + // Expected AU 0 so it is pushed to outbuf + deint_buf.push_and_pop(0.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 1); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.0.is_none()); + + // **** + // * P1. AUs with indices: 1 & 4 + + // Expected AU 0 so it is pushed to outbuf + deint_buf.push_and_pop(1.into(), &mut outbuf).unwrap(); + + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 2); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 2 is missing when pushing AU 4 so it is buffered + deint_buf.push_and_pop(4.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 2); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.take().is_none()); + + // **** + // * P2. AUs with indices: 2, 5 & 8 + + // Expected AU 2 so it is pushed to outbuf + deint_buf.push_and_pop(2.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 3); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 3 is missing when pushing AU 5 so it is buffered + deint_buf.push_and_pop(5.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 3); + matches!(outbuf.0, Some(SingleAuOrList::Single(_))); + + // Expected AU 3 is missing when pushing AU 8 so it is buffered + deint_buf.push_and_pop(8.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 3); + + // End of the RTP packet + matches!(outbuf.take(), Some(SingleAuOrList::Single(_))); + assert!(outbuf.take().is_none()); + + // **** + // * P3. AUs with indices: 3, 6, 9 & 12 + + // Expected AU 3 so it is pushed to outbuf + // and this also pops AU 4 & 5 + // Remaining: 8 + deint_buf.push_and_pop(3.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 6); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 3); + + // Expected AU 6 so it is pushed to outbuf + deint_buf.push_and_pop(6.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 7); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + + // Expected AU 7 is missing when pushing AU 9 so it is buffered + deint_buf.push_and_pop(9.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 7); + matches!(outbuf.0, Some(SingleAuOrList::List(_))); + + // Expected AU 7 is missing when pushing AU 12 so it is buffered + deint_buf.push_and_pop(12.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 7); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + assert!(outbuf.0.is_none()); + + // **** + // * P4. AUs with indices: 7, 10, 13 & 16 + + // Expected AU 7 so it is pushed to outbuf + // and this also pops AU 8 & 9 + // Remaining: 12 + deint_buf.push_and_pop(7.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 10); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 3); + + // Expected AU 10 so it is pushed to outbuf + deint_buf.push_and_pop(10.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 11); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + + // Expected AU 11 is missing when pushing AU 13 so it is buffered + deint_buf.push_and_pop(13.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 11); + matches!(outbuf.0, Some(SingleAuOrList::List(_))); + + // Expected AU 11 is missing when pushing AU 16 so it is buffered + deint_buf.push_and_pop(16.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 11); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + assert!(outbuf.0.is_none()); + + // **** + // * P5. AUs with indices: 11, 14, 17 & 20 + + // Expected AU 11 so it is pushed to outbuf + // and this also pops AU 12 & 13 + // Remaining: 16 + deint_buf.push_and_pop(11.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 14); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 3); + + // Expected AU 14 so it is pushed to outbuf + deint_buf.push_and_pop(14.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 15); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + + // Expected AU 15 is missing when pushing AU 17 so it is buffered + deint_buf.push_and_pop(17.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 2); + assert_eq!(deint_buf.expected_index.unwrap(), 15); + matches!(outbuf.0, Some(SingleAuOrList::List(_))); + + // Expected AU 15 is missing when pushing AU 20 so it is buffered + deint_buf.push_and_pop(20.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 3); + assert_eq!(deint_buf.expected_index.unwrap(), 15); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + assert!(outbuf.0.is_none()); + + // **** + // * P6. AUs with indices: 15 & 18 + + // Expected AU 15 so it is pushed to outbuf + // and this also pops AU 16 & 17 + // Remaining: 20 + deint_buf.push_and_pop(15.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 18); + let Some(SingleAuOrList::List(ref buflist)) = outbuf.0 else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 3); + + // Expected AU 18 so it is pushed to outbuf + deint_buf.push_and_pop(18.into(), &mut outbuf).unwrap(); + + assert_eq!(deint_buf.early_aus.len(), 1); + assert_eq!(deint_buf.expected_index.unwrap(), 19); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 4); + assert!(outbuf.0.is_none()); + + // **** + // * P7. AUs with index: 19 + deint_buf.push_and_pop(19.into(), &mut outbuf).unwrap(); + + // Expected AU 19 so it is pushed to outbuf + // and this also pops AU 20 + assert!(deint_buf.early_aus.is_empty()); + assert_eq!(deint_buf.expected_index.unwrap(), 21); + + // End of the RTP packet + let Some(SingleAuOrList::List(ref buflist)) = outbuf.take() else { + panic!("Expecting a List"); + }; + assert_eq!(buflist.len(), 2); + assert!(outbuf.0.is_none()); + } +} diff --git a/net/rtp/src/mp4g/depay/imp.rs b/net/rtp/src/mp4g/depay/imp.rs new file mode 100644 index 00000000..2d1da783 --- /dev/null +++ b/net/rtp/src/mp4g/depay/imp.rs @@ -0,0 +1,641 @@ +// GStreamer RTP MPEG-4 Generic elementary streams Depayloader +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-rtpmp4gdepay2 + * @see_also: rtpmp4gpay2, rtpmp4gdepay, rtpmp4gpay + * + * Depayload an MPEG-4 Generic elementary stream from RTP packets as per [RFC 3640][rfc-3640]. + * + * [rfc-3640]: https://www.rfc-editor.org/rfc/rfc3640.html#section-4 + * + * ## Example pipeline + * + * |[ + * gst-launch-1.0 udpsrc caps='application/x-rtp,media=audio,clock-rate=44100,encoding-name=MPEG4-GENERIC,payload=96,encoding-params=1,streamtype=5,profile-level-id=2,mode=AAC-hbr,config=(string)1208,sizelength=13,indexlength=3,indexdeltalength=3' ! rtpjitterbuffer ! rtpmp4gdepay2 ! decodebin3 ! audioconvert ! audioresample ! autoaudiosink + * ]| This will depayload an incoming RTP MPEG-4 generic elementary stream AAC-hbr with + * 1 channel @ 44100 sampling rate (default `audiotestsrc ! fdkaacenc` negotiation). + * You can use the #rtpmp4gpay2 or #rtpmp4gpay elements to create such an RTP stream. + * + * Since: plugins-rs-0.13.0 + */ +use anyhow::Context; +use atomic_refcell::AtomicRefCell; +use once_cell::sync::Lazy; + +use gst::{glib, prelude::*, subclass::prelude::*}; + +use std::ops::{ControlFlow, RangeInclusive}; + +use crate::basedepay::{Packet, PacketToBufferRelation, RtpBaseDepay2Ext, TimestampOffset}; + +use crate::mp4g::{ModeConfig, RtpTimestamp}; + +use super::parsers::PayloadParser; +use super::{ + AccessUnit, DeinterleaveAuBuffer, MaybeSingleAuOrList, Mpeg4GenericDepayError, SingleAuOrList, +}; + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpmp4gdepay2", + gst::DebugColorFlags::empty(), + Some("RTP MPEG-4 generic Depayloader"), + ) +}); + +#[derive(Default)] +pub struct RtpMpeg4GenericDepay { + state: AtomicRefCell, +} + +#[glib::object_subclass] +impl ObjectSubclass for RtpMpeg4GenericDepay { + const NAME: &'static str = "GstRtpMpeg4GenericDepay"; + type Type = super::RtpMpeg4GenericDepay; + type ParentType = crate::basedepay::RtpBaseDepay2; +} + +impl ObjectImpl for RtpMpeg4GenericDepay {} + +impl GstObjectImpl for RtpMpeg4GenericDepay {} + +impl ElementImpl for RtpMpeg4GenericDepay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP MPEG-4 Generic ES Depayloader", + "Codec/Depayloader/Network/RTP", + "Depayload MPEG-4 Generic elementary streams from RTP packets (RFC 3640)", + "François Laignel ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder("application/x-rtp") + // TODO "application" is also present in rtpmp4gdepay caps template + // but it doesn't handle it in gst_rtp_mp4g_depay_setcaps + .field("media", gst::List::new(["audio", "video"])) + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", "MPEG4-GENERIC") + // Required string params: + // "streamtype = { \"4\", \"5\" }, " Not set by Wowza 4 = video, 5 = audio + // "profile-level-id = [1,MAX], " + // "config = (string)" + .field( + "mode", + gst::List::new(["generic", "AAC-lbr", "AAC-hbr", "aac-hbr"]), + ) + // Optional general parameters: + // "objecttype = [1,MAX], " + // "constantsize = [1,MAX], " // constant size of each AU + // "constantduration = [1,MAX], " // constant duration of each AU + // "maxdisplacement = [1,MAX], " + // "de-interleavebuffersize = [1,MAX], " + // Optional configuration parameters: + // "sizelength = [1, 32], " + // "indexlength = [1, 32], " + // "indexdeltalength = [1, 32], " + // "ctsdeltalength = [1, 32], " + // "dtsdeltalength = [1, 32], " + // "randomaccessindication = {0, 1}, " + // "streamstateindication = [0, 32], " + // "auxiliarydatasizelength = [0, 32]" ) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false) + .build(), + ) + .structure( + gst::Structure::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("stream-format", "raw") + .build(), + ) + .build(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +#[derive(Debug, Default)] +struct State { + parser: PayloadParser, + deint_buf: Option, + au_acc: Option, + seqnum_base: Option, + clock_rate: u32, + can_parse: bool, + max_au_index: Option, + prev_au_index: Option, + prev_rtptime: Option, + last_au_index: Option, +} + +impl State { + fn flush(&mut self) { + self.parser.reset(); + + if let Some(deint_buf) = self.deint_buf.as_mut() { + deint_buf.flush(); + } + self.can_parse = false; + self.max_au_index = None; + self.prev_au_index = None; + self.prev_rtptime = None; + self.last_au_index = None; + } +} + +struct CodecData; +impl CodecData { + fn from_caps(s: &gst::StructureRef) -> anyhow::Result> { + let conf_str = s.get_optional::<&str>("config").context("config field")?; + let Some(conf_str) = conf_str else { + return Ok(None); + }; + let data = hex::decode(conf_str).context("decoding config")?; + + Ok(Some(gst::Buffer::from_mut_slice(data))) + } +} + +/// Accumulates packets for a fragmented AU. +/// +/// Used for packets containing fragments for a single AU. +/// +/// From https://www.rfc-editor.org/rfc/rfc3640.html#section-3.2.3: +/// +/// > The Access Unit Data Section contains an integer number of complete +/// > Access Units or a single fragment of one AU. +#[derive(Debug)] +struct AuAccumulator(AccessUnit); + +impl AuAccumulator { + #[inline] + fn new(au: AccessUnit) -> Self { + AuAccumulator(au) + } + #[inline] + fn try_append(&mut self, mut au: AccessUnit) -> Result<(), Mpeg4GenericDepayError> { + use Mpeg4GenericDepayError::*; + + // FIXME add comment about fragments having the same RTP timestamp + if self.0.cts_delta.opt_ne(au.cts_delta).unwrap_or(false) { + return Err(FragmentedAuRtpTsMismatch { + expected: self.0.cts_delta.unwrap(), + found: au.cts_delta.unwrap(), + ext_seqnum: au.ext_seqnum, + }); + } + + if self.0.dts_delta.opt_ne(au.dts_delta).unwrap_or(false) { + // § 3.2.1.1 + // > The DTS-delta field MUST have the same value + // > for all fragments of an Access Unit + return Err(FragmentedAuDtsMismatch { + expected: self.0.dts_delta.unwrap(), + found: au.dts_delta.unwrap(), + ext_seqnum: au.ext_seqnum, + }); + } + + self.0.data.append(&mut au.data); + + Ok(()) + } + + #[inline] + fn try_into_au(self) -> Result { + let au = self.0; + if let Some(expected) = au.size { + if expected as usize != au.data.len() { + return Err(Mpeg4GenericDepayError::FragmentedAuSizeMismatch { + expected, + found: au.data.len(), + ext_seqnum: au.ext_seqnum, + }); + } + } + + Ok(au) + } +} + +impl crate::basedepay::RtpBaseDepay2Impl for RtpMpeg4GenericDepay { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio", "video"]; + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + + Ok(()) + } + + fn drain(&self) -> Result { + let mut state = self.state.borrow_mut(); + + if let Some(ref mut deint_buf) = state.deint_buf { + if let Some(aus) = deint_buf.drain().take() { + self.finish_buffer_or_list(&state, None, aus)?; + } + } + + Ok(gst::FlowSuccess::Ok) + } + + fn flush(&self) { + gst::debug!(CAT, imp: self, "Flushing"); + self.state.borrow_mut().flush(); + } + + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let mode = s.get::<&str>("mode").expect("Required by Caps"); + if mode.starts_with("CELP") { + gst::error!(CAT, imp: self, "{mode} not supported yet"); + return false; + } + + let mut caps_builder = match s.get::<&str>("media").expect("Required by Caps") { + "audio" => gst::Caps::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("stream-format", "raw"), + "video" => gst::Caps::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false), + // TODO handle "application" + _ => unreachable!(), + }; + + let mode_config = match ModeConfig::from_caps(s) { + Ok(h) => h, + Err(err) => { + gst::error!(CAT, imp: self, "Error parsing Header in Caps: {err:#}"); + return false; + } + }; + + match CodecData::from_caps(s) { + Ok(codec_data) => { + caps_builder = caps_builder.field("codec_data", codec_data); + } + Err(err) => { + gst::error!(CAT, imp: self, "Error parsing Caps: {err:#}"); + return false; + } + } + + let clock_rate = s.get::("clock-rate").expect("Required by Caps"); + debug_assert!(clock_rate.is_positive()); // constrained by Caps + let clock_rate = clock_rate as u32; + + { + let mut state = self.state.borrow_mut(); + state.seqnum_base = s.get_optional::("seqnum-base").unwrap(); + if let Some(seqnum_base) = state.seqnum_base { + gst::info!(CAT, imp: self, "Got seqnum_base {seqnum_base}"); + } + state.clock_rate = clock_rate; + + if let Some(max_displacement) = mode_config.max_displacement() { + state.deint_buf = Some(DeinterleaveAuBuffer::new(max_displacement)); + } + + state.parser.set_config(mode_config); + } + + self.obj().set_src_caps(&caps_builder.build()); + + true + } + + fn handle_packet( + &self, + packet: &crate::basedepay::Packet, + ) -> Result { + let mut state = self.state.borrow_mut(); + + if self.check_initial_packet(&mut state, packet).is_break() { + self.obj().drop_packets(..=packet.ext_seqnum()); + return Ok(gst::FlowSuccess::Ok); + } + + let State { + parser, + au_acc, + deint_buf, + .. + } = &mut *state; + + let payload = packet.payload(); + let ext_seqnum = packet.ext_seqnum(); + let packet_ts = RtpTimestamp::from_ext(packet.ext_timestamp()); + let au_iter = match parser.parse(payload, ext_seqnum, packet_ts) { + Ok(au_iter) => au_iter, + Err(err) => { + gst::warning!(CAT, imp: self, "Failed to parse payload for packet {ext_seqnum}: {err:#}"); + *au_acc = None; + self.obj().drop_packets(..=packet.ext_seqnum()); + + return Ok(gst::FlowSuccess::Ok); + } + }; + + let mut aus = MaybeSingleAuOrList::default(); + for au in au_iter { + let au = match au { + Ok(au) => au, + Err(err) => { + gst::warning!(CAT, imp: self, + "Failed to parse AU from packet {}: {err:#}", packet.ext_seqnum(), + ); + + continue; + } + }; + + // § 3.1: The marker indicates that: + // > the RTP packet payload contains either the final fragment of + // > a fragmented Access Unit or one or more complete Access Units + if !packet.marker_bit() { + if !au.is_fragment { + gst::warning!(CAT, imp: self, "Dropping non fragmented AU {au} in un-marked packet"); + continue; + } + + if let Some(ref mut acc) = au_acc { + if let Err(err) = acc.try_append(au) { + gst::warning!(CAT, imp: self, "Discarding pending fragmented AU: {err}"); + *au_acc = None; + parser.reset(); + self.obj().drop_packets(..=packet.ext_seqnum()); + + return Ok(gst::FlowSuccess::Ok); + } + } else { + *au_acc = Some(AuAccumulator::new(au)); + } + + gst::trace!(CAT, imp: self, "Non-final fragment"); + + return Ok(gst::FlowSuccess::Ok); + } + + // Packet marker set + + let au = match au_acc.take() { + Some(mut acc) => { + if au.is_fragment { + if let Err(err) = acc.try_append(au) { + gst::warning!(CAT, imp: self, "Discarding pending fragmented AU: {err}"); + parser.reset(); + self.obj().drop_packets(..=packet.ext_seqnum()); + + return Ok(gst::FlowSuccess::Ok); + } + + match acc.try_into_au() { + Ok(au) => au, + Err(err) => { + gst::warning!(CAT, imp: self, "Discarding pending fragmented AU: {err}"); + let Mpeg4GenericDepayError::FragmentedAuSizeMismatch { .. } = err + else { + unreachable!(); + }; + parser.reset(); + self.obj().drop_packets(..=packet.ext_seqnum()); + + return Ok(gst::FlowSuccess::Ok); + } + } + } else { + gst::warning!(CAT, imp: self, + "Discarding pending fragmented AU {} due to incoming non fragmented AU {au}", + acc.0, + ); + self.obj().drop_packets(..au.ext_seqnum); + + au + } + } + None => au, + }; + + if let Some(ref mut deint_buf) = deint_buf { + if let Err(err) = deint_buf.push_and_pop(au, &mut aus) { + gst::warning!(CAT, imp: self, "Failed to push AU to deinterleave buffer: {err}"); + // The AU has been dropped, just keep going + // Packet will be dropped eventually + } + + continue; + } + + if au.is_interleaved { + // From gstrtpmp4gdepay.c:616: + // > some broken non-interleaved streams have AU-index jumping around + // > all over the place, apparently assuming receiver disregards + + gst::warning!(CAT, imp: self, "Interleaved AU, but no `max_displacement` was defined"); + } + + aus.push(au); + } + + if let Some(aus) = aus.take() { + self.finish_buffer_or_list(&state, Some(packet.ext_seqnum()), aus)?; + } + + Ok(gst::FlowSuccess::Ok) + } +} + +impl RtpMpeg4GenericDepay { + #[inline] + fn check_initial_packet(&self, state: &mut State, packet: &Packet) -> ControlFlow<()> { + if state.can_parse { + return ControlFlow::Continue(()); + } + + let seqnum = (packet.ext_seqnum() & 0xffff) as u16; + + if let Some(seqnum_base) = state.seqnum_base { + let seqnum_base = (seqnum_base & 0xffff) as u16; + + // Assume seqnum_base and the initial ext_seqnum are in the same cycle + // This should be guaranteed by the JitterBuffer + let delta = crate::utils::seqnum_distance(seqnum, seqnum_base); + + if delta == 0 { + gst::debug!(CAT, imp: self, + "Got initial packet {seqnum_base} @ ext seqnum {}", packet.ext_seqnum(), + ); + state.can_parse = true; + + return ControlFlow::Continue(()); + } + + if delta < 0 { + gst::log!(CAT, imp: self, + "Waiting for initial packet {seqnum_base}, got {seqnum} (ext seqnum {})", + packet.ext_seqnum(), + ); + + return ControlFlow::Break(()); + } + + gst::debug!(CAT, imp: self, + "Packet {seqnum} (ext seqnum {}) passed expected initial packet {seqnum_base}, will sync on next marker", + packet.ext_seqnum(), + ); + + state.seqnum_base = None; + } + + // Wait until a marked packet is found and start parsing from the next packet + if packet.marker_bit() { + gst::debug!(CAT, imp: self, + "Found first marked packet {seqnum} (ext seqnum {}). Will start parsing from next packet", + packet.ext_seqnum(), + ); + + assert!(state.au_acc.is_none()); + state.can_parse = true; + } else { + gst::log!(CAT, imp: self, + "First marked packet not found yet, skipping packet {seqnum} (ext seqnum {})", + packet.ext_seqnum(), + ); + } + + ControlFlow::Break(()) + } + + fn finish_buffer_or_list( + &self, + state: &State, + packet_ext_seqnum: Option, + aus: SingleAuOrList, + ) -> Result { + use SingleAuOrList::*; + + fn get_packet_to_buffer_relation( + au: &AccessUnit, + clock_rate: u32, + range: RangeInclusive, + ) -> PacketToBufferRelation { + if let Some((cts_delta, dts_delta)) = Option::zip(au.cts_delta, au.dts_delta) { + let pts_offset = gst::Signed::::from(cts_delta as i64) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .unwrap(); + let dts_offset = gst::Signed::::from(dts_delta as i64) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .unwrap(); + PacketToBufferRelation::SeqnumsWithOffset { + seqnums: range, + timestamp_offset: TimestampOffset::PtsAndDts(pts_offset, dts_offset), + } + } else if let Some(cts_delta) = au.cts_delta { + let pts_offset = gst::Signed::::from(cts_delta as i64) + .mul_div_floor(*gst::ClockTime::SECOND, clock_rate as u64) + .unwrap(); + PacketToBufferRelation::SeqnumsWithOffset { + seqnums: range, + timestamp_offset: TimestampOffset::Pts(pts_offset), + } + } else { + PacketToBufferRelation::Seqnums(range) + } + } + + match aus { + Single(au) => { + let range = if let Some(packet_ext_seqnum) = packet_ext_seqnum { + au.ext_seqnum..=packet_ext_seqnum + } else { + au.ext_seqnum..=au.ext_seqnum + }; + + let packet_to_buffer_relation = + get_packet_to_buffer_relation(&au, state.clock_rate, range); + + gst::trace!(CAT, imp: self, "Finishing AU buffer {packet_to_buffer_relation:?}"); + + let buffer = Self::new_buffer(au, state); + + self.obj().queue_buffer(packet_to_buffer_relation, buffer)?; + } + List(au_list) => { + for au in au_list { + let range = if let Some(packet_ext_seqnum) = packet_ext_seqnum { + au.ext_seqnum..=packet_ext_seqnum + } else { + au.ext_seqnum..=au.ext_seqnum + }; + + let packet_to_buffer_relation = + get_packet_to_buffer_relation(&au, state.clock_rate, range); + + gst::trace!(CAT, imp: self, "Finishing AU buffer {packet_to_buffer_relation:?}"); + + let buffer = Self::new_buffer(au, state); + + self.obj().queue_buffer(packet_to_buffer_relation, buffer)?; + } + } + } + + Ok(gst::FlowSuccess::Ok) + } + + #[inline] + fn new_buffer(au: AccessUnit, state: &State) -> gst::Buffer { + let mut buf = gst::Buffer::from_mut_slice(au.data); + let buf_mut = buf.get_mut().unwrap(); + + if au.maybe_random_access == Some(false) { + buf_mut.set_flags(gst::BufferFlags::DELTA_UNIT) + } + + if let Some(duration) = au.duration { + let duration = (duration as u64) + .mul_div_floor(*gst::ClockTime::SECOND, state.clock_rate as u64) + .map(gst::ClockTime::from_nseconds); + + buf_mut.set_duration(duration); + } + + buf + } +} diff --git a/net/rtp/src/mp4g/depay/mod.rs b/net/rtp/src/mp4g/depay/mod.rs new file mode 100644 index 00000000..f26528f6 --- /dev/null +++ b/net/rtp/src/mp4g/depay/mod.rs @@ -0,0 +1,185 @@ +// GStreamer RTP MPEG-4 generic elementary streams Depayloader +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +pub mod imp; +pub(crate) mod parsers; + +mod deint_buf; +pub(crate) use deint_buf::DeinterleaveAuBuffer; + +use gst::glib; +use gst::prelude::*; +use smallvec::SmallVec; + +use std::fmt; + +use crate::mp4g::{AccessUnitIndex, Mpeg4GenericError}; + +glib::wrapper! { + pub struct RtpMpeg4GenericDepay(ObjectSubclass) + @extends crate::basedepay::RtpBaseDepay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + gst::Element::register( + Some(plugin), + "rtpmp4gdepay2", + gst::Rank::MARGINAL, + RtpMpeg4GenericDepay::static_type(), + ) +} + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum Mpeg4GenericDepayError { + #[error("{}", .0)] + Mpeg4Generic(#[from] Mpeg4GenericError), + + #[error("AU header section too large: expected end {expected_end} / {total}")] + AuHeaderSectionTooLarge { expected_end: usize, total: usize }, + + #[error("AU auxiliary section too large: expected end {expected_end} / {total}")] + AuAuxiliarySectionTooLarge { expected_end: usize, total: usize }, + + #[error("Empty AU Data section")] + EmptyAuData, + + #[error("Unknown AU size, but multiple AUs in the packet")] + MultipleAusUnknownSize, + + #[error("Multiple AUs in packet but the AU size {au_size} is > AU data size {au_data_size}")] + MultipleAusGreaterSizeThanAuData { au_size: usize, au_data_size: usize }, + + #[error("No more AU data left for AU with index {index}")] + NoMoreAuDataLeft { index: AccessUnitIndex }, + + #[error("Got AU with index {index} which is earlier than the expected index {expected_index}")] + TooEarlyAU { + index: AccessUnitIndex, + expected_index: AccessUnitIndex, + }, + + #[error("Unexpected non-zero first AU index {index} in packet {ext_seqnum} due to configured constant duration")] + ConstantDurationAuNonZeroIndex { + index: AccessUnitIndex, + ext_seqnum: u64, + }, + + #[error("Constant duration not configured and no headers in packet {ext_seqnum}")] + NonConstantDurationNoAuHeaders { ext_seqnum: u64 }, + + #[error("Constant duration not configured and no CTS delta for AU index {index} in packet {ext_seqnum}")] + NonConstantDurationAuNoCtsDelta { + index: AccessUnitIndex, + ext_seqnum: u64, + }, + + #[error( + "Fragmented AU size mismatch: expected {expected}, found {found}. Packet {ext_seqnum}" + )] + FragmentedAuSizeMismatch { + expected: u32, + found: usize, + ext_seqnum: u64, + }, + + #[error("Fragmented AU CTS mismatch: expected {expected}, found {found}. Packet {ext_seqnum}")] + FragmentedAuRtpTsMismatch { + expected: i32, + found: i32, + ext_seqnum: u64, + }, + + #[error("Fragmented AU DTS mismatch: expected {expected}, found {found}. Packet {ext_seqnum}")] + FragmentedAuDtsMismatch { + expected: i32, + found: i32, + ext_seqnum: u64, + }, +} + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub enum SingleAuOrList { + Single(AccessUnit), + List(SmallVec<[AccessUnit; 5]>), +} + +impl SingleAuOrList { + pub fn new_list(capacity: usize) -> Self { + SingleAuOrList::List(SmallVec::with_capacity(capacity)) + } + + #[inline] + pub fn push(&mut self, au: AccessUnit) { + use SingleAuOrList::*; + match self { + Single(_) => { + let list = List(SmallVec::new()); + let prev = std::mem::replace(self, list); + let Single(prev) = prev else { unreachable!() }; + let List(list) = self else { unreachable!() }; + list.push(prev); + list.push(au); + } + List(list) => list.push(au), + } + } +} + +#[derive(Debug, Default)] +pub struct MaybeSingleAuOrList(Option); + +impl MaybeSingleAuOrList { + pub fn new_list(capacity: usize) -> Self { + MaybeSingleAuOrList(Some(SingleAuOrList::new_list(capacity))) + } + + pub fn push(&mut self, au: AccessUnit) { + match &mut self.0 { + Some(inner) => inner.push(au), + None => self.0 = Some(SingleAuOrList::Single(au)), + } + } + + #[inline] + pub fn take(&mut self) -> Option { + self.0.take() + } +} + +impl From for MaybeSingleAuOrList { + fn from(au: AccessUnit) -> Self { + MaybeSingleAuOrList(Some(SingleAuOrList::Single(au))) + } +} + +/// A parsed Access Unit. +/// +/// All timestamps and duration in clock rate ticks. +/// All timestamps are based on the RTP timestamp of the packet. +#[derive(Debug, Default)] +pub struct AccessUnit { + pub(crate) ext_seqnum: u64, + pub(crate) is_fragment: bool, + pub(crate) size: Option, + pub(crate) index: AccessUnitIndex, + pub(crate) cts_delta: Option, + pub(crate) dts_delta: Option, + pub(crate) duration: Option, + pub(crate) maybe_random_access: Option, + pub(crate) is_interleaved: bool, + pub(crate) data: Vec, +} + +impl fmt::Display for AccessUnit { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "index {} from Packet {}", self.index, self.ext_seqnum) + } +} diff --git a/net/rtp/src/mp4g/depay/parsers.rs b/net/rtp/src/mp4g/depay/parsers.rs new file mode 100644 index 00000000..42b74f0c --- /dev/null +++ b/net/rtp/src/mp4g/depay/parsers.rs @@ -0,0 +1,1050 @@ +use anyhow::Context; +use bitstream_io::{BigEndian, BitRead, BitReader, ByteRead, ByteReader}; + +use std::io::Cursor; + +use super::{AccessUnit, Mpeg4GenericDepayError}; +use crate::mp4g::{AccessUnitIndex, AuHeader, AuHeaderContext, ModeConfig, RtpTimestamp}; + +/// The reference Packet to compute constant duration when applicable. +#[derive(Debug, Default)] +struct ConstantDurationProbation { + ts: RtpTimestamp, + frames: u32, +} + +impl From for ConstantDurationProbation { + fn from(ts: RtpTimestamp) -> Self { + ConstantDurationProbation { ts, frames: 0 } + } +} + +/// MPEG-4 generic Payload: https://www.rfc-editor.org/rfc/rfc3640.html#section-2.11 +/// +/// +---------+-----------+-----------+---------------+ +/// | RTP | AU Header | Auxiliary | Access Unit | +/// | Header | Section | Section | Data Section | +/// +---------+-----------+-----------+---------------+ +/// +/// . <----------RTP Packet Payload-----------> +#[derive(Debug, Default)] +pub struct PayloadParser { + config: ModeConfig, + const_dur_probation: Option, + // Constant duration as provided by the config + // or determined while parsing the payloads + constant_duration: Option, +} + +impl PayloadParser { + #[allow(dead_code)] + pub fn new_for(config: ModeConfig) -> Self { + PayloadParser { + constant_duration: config.constant_duration(), + config, + ..Default::default() + } + } + + pub fn set_config(&mut self, config: ModeConfig) { + self.config = config; + self.reset(); + } + + pub fn reset(&mut self) { + self.constant_duration = self.config.constant_duration(); + self.const_dur_probation = None; + } + + pub fn parse<'a>( + &'a mut self, + payload: &'a [u8], + ext_seqnum: u64, + packet_ts: RtpTimestamp, + ) -> anyhow::Result> { + use Mpeg4GenericDepayError::*; + + let mut headers_len = 0; + let mut headers = None; + let mut data_offset = 0; + + let mut r = ByteReader::endian(payload, BigEndian); + + if self.config.has_header_section() { + // AU Header section: https://www.rfc-editor.org/rfc/rfc3640.html#section-3.2.1 + // + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+ + // |AU-headers-length|AU-header|AU-header| |AU-header|padding| + // | | (1) | (2) | | (n) * | bits | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+-+ + + // This is expressed in bits + headers_len = r.read::().context("AU-headers-length")?; + + // Up to 7 bits of padding + let headers_len_bytes = (headers_len as usize + 7) / 8; + + data_offset = 2 + headers_len_bytes; + if data_offset > payload.len() { + Err(AuHeaderSectionTooLarge { + expected_end: data_offset, + total: payload.len(), + })?; + } + + r.skip(headers_len_bytes as u32) + .expect("availability checked above"); + + headers = Some(&payload[2..data_offset]); + } else if self.constant_duration.is_none() { + // No headers and non-constant duration + + // § 3.2.3.2: + // > When transmitting Access Units of variable duration, then the + // > "constantDuration" parameter MUST NOT be present [...] + // > the CTS-delta MUST be coded in the AU header for each non-first AU + // > in the RTP packet + + Err(NonConstantDurationNoAuHeaders { ext_seqnum })?; + } + + if self.config.has_auxiliary_section() { + // Move the AU reader after the Auxiliary Section + // + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+ + // | auxiliary-data-size | auxiliary-data |padding bits | + // +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- .. -+-+-+-+-+-+-+-+-+ + + // This is expressed in bits + let aux_len = r.read::().context("auxiliary-data-size")?; + + // Up to 7 bits of padding + let aux_len_bytes = (aux_len as usize + 7) / 8; + + data_offset += 2 + aux_len_bytes; + if data_offset > payload.len() { + Err(AuAuxiliarySectionTooLarge { + expected_end: data_offset, + total: payload.len(), + })?; + } + } + + if data_offset >= payload.len() { + Err(EmptyAuData)?; + } + + Ok(AccessUnitIter { + parser: self, + ext_seqnum, + packet_ts, + prev_index: None, + headers_r: headers.map(|h| BitReader::endian(Cursor::new(h), BigEndian)), + headers_len, + data: &payload[data_offset..], + cur: 0, + }) + } +} + +#[derive(Debug)] +pub struct AccessUnitIter<'a> { + parser: &'a mut PayloadParser, + ext_seqnum: u64, + packet_ts: RtpTimestamp, + prev_index: Option, + headers_r: Option, BigEndian>>, + headers_len: u16, + data: &'a [u8], + cur: u32, +} + +impl<'a> Iterator for AccessUnitIter<'a> { + type Item = anyhow::Result; + + fn next(&mut self) -> Option { + let res = self.next_priv(); + if let Some(Err(_)) = res.as_ref() { + self.parser.reset(); + } + + res + } +} + +impl<'a> AccessUnitIter<'a> { + fn next_priv(&mut self) -> Option> { + use Mpeg4GenericDepayError::*; + + let mut cts_delta = None; + let mut duration = None; + let header = if let Some(r) = self.headers_r.as_mut() { + let pos = r.position_in_bits().unwrap() as u16; + if pos >= self.headers_len { + // No more AUs + return None; + } + + let ctx = AuHeaderContext { + config: &self.parser.config, + prev_index: self.prev_index, + }; + + let header = match r.parse_with::(&ctx) { + Ok(header) => header, + Err(err) => { + return Some( + Err(err).with_context(|| format!("AuHeader in packet {}", self.ext_seqnum)), + ) + } + }; + + if self.prev_index.is_none() { + // First AU header of the packet + if header.index.is_zero() { + if self.parser.constant_duration.is_none() { + // > In the absence of the constantDuration parameter + // > receivers MUST conclude that the AUs have constant duration + // > if the AU-index is zero in two consecutive RTP packets. + + if let Some(cd_prob) = self.parser.const_dur_probation.take() { + let dur = *(self.packet_ts - cd_prob.ts) / cd_prob.frames; + self.parser.constant_duration = Some(dur); + } else { + // Keep first packet for constant duration probation: + self.parser.const_dur_probation = Some(self.packet_ts.into()); + } + } + } else if self.parser.constant_duration.is_some() { + // § 3.2.3.2: + // > when transmitting Access Units of constant duration, the AU-Index, + // > if present, MUST be set to the value 0 + return Some(Err(ConstantDurationAuNonZeroIndex { + index: header.index, + ext_seqnum: self.ext_seqnum, + } + .into())); + } else if self.parser.const_dur_probation.is_some() { + // Constant duration was in probation but index is not zero + // => not switching to constant duration yet + self.parser.const_dur_probation = None; + } + } + if let Some(delta) = header.cts_delta { + cts_delta = Some(delta); + } else if let Some(constant_duration) = self.parser.constant_duration { + // § 3.2.3.2: + // > If the "constantDuration" parameter is present, the receiver can + // > reconstruct the original Access Unit timing based solely on the RTP + // > timestamp and AU-Index-delta. + cts_delta = Some((*header.index * constant_duration) as i32); + duration = Some(constant_duration); + } else if self.prev_index.is_some() && self.parser.const_dur_probation.is_none() { + // Non-constant duration, no CTS-delta, not first header, + // and not constant duration probation in progress + + // § 3.2.3.2: + // > When transmitting Access Units of variable duration, then the + // > "constantDuration" parameter MUST NOT be present [...] + // > the CTS-delta MUST be coded in the AU header for each non-first AU + // > in the RTP packet + + return Some(Err(NonConstantDurationAuNoCtsDelta { + index: header.index, + ext_seqnum: self.ext_seqnum, + } + .into())); + } else if self.prev_index.is_none() { + // First AU but unknown duration + cts_delta = Some(0); + } + + if header.size.is_none() && self.parser.config.constant_size == 0 { + // § 3.2.3: + // > The absence of both AU-size in the AU-header and the constantSize + // > MIME format parameter indicates the carriage of a single AU + // > (fragment), i.e., that a single Access Unit (fragment) is transported + // > in each RTP packet for that stream + + let pos = r.position_in_bits().unwrap() as u16; + if pos < self.headers_len { + // More headers to read + return Some(Err(MultipleAusUnknownSize.into())); + } + } + + if self.data.is_empty() { + // We have exhausted the data section, but there are more headers + return Some(Err(NoMoreAuDataLeft { + index: header.index, + } + .into())); + } + + self.prev_index = Some(header.index); + + header + } else { + // No header section + + if self.data.is_empty() { + // We have exhausted the data section + return None; + } + + let constant_duration = self + .parser + .constant_duration + .expect("checked in PayloadParser::parse"); + + cts_delta = Some((self.cur * constant_duration) as i32); + duration = Some(constant_duration); + + AuHeader::new_with(self.cur) + }; + + let mut is_fragment = false; + + // § 3.2.3 + // > If the AU size is variable, then the + // > size of each AU MUST be indicated in the AU-size field of the + // > corresponding AU-header. However, if the AU size is constant for a + // > stream, this mechanism SHOULD NOT be used; instead, the fixed size + // > SHOULD be signaled by the MIME format parameter "constantSize" + + let au_size = if self.parser.config.constant_size > 0 { + self.parser.config.constant_size as usize + } else if let Some(size) = header.size { + size as usize + } else { + // Unknown size + // Note: MultipleAusUnknownSize case checked above + + self.data.len() + }; + + let data = if au_size <= self.data.len() { + let data = self.data[..au_size].to_owned(); + + // Update self.data for next AU + self.data = &self.data[au_size..]; + + data + } else { + // The len of the AU can exceed the AU data len in case of a framgment + if self.cur > 0 { + return Some(Err(MultipleAusGreaterSizeThanAuData { + au_size, + au_data_size: self.data.len(), + } + .into())); + } + + is_fragment = true; + + let data = self.data[..].to_owned(); + self.data = &[]; + + data + }; + + self.cur += 1; + + if let Some(ref mut cd_prob) = self.parser.const_dur_probation { + cd_prob.frames = self.cur; + } + + Some(Ok(AccessUnit { + ext_seqnum: self.ext_seqnum, + is_fragment, + size: header.size, + index: header.index, + cts_delta, + dts_delta: header.dts_delta, + duration, + maybe_random_access: header.maybe_random_access, + is_interleaved: header.is_interleaved, + data, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use Mpeg4GenericDepayError::*; + + const TS0: RtpTimestamp = RtpTimestamp::ZERO; + const TS21: RtpTimestamp = RtpTimestamp::new(21); + + #[test] + fn no_headers_one_au() { + const CONSTANT_DURATION: u32 = 3; + let mut parser = PayloadParser::new_for(ModeConfig { + constant_duration: CONSTANT_DURATION, + ..Default::default() + }); + + let payload = &[0, 1, 2, 3]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 4); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); // <== + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert!(au.maybe_random_access.is_none()); + assert_eq!(au.data, payload); + + assert!(iter.next().is_none()); + } + + #[test] + fn no_headers_one_au_fragmented() { + const CONSTANT_DURATION: u32 = 3; + let mut parser = PayloadParser::new_for(ModeConfig { + constant_size: 6, + constant_duration: 3, + ..Default::default() + }); + + let payload = &[0, 1, 2, 3]; + let mut iter = parser.parse(payload, 42, TS0).unwrap(); + assert_eq!(iter.data.len(), 4); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(au.is_fragment); // <== + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload); + + assert!(iter.next().is_none()); + } + + #[test] + fn no_headers_two_aus() { + const CONSTANT_DURATION: u32 = 5; + let mut parser = PayloadParser::new_for(ModeConfig { + constant_size: 2, + constant_duration: CONSTANT_DURATION, + ..Default::default() + }); + + let payload = &[0, 1, 2, 3]; + let mut iter = parser.parse(payload, 42, TS21).unwrap(); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 1); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(CONSTANT_DURATION as i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[2..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn no_headers_empty_au_data() { + let mut parser = PayloadParser::new_for(ModeConfig { + constant_size: 2, + constant_duration: 2, + ..Default::default() + }); + + let payload = &[]; + let err = parser + .parse(payload, 0, TS0) + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!(err, EmptyAuData); + } + + #[test] + fn no_headers_two_aus_one_fragment() { + let mut parser = PayloadParser::new_for(ModeConfig { + constant_size: 3, + constant_duration: 2, + ..Default::default() + }); + + let payload = &[0, 1, 2, 3]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[..3]); + + let err = iter + .next() + .unwrap() + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!( + err, + MultipleAusGreaterSizeThanAuData { + au_size: 3, + au_data_size: 1 + } + ); + } + + #[test] + fn header_one_au() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 2 bits: data size (3 here => b11.._....) + let payload = &[0x00, 0x02, 0xc0, 0x01, 0x02, 0x03]; + let mut iter = parser.parse(payload, 42, TS0).unwrap(); + assert_eq!(iter.data.len(), 3); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); // <== + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert!(au.maybe_random_access.is_none()); + assert_eq!(au.data, payload[3..][..3]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_one_au_cts() { + let mut parser = PayloadParser::new_for(ModeConfig { + cts_delta_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 1 bits: CTS flag not set because 1st header (0 here => b0...) + // * following 2 bits for CTS delta not present + let payload = &[0x00, 0x01, 0x00, 0x01, 0x02]; + let mut iter = parser.parse(payload, 0, TS21).unwrap(); + assert_eq!(iter.data.len(), 2); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 0); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert!(au.maybe_random_access.is_none()); + assert_eq!(au.data, payload[3..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_one_au_cts_set() { + use crate::mp4g::header::AuHeaderError::{self, *}; + + let mut parser = PayloadParser::new_for(ModeConfig { + cts_delta_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 3 bits: CTS flag + CTS delta (-1 here => b111.) + let payload = &[0x00, 0x03, 0xe0, 0x01, 0x02]; + let mut iter = parser.parse(payload, 42, TS21).unwrap(); + assert_eq!(iter.data.len(), 2); + + let err = iter + .next() + .unwrap() + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!(err, CtsFlagSetInFirstAuHeader(AccessUnitIndex::ZERO)); + } + + #[test] + fn header_one_au_random_access() { + let mut parser = PayloadParser::new_for(ModeConfig { + random_access_indication: true, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 1 bit: random access flag (1 here => b1...) + let payload = &[0x00, 0x01, 0x80, 0x01, 0x02]; + let mut iter = parser.parse(payload, 0, TS21).unwrap(); + assert_eq!(iter.data.len(), 2); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 0); + assert!(!au.is_interleaved); + assert_eq!(au.maybe_random_access, Some(true)); + assert_eq!(au.data, payload[3..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_one_au_random_access_stream_state() { + let mut parser = PayloadParser::new_for(ModeConfig { + random_access_indication: true, + stream_state_indication: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 1 bit: random access flag (1 here => b1...) + // * 2 bits: stream state (3 here => b.11._....) + let payload = &[0x00, 0x03, 0xe0, 0x01, 0x02]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 2); + + let au = iter.next().unwrap().unwrap(); + assert!(!au.is_interleaved); + assert_eq!(au.maybe_random_access, Some(true)); + assert_eq!(au.data, payload[3..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_one_au_fragemented() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 2 bits: data size (3 here => b11.._....) + let payload = &[0x00, 0x02, 0xc0, 0x01, 0x02]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 2); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(au.is_fragment); // <== + assert_eq!(au.data, payload[3..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_two_aus() { + const CONSTANT_DURATION: u32 = 5; + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + constant_duration: CONSTANT_DURATION, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Each AU-header: + // * 2 bits: data size (AU1: 2 => b10.._...., AU2: 3 => b..11_....) + let payload = &[0x00, 0x04, 0xb0, 0x01, 0x02, 0x03, 0x04, 0x05]; + let mut iter = parser.parse(payload, 42, TS0).unwrap(); + assert_eq!(iter.data.len(), 5); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[3..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 1); + assert_eq!(au.cts_delta, Some(CONSTANT_DURATION as i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[5..][..3]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_two_aus_const_duration_probation() { + const CONSTANT_DURATION: u32 = 2; + + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + ..Default::default() + }); + + assert!(parser.constant_duration.is_none()); + assert!(parser.const_dur_probation.is_none()); + + // Header section size: 2 bytes. + // Each AU-header: + // * 2 bits: data size (AU1: 2 => b10.._...., AU2: 3 => b..11_....) + let payload = &[0x00, 0x04, 0xb0, 0x01, 0x02, 0x03, 0x04, 0x05]; + let mut iter = parser.parse(payload, 42, TS0).unwrap(); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert_eq!(au.data, payload[3..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert!(au.cts_delta.is_none()); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert_eq!(au.data, payload[5..][..3]); + + assert!(iter.next().is_none()); + + assert!(parser.constant_duration.is_none()); + assert!(parser.const_dur_probation.is_some()); + + let ts4 = TS0 + 2 * CONSTANT_DURATION; + + // Header section size: 2 bytes. + // Each AU-header: + // * 2 bits: data size (AU1: 2 => b10.._...., AU2: 3 => b..11_....) + let payload = &[0x00, 0x04, 0xb0, 0x01, 0x02, 0x03, 0x04, 0x05]; + let mut iter = parser.parse(payload, 42, ts4).unwrap(); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload[3..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.cts_delta, Some(CONSTANT_DURATION as i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload[5..][..3]); + + assert!(iter.next().is_none()); + + assert_eq!(parser.constant_duration, Some(CONSTANT_DURATION)); + assert!(parser.const_dur_probation.is_none()); + } + + #[test] + fn header_two_au_cts() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + cts_delta_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * Header 1 + // - 2 bits: len 2 => b10.. + // - 1 bit: CTS flag not set because 1st header => b..0. + // * Header 2 + // - 2 bits: len 3 => b...1_1... + // - 3 bits: CTS flag + CTS delta +1 in 2's comp => b...._.101) + let payload = &[0x00, 0x08, 0x9d, 0x01, 0x02, 0x03, 0x04, 0x05]; + let mut iter = parser.parse(payload, 42, TS21).unwrap(); + assert_eq!(iter.data.len(), 5); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[3..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 42); + assert_eq!(au.index, 1); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some(1i32)); + assert!(au.dts_delta.is_none()); + assert!(au.duration.is_none()); + assert_eq!(au.data, payload[5..][..3]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_three_aus_delta_index() { + const CONSTANT_DURATION: u32 = 5; + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + constant_duration: CONSTANT_DURATION, + index_delta_len: 1, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Each AU-header: 2 bits: data size + // For headers 2 & 3: 1 bits: AU-index-delta + // + // - AU1: size 2 => b10.._.... + // - AU2: size 3, delta 0 => b..11_0... + // - AU3: size 1, delta 1 => b...._.011 + let payload = &[0x00, 0x07, 0xb3, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 6); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some(0i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload[3..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 1); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some((*au.index * CONSTANT_DURATION) as i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload[5..][..3]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 3); + assert!(au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some((*au.index * CONSTANT_DURATION) as i32)); + assert!(au.dts_delta.is_none()); + assert_eq!(au.duration, Some(CONSTANT_DURATION)); + assert_eq!(au.data, payload[8..][..1]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_three_au_cts() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + cts_delta_len: 3, + dts_delta_len: 3, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * Header 1 + // - 2 bits: len 2 => b10.. + // - 1 bit: CTS flag not set because 1st header => b..0. + // - 4 bits: DTS flag + DTS delta -2 in 2's comp => b...1_110.) + // * Header 2 + // - 2 bits: len 3 => b...._...1 1..._.... + // - 4 bits: CTS flag + CTS delta +1 in 2's comp => b.100_1...) + // - 4 bit: DTS flag + DTS delta 0 (i.e. same as CTS) => b...._.100 0..._....) + // * Header 3 + // - 2 bits: len 1 => b.01._.... + // - 4 bits: CTS flag + CTS delta +2 in 2's comp => b...1_010.) + // - 4 bits: DTS flag + DTS delta -2 in 2's comp => b...._...1 110._....) + let payload = &[ + 0x00, 0x1b, 0x9d, 0xcc, 0x35, 0xc0, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, + ]; + let mut iter = parser.parse(payload, 0, TS21).unwrap(); + assert_eq!(iter.data.len(), 6); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 0); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some(0i32)); + assert_eq!(au.dts_delta, Some(-2i32)); + assert_eq!(au.data, payload[6..][..2]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 0); + assert_eq!(au.index, 1); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.cts_delta, Some(1i32)); + assert_eq!(au.dts_delta, Some(0i32)); + assert_eq!(au.data, payload[8..][..3]); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.ext_seqnum, 0); + assert_eq!(au.index, 2); + assert!(!au.is_interleaved); + assert_eq!(au.cts_delta, Some(2i32)); + assert_eq!(au.dts_delta, Some(-2i32)); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[11..][..1]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_one_au_unknown_size() { + let mut parser = PayloadParser::new_for(ModeConfig { + index_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Header: + // * 2 bits: AU-index. (set to 0) + let payload = &[0x00, 0x02, 0x00, 0x01, 0x02]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 2); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_fragment); // <== + assert_eq!(au.data, payload[3..][..2]); + + assert!(iter.next().is_none()); + } + + #[test] + fn header_two_aus_unknown_size() { + let mut parser = PayloadParser::new_for(ModeConfig { + index_len: 1, + index_delta_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // First AU-header: + // * 1 bit: AU-index. (set to 0) + // Second AU-header: + // * 2 bits: AU-index-delta. (set to 2) + let payload = &[0x00, 0x03, 0x40, 0x01, 0x02, 0x03, 0x04]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 4); + + let err = iter + .next() + .unwrap() + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!(err, MultipleAusUnknownSize); + } + + #[test] + fn header_two_aus_one_fragment() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 2, + constant_duration: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Each AU-header: + // * 2 bits: data size (AU1: 2 => b10.._...., AU2: 3 => b..11_....) + let payload = &[0x00, 0x04, 0xb0, 0x01, 0x02, 0x03]; + let mut iter = parser.parse(payload, 0, TS0).unwrap(); + assert_eq!(iter.data.len(), 3); + + let au = iter.next().unwrap().unwrap(); + assert_eq!(au.index, 0); + assert!(!au.is_interleaved); + assert!(!au.is_fragment); + assert_eq!(au.data, payload[3..][..2]); + + let err = iter + .next() + .unwrap() + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!( + err, + MultipleAusGreaterSizeThanAuData { + au_size: 3, + au_data_size: 1 + } + ); + } + + #[test] + fn header_section_too_large() { + let mut parser = PayloadParser::new_for(ModeConfig { + size_len: 4, + ..Default::default() + }); + + // Header section size: 2 bytes. + // Each Au-Header: 4 bits, so 3 of them here. + let payload = &[0x00, 0x0c, 0x00]; + let err = parser + .parse(payload, 0, TS0) + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!( + err, + AuHeaderSectionTooLarge { + expected_end: 4, + total: 3 + } + ); + } + + #[test] + fn auxiliary_section_too_large() { + let mut parser = PayloadParser::new_for(ModeConfig { + random_access_indication: true, + auxiliary_data_size_len: 2, + ..Default::default() + }); + + // Header section size: 2 bytes. + // * 2x Au-Header: 3 bits each. 3 of them here + padding => 1 byte + // Auxiliary section size: 2 bytes. (one byte available instead of 2 advertised) + let payload = &[0x00, 0x06, 0x00, 0x00, 0x0c, 0x00]; + let err = parser + .parse(payload, 0, TS0) + .unwrap_err() + .downcast::() + .unwrap(); + assert_eq!( + err, + AuAuxiliarySectionTooLarge { + expected_end: 7, + total: 6, + } + ); + } +} diff --git a/net/rtp/src/mp4g/header.rs b/net/rtp/src/mp4g/header.rs new file mode 100644 index 00000000..fe2e54d0 --- /dev/null +++ b/net/rtp/src/mp4g/header.rs @@ -0,0 +1,235 @@ +//! Access Unit Header and its parser & writer. + +use bitstream_io::{BitRead, BitWrite, FromBitStreamWith, ToBitStreamWith}; + +use crate::mp4g::{AccessUnitIndex, ModeConfig}; +use crate::utils::{mask_valid_2_comp, raw_2_comp_to_i32}; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum AuHeaderError { + #[error("Unexpected zero-sized AU {}", .0)] + ZeroSizedAu(AccessUnitIndex), + + #[error("Undefined mandatory size for AU {}", .0)] + UndefinedMandatorySize(AccessUnitIndex), + + #[error("Inconsistent delta index {index}. Previous index: {prev_index}")] + InconsistentDeltaIndex { + index: AccessUnitIndex, + prev_index: AccessUnitIndex, + }, + + #[error("Unexpected CTS flag set for the first AU header {}", .0)] + CtsFlagSetInFirstAuHeader(AccessUnitIndex), + + #[error("Out of range CTS-delta {cts_delta} for AU {index}")] + OutOfRangeSizeCtsDelta { + cts_delta: i32, + index: AccessUnitIndex, + }, + + #[error("Out of range DTS-delta {dts_delta} for AU {index}")] + OutOfRangeSizeDtsDelta { + dts_delta: i32, + index: AccessUnitIndex, + }, +} + +#[derive(Debug)] +pub struct AuHeaderContext<'a> { + pub(crate) config: &'a ModeConfig, + pub(crate) prev_index: Option, +} + +#[derive(Debug, Default)] +pub struct AuHeader { + pub(crate) size: Option, + pub(crate) index: AccessUnitIndex, + pub(crate) cts_delta: Option, + pub(crate) dts_delta: Option, + pub(crate) maybe_random_access: Option, + pub(crate) is_interleaved: bool, +} + +impl AuHeader { + #[inline] + pub(crate) fn new_with(index: impl Into) -> Self { + AuHeader { + index: index.into(), + ..Default::default() + } + } +} + +impl<'a> FromBitStreamWith<'a> for AuHeader { + type Context = AuHeaderContext<'a>; + type Error = anyhow::Error; + + fn from_reader( + r: &mut R, + ctx: &AuHeaderContext, + ) -> Result { + use anyhow::Context; + use AuHeaderError::*; + + let mut this = AuHeader::default(); + + if ctx.config.size_len > 0 { + let val = r + .read::(ctx.config.size_len as u32) + .context("AU-size")?; + + // Will ensure the size is non-zero after we get the index + this.size = Some(val); + } + + this.index = match ctx.prev_index { + None => r + .read::(ctx.config.index_len as u32) + .context("AU-Index")? + .into(), + Some(prev_index) => { + let delta = r + .read::(ctx.config.index_delta_len as u32) + .context("AU-Index-delta")?; + if delta > 0 { + this.is_interleaved = true; + } + + prev_index + 1u32 + delta + } + }; + + if let Some(0) = this.size { + Err(ZeroSizedAu(this.index))?; + } + + if ctx.config.cts_delta_len > 0 && r.read_bit().context("CTS-flag")? { + if ctx.prev_index.is_none() { + // § 3.2.1.1: + // > the CTS-flag field MUST have the value 0 in the first AU-header + Err(CtsFlagSetInFirstAuHeader(this.index))?; + } + + let delta = r + .read::(ctx.config.cts_delta_len as u32) + .context("CTS-delta")?; + let delta = raw_2_comp_to_i32(delta, ctx.config.cts_delta_len); + this.cts_delta = Some(delta); + } + + if ctx.config.dts_delta_len > 0 && r.read_bit().context("DTS-flag")? { + let delta = r + .read::(ctx.config.dts_delta_len as u32) + .context("DTS-delta")?; + let delta = raw_2_comp_to_i32(delta, ctx.config.dts_delta_len); + this.dts_delta = Some(delta); + } + + if ctx.config.random_access_indication { + this.maybe_random_access = Some(r.read_bit().context("RAP-flag")?); + } + + // ignored by gstrtpmp4gdepay + if ctx.config.stream_state_indication > 0 { + r.skip(ctx.config.stream_state_indication as u32) + .context("Stream-state")?; + } + + Ok(this) + } +} + +impl<'a> ToBitStreamWith<'a> for AuHeader { + type Context = AuHeaderContext<'a>; + type Error = anyhow::Error; + + fn to_writer( + &self, + w: &mut W, + ctx: &AuHeaderContext, + ) -> Result<(), Self::Error> { + use anyhow::Context; + use AuHeaderError::*; + + if ctx.config.size_len > 0 { + let Some(size) = self.size else { + return Err(UndefinedMandatorySize(self.index).into()); + }; + + if size == 0 { + Err(ZeroSizedAu(self.index))?; + } + + w.write(ctx.config.size_len as u32, size) + .context("AU-size")?; + } + + match ctx.prev_index { + None => w + .write(ctx.config.index_len as u32, *self.index) + .context("AU-Index")?, + Some(prev_index) => { + let index_delta = self + .index + .checked_sub(*prev_index) + .and_then(|delta| delta.checked_sub(1)) + .ok_or(InconsistentDeltaIndex { + index: self.index, + prev_index, + }) + .context("AU-Index-delta")?; + w.write(ctx.config.index_delta_len as u32, index_delta) + .context("AU-Index-delta")?; + } + } + + if ctx.config.cts_delta_len > 0 { + // § 3.2.1.1: + // > the CTS-flag field MUST have the value 0 in the first AU-header + // > the CTS-flag field SHOULD be 0 for any non-first fragment of an Access Unit + if ctx.prev_index.is_none() { + w.write_bit(false).context("CTS-flag")?; + } else if let Some(cts_delta) = self.cts_delta { + let Some(cts_delta) = mask_valid_2_comp(cts_delta, ctx.config.cts_delta_len) else { + return Err(OutOfRangeSizeCtsDelta { + cts_delta, + index: self.index, + } + .into()); + }; + + w.write_bit(true).context("CTS-flag")?; + w.write(ctx.config.cts_delta_len as u32, cts_delta) + .context("CTS-delta")?; + } else { + w.write_bit(false).context("CTS-flag")?; + } + } + + if ctx.config.dts_delta_len > 0 { + if let Some(dts_delta) = self.dts_delta { + let Some(dts_delta) = mask_valid_2_comp(dts_delta, ctx.config.dts_delta_len) else { + return Err(OutOfRangeSizeDtsDelta { + dts_delta, + index: self.index, + } + .into()); + }; + + w.write_bit(true).context("DTS-flag")?; + w.write(ctx.config.dts_delta_len as u32, dts_delta) + .context("DTS-delta")?; + } else { + w.write_bit(false).context("DTS-flag")?; + } + } + + if ctx.config.random_access_indication { + w.write_bit(self.maybe_random_access.unwrap_or(false)) + .context("RAP-flag")?; + } + + Ok(()) + } +} diff --git a/net/rtp/src/mp4g/mod.rs b/net/rtp/src/mp4g/mod.rs new file mode 100644 index 00000000..5eac6c77 --- /dev/null +++ b/net/rtp/src/mp4g/mod.rs @@ -0,0 +1,34 @@ +// SPDX-License-Identifier: MPL-2.0 + +pub mod depay; +mod header; +pub use header::{AuHeader, AuHeaderContext}; +mod mode; +pub use mode::ModeConfig; +pub mod pay; + +#[cfg(test)] +mod tests; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum Mpeg4GenericError { + #[error("Can't compare AU index 0x8000_0000 to 0")] + AuIndexComparisonLimit, + + #[error("Can't compare RTP timestamps 0x8000_0000 to 0")] + RTPTimestampComparisonLimit, +} + +/// An Access Unit Index implemented as a comparable new type on a `[std::num::Wrapping]::`. +define_wrapping_comparable_u32_with_display!( + AccessUnitIndex, + Mpeg4GenericError, + AuIndexComparisonLimit, +); + +/// An RTP timestamp implemented as a comparable new type on a `[std::num::Wrapping]::`. +define_wrapping_comparable_u32_with_display!( + RtpTimestamp, + Mpeg4GenericError, + RTPTimestampComparisonLimit, +); diff --git a/net/rtp/src/mp4g/mode.rs b/net/rtp/src/mp4g/mode.rs new file mode 100644 index 00000000..663518d2 --- /dev/null +++ b/net/rtp/src/mp4g/mode.rs @@ -0,0 +1,199 @@ +//! MPEG-4 Generic mode. + +use gst::caps::NoFeature; + +use std::str::FromStr; + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum ModeError { + #[error("sizelength & constantsize can't be both defined")] + BothAuSizeLenAndConstantSize, + + #[error("Neither sizelength nor constantsize are defined, need at least one of them")] + NeitherAuSizeLenNorConstantSize, + + #[error("indexlength > 0 but indexdeltalength not defined")] + MandatoryIndexDeltaLength, +} + +#[derive(Debug, Default)] +pub struct ModeConfig { + pub(crate) size_len: u8, + pub(crate) index_len: u8, + pub(crate) index_delta_len: u8, + pub(crate) cts_delta_len: u8, + pub(crate) dts_delta_len: u8, + pub(crate) random_access_indication: bool, + pub(crate) stream_state_indication: u8, + pub(crate) auxiliary_data_size_len: u8, + pub(crate) constant_size: u32, + pub(crate) constant_duration: u32, + pub(crate) max_displacement: u32, +} + +impl ModeConfig { + #[inline] + pub fn has_header_section(&self) -> bool { + self.size_len > 0 + || self.index_len > 0 + || self.index_delta_len > 0 + || self.cts_delta_len > 0 + || self.dts_delta_len > 0 + || self.random_access_indication + || self.stream_state_indication > 0 + } + + #[inline] + pub fn has_auxiliary_section(&self) -> bool { + self.auxiliary_data_size_len > 0 + } + + #[inline] + pub fn constant_duration(&self) -> Option { + if self.constant_duration == 0 { + return None; + } + + Some(self.constant_duration) + } + + #[inline] + pub fn max_displacement(&self) -> Option { + if self.max_displacement == 0 { + return None; + } + + Some(self.max_displacement) + } + + /// Returns the max length in bits of the AU headers + pub fn max_header_bit_len(&self) -> usize { + self.size_len as usize + + std::cmp::max(self.index_len, self.index_delta_len) as usize + + self.cts_delta_len as usize + + self.dts_delta_len as usize + + if self.random_access_indication { 1 } else { 0 } + + self.stream_state_indication as usize + } + + pub fn from_caps(s: &gst::StructureRef) -> anyhow::Result { + use ModeError::*; + + // These values are optional and have a default value of 0 (no header) + + let size_len = Self::parse_int::(s, "sizelength")?; + let constant_size = Self::parse_int::(s, "constantsize")?; + + if size_len != 0 && constant_size != 0 { + Err(BothAuSizeLenAndConstantSize)?; + } + + if size_len == 0 && constant_size == 0 { + Err(NeitherAuSizeLenNorConstantSize)?; + } + + // § 3.2.1 + // > If the AU-Index field is present in the first AU-header in the AU + // > Header Section, then the AU-Index-delta field MUST be present in + // > any subsequent (non-first) AU-header. + + let index_len = Self::parse_int::(s, "indexlength")?; + let index_delta_len = Self::parse_int::(s, "indexdeltalength")?; + + if index_len > 0 && index_delta_len == 0 { + Err(MandatoryIndexDeltaLength)?; + } + + // TODO check mode & mode_config conformity + + Ok(ModeConfig { + size_len, + index_len, + index_delta_len, + cts_delta_len: Self::parse_int::(s, "ctsdeltalength")?, + dts_delta_len: Self::parse_int::(s, "dtsdeltalength")?, + random_access_indication: Self::parse_int::(s, "randomaccessindication")? > 0, + stream_state_indication: Self::parse_int::(s, "streamstateindication")?, + auxiliary_data_size_len: Self::parse_int::(s, "auxiliarydatasizelength")?, + constant_size, + constant_duration: Self::parse_int::(s, "constantduration")?, + max_displacement: Self::parse_int::(s, "maxdisplacement")?, + }) + } + + /// Tries to read the `field` from the provided structure as an integer of type `T`. + /// + /// Returns: + /// + /// * `Ok(val)` if the field is present and its value could be parsed. + /// * `Ok(0)` if the field is not present. + /// * `Err(_)` otherwise. + fn parse_int<'a, T>(s: &'a gst::StructureRef, field: &'static str) -> anyhow::Result + where + T: TryFrom + FromStr + gst::glib::value::FromValue<'a>, + >::Error: std::error::Error + Send + Sync + 'static, + ::Err: std::error::Error + Send + Sync + 'static, + { + use anyhow::Context; + use gst::structure::GetError::*; + + match s.get::(field) { + Ok(val) => Ok(val), + Err(FieldNotFound { .. }) => Ok(T::try_from(0i32).unwrap()), + Err(ValueGetError { .. }) => match s.get::(field) { + Ok(val) => Ok(T::try_from(val).context(field)?), + Err(_) => Ok(s + .get::<&str>(field) + .context(field)? + .parse::() + .context(field)?), + }, + } + } + + pub fn add_to_caps( + &self, + builder: gst::caps::Builder, + ) -> Result, ModeError> { + use ModeError::*; + + if self.size_len != 0 && self.constant_size != 0 { + Err(BothAuSizeLenAndConstantSize)?; + } + + if self.size_len == 0 && self.constant_size == 0 { + Err(NeitherAuSizeLenNorConstantSize)?; + } + + if self.index_len > 0 && self.index_delta_len == 0 { + Err(MandatoryIndexDeltaLength)?; + } + + if self.stream_state_indication > 0 { + panic!("AU Header Stream State not supported"); + } + + Ok(builder + .field("sizelength", self.size_len as i32) + .field("indexlength", self.index_len as i32) + .field("indexdeltalength", self.index_delta_len as i32) + .field("ctsdeltalength", self.cts_delta_len as i32) + .field("dtsdeltalength", self.dts_delta_len as i32) + .field( + "randomaccessindication", + if self.random_access_indication { + 1u8 + } else { + 0u8 + }, + ) + .field("streamstateindication", self.stream_state_indication as i32) + .field( + "auxiliarydatasizelength", + self.auxiliary_data_size_len as i32, + ) + .field("constantsize", self.constant_size as i32) + .field("constantduration", self.constant_duration as i32) + .field("maxdisplacement", self.max_displacement as i32)) + } +} diff --git a/net/rtp/src/mp4g/pay/imp.rs b/net/rtp/src/mp4g/pay/imp.rs new file mode 100644 index 00000000..78f27457 --- /dev/null +++ b/net/rtp/src/mp4g/pay/imp.rs @@ -0,0 +1,921 @@ +// GStreamer RTP MPEG-4 Generic Payloader +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +/** + * SECTION:element-rtpmp4gpay2 + * @see_also: rtpmp4gpay2, rtpmp4gpay, rtpmp4gpay, fdkaacenc, fdkaacdec, avenc_mpeg4, avdec_mpeg4 + * + * Payload an MPEG-4 Generic elementary stream into RTP packets as per [RFC 3640][rfc-3640]. + * Also see the [IANA media-type page for MPEG-4 Generic][iana-mpeg4-generic]. + * + * [rfc-3640]: https://www.rfc-editor.org/rfc/rfc3640.html#section-4 + * [iana-mpeg4-generic]: https://www.iana.org/assignments/media-types/application/mpeg4-generic + * + * ## Aggregation Modes + * + * The default aggregation mode is `auto`: If upstream is live, the payloader will send out + * AUs immediately, even if they don't completely fill a packet, in order to minimise + * latency. If upstream is not live, the payloader will by default aggregate AUs until + * it has completely filled an RTP packet as per the configured MTU size or the `max-ptime` + * property if it is set (it is not set by default). + * + * The aggregation mode can be controlled via the `aggregate-mode` property. + * + * ## Example pipeline + * |[ + * gst-launch-1.0 audiotestsrc ! fdkaacenc ! rtpmp4gpay2 ! udpsink host=127.0.0.1 port=5004 + * ]| This will encode an audio test signal to AAC and then payload the encoded audio + * into RTP packets and send them out via UDP to localhost (IPv4) port 5004. + * You can use the #rtpmp4gdepay2 or #rtpmp4gdepay elements to depayload such a stream, and + * the #fdkaacdec element to decode the depayloaded stream. + * + * Since: plugins-rs-0.13.0 + */ +use atomic_refcell::AtomicRefCell; +use bitstream_io::{BigEndian, BitCounter, BitRead, BitReader, BitWrite, BitWriter}; +use once_cell::sync::Lazy; + +use gst::{glib, prelude::*, subclass::prelude::*}; +use smallvec::SmallVec; + +use std::collections::VecDeque; +use std::sync::Mutex; + +use crate::basepay::{PacketToBufferRelation, RtpBasePay2Ext, RtpBasePay2Impl, RtpBasePay2ImplExt}; + +use super::RtpMpeg4GenericPayAggregateMode; +use crate::mp4a::parsers::{AudioSpecificConfig, ProfileLevel}; +use crate::mp4g::{AccessUnitIndex, AuHeader, AuHeaderContext, ModeConfig}; + +const VOS_STARTCODE: u32 = 0x000001B0; + +/// The size of the field representing the AU headers section len. +const HEADERS_LEN_SIZE: usize = 2; + +/// Access Unit maximum header len in bytes. +/// This depends on the supported mode. In current implementation, 3 is the maximum. +const HEADER_MAX_LEN: usize = 3; + +#[derive(Clone)] +struct Settings { + max_ptime: Option, + aggregate_mode: RtpMpeg4GenericPayAggregateMode, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + aggregate_mode: RtpMpeg4GenericPayAggregateMode::Auto, + max_ptime: None, + } + } +} + +#[derive(Default)] +pub struct RtpMpeg4GenericPay { + state: AtomicRefCell, + settings: Mutex, + is_live: Mutex>, +} + +#[derive(Debug)] +struct AccessUnit { + id: u64, + pts: Option, + dts_delta: Option, + duration: Option, + maybe_random_access: Option, + buffer: gst::MappedBuffer, +} + +#[derive(Default)] +struct State { + /// Configuration of current Mode. + mode: ModeConfig, + + /// Maximum bit length needed to store an AU Header. + max_header_bit_len: usize, + + /// Minimum MTU necessary to handle the outgoing packets. + min_mtu: usize, + + /// Pending AU (we collect until ptime/max-ptime is hit or the packet is full) + pending_aus: VecDeque, + pending_size: usize, + pending_duration: Option, + clock_rate: u32, + + /// Desired "packet time", i.e. packet duration, from the downstream caps, if set + ptime: Option, + max_ptime: Option, +} + +impl State { + fn flush(&mut self) { + self.pending_aus.clear(); + self.pending_size = 0; + self.pending_duration = None; + } +} + +static CAT: Lazy = Lazy::new(|| { + gst::DebugCategory::new( + "rtpmp4gpay2", + gst::DebugColorFlags::empty(), + Some("RTP MPEG-4 Generic Payloader"), + ) +}); + +#[glib::object_subclass] +impl ObjectSubclass for RtpMpeg4GenericPay { + const NAME: &'static str = "GstRtpMpeg4GenericPay"; + type Type = super::RtpMpeg4GenericPay; + type ParentType = crate::basepay::RtpBasePay2; +} + +impl ObjectImpl for RtpMpeg4GenericPay { + fn properties() -> &'static [glib::ParamSpec] { + static PROPERTIES: Lazy> = Lazy::new(|| { + vec![ + glib::ParamSpecEnum::builder_with_default( + "aggregate-mode", + Settings::default().aggregate_mode, + ) + .nick("Aggregate Mode") + .blurb( + "Whether to send out AUs immediately or aggregate them until a packet is full.", + ) + .build(), + // Using same type/semantics as C payloaders + glib::ParamSpecInt64::builder("max-ptime") + .nick("Maximum Packet Time") + .blurb("Maximum duration of the packet data in ns (-1 = unlimited up to MTU)") + .default_value( + Settings::default() + .max_ptime + .map(gst::ClockTime::nseconds) + .map(|x| x as i64) + .unwrap_or(-1), + ) + .minimum(-1) + .maximum(i64::MAX) + .mutable_playing() + .build(), + ] + }); + + PROPERTIES.as_ref() + } + + fn set_property(&self, _id: usize, value: &glib::Value, pspec: &glib::ParamSpec) { + let mut settings = self.settings.lock().unwrap(); + + match pspec.name() { + "aggregate-mode" => { + settings.aggregate_mode = value + .get::() + .expect("type checked upstream"); + } + "max-ptime" => { + let new_max_ptime = match value.get::().unwrap() { + -1 => None, + v @ 0.. => Some(gst::ClockTime::from_nseconds(v as u64)), + _ => unreachable!(), + }; + let changed = settings.max_ptime != new_max_ptime; + settings.max_ptime = new_max_ptime; + drop(settings); + + if changed { + let _ = self + .obj() + .post_message(gst::message::Latency::builder().src(&*self.obj()).build()); + } + } + _ => unimplemented!(), + }; + } + + fn property(&self, _id: usize, pspec: &glib::ParamSpec) -> glib::Value { + let settings = self.settings.lock().unwrap(); + + match pspec.name() { + "aggregate-mode" => settings.aggregate_mode.to_value(), + "max-ptime" => (settings + .max_ptime + .map(gst::ClockTime::nseconds) + .map(|x| x as i64) + .unwrap_or(-1)) + .to_value(), + _ => unimplemented!(), + } + } +} + +impl GstObjectImpl for RtpMpeg4GenericPay {} + +impl ElementImpl for RtpMpeg4GenericPay { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "RTP MPEG-4 Generic Payloader", + "Codec/Payloader/Network/RTP", + "Payload an MPEG-4 Generic elementary stream into RTP packets (RFC 3640)", + "François Laignel ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let sink_pad_template = gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &gst::Caps::builder_full() + .structure( + gst::Structure::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false) + .build(), + ) + .structure( + gst::Structure::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("stream-format", "raw") + .build(), + ) + .build(), + ) + .unwrap(); + + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("application/x-rtp") + // TODO "application" is also present in rtpmp4gpay caps template + // but it doesn't handle it in gst_rtp_mp4g_pay_setcaps + .field("media", gst::List::new(["audio", "video"])) + .field("clock-rate", gst::IntRange::new(1i32, i32::MAX)) + .field("encoding-name", "MPEG4-GENERIC") + // Required string params: + .field("streamtype", gst::List::new(["4", "5"])) // 4 = video, 5 = audio + // "profile-level-id = [1,MAX], " + // "config = (string)" + .field( + "mode", + gst::List::new(["generic", "AAC-lbr", "AAC-hbr", "aac-hbr"]), + ) + // Optional general parameters: + // "objecttype = [1,MAX], " + // "constantsize = [1,MAX], " // constant size of each AU + // "constantduration = [1,MAX], " // constant duration of each AU + // "maxdisplacement = [1,MAX], " + // "de-interleavebuffersize = [1,MAX], " + // Optional configuration parameters: + // "sizelength = [1, 32], " + // "indexlength = [1, 32], " + // "indexdeltalength = [1, 32], " + // "ctsdeltalength = [1, 32], " + // "dtsdeltalength = [1, 32], " + // "randomaccessindication = {0, 1}, " + // "streamstateindication = [0, 32], " + // "auxiliarydatasizelength = [0, 32]" ) + .build(), + ) + .unwrap(); + + vec![sink_pad_template, src_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +/// Returns the difference between `ClockTime`s `ct1` & `ct2` in RTP scale. +/// +/// Returns `None` if at least one of the `ClockTime`s is `None`. +/// Returns `Some(None)` if an overflow occurred, error management is left to the caller. +/// Returns `Some(delta)` if the difference could be computed. +fn ct_delta_to_rtp( + ct1: Option, + ct0: Option, + clock_rate: u32, +) -> Option> { + ct1.into_positive().opt_sub(ct0).map(|delta_ct| { + delta_ct + .into_inner_signed() + .try_into() + .ok() + .and_then(|delta_inner: i64| { + delta_inner + .mul_div_ceil(clock_rate as i64, *gst::ClockTime::SECOND as i64) + .and_then(|dts_delta| dts_delta.try_into().ok()) + }) + }) +} + +impl RtpBasePay2Impl for RtpMpeg4GenericPay { + const ALLOWED_META_TAGS: &'static [&'static str] = &["audio"]; + + fn set_sink_caps(&self, caps: &gst::Caps) -> bool { + let s = caps.structure(0).unwrap(); + + let codec_data = match s.get::<&gst::BufferRef>("codec_data") { + Ok(codec_data) => codec_data, + Err(err) => { + gst::error!(CAT, imp: self, "Error getting codec_data from Caps: {err}"); + return false; + } + }; + + let Ok(codec_data) = codec_data.map_readable() else { + gst::error!(CAT, imp: self, "Failed to map codec_data as readable"); + return false; + }; + + let codec_data_str = hex::encode(&codec_data); + + let caps_builder = gst::Caps::builder("application/x-rtp") + .field("seqnum-base", self.obj().property::("seqnum") + 1) + .field("mpegversion", 4i32) + .field("encoding-name", "MPEG4-GENERIC") + .field("config", codec_data_str); + + let (clock_rate, mode, caps_builder) = match s.name().as_str() { + "audio/mpeg" => { + let mut r = BitReader::endian(codec_data.as_slice(), BigEndian); + let config = match r.parse::() { + Ok(config) => config, + Err(err) => { + gst::error!(CAT, imp: self, "Error parsing audio codec_data: {err:#}"); + return false; + } + }; + + if config.audio_object_type == 0 || config.audio_object_type > 6 { + gst::error!(CAT, imp: self, "Unsupported Audio Object Type {}", config.audio_object_type); + return false; + } + + let profile_level = match ProfileLevel::from_caps(s) { + Ok(profile_level) => profile_level, + Err(err) => { + gst::error!(CAT, imp: self, "Error getting profile level from Caps: {err:#}"); + return false; + } + }; + + gst::log!(CAT, imp: self, "Using audio codec_data {config:?}"); + + // AAC-hbr: also used by rtpmp4gpay + // RFC 3640 also defines AAC-lbr, with a maximum encoded buffer + // size of 63 bytes and which can't be fragmented. Only AAC-hbr + // is used because it is more flexible. We could implement AAC-lbr + // provided make sure the encoded buffers can't exceed the limit + // and add a flag to prevent fragmentation in `send_packets()`. + // See https://www.rfc-editor.org/rfc/rfc3640.html#section-3.3.5 + let mode = ModeConfig { + size_len: 13, + index_len: 3, + index_delta_len: 3, + constant_duration: config.frame_len as u32, + ..Default::default() + }; + + let caps_builder = mode + .add_to_caps( + caps_builder + .field("media", "audio") + .field("streamtype", "5") + .field("mode", "AAC-hbr") + .field("clock-rate", config.sampling_freq as i32) + .field("profile", &profile_level.profile) + .field("level", &profile_level.level) + .field("profile-level-id", profile_level.id) + .field("encoding-params", config.channel_conf as i32), + ) + .expect("invalid audio mode"); + + (config.sampling_freq, mode, caps_builder) + } + "video/mpeg" => { + if codec_data.len() < 5 { + gst::error!(CAT, imp: self, "Error parsing video codec_data: too short"); + return false; + } + + let code = u32::from_be_bytes(codec_data[..4].try_into().unwrap()); + let profile = if code == VOS_STARTCODE { + let profile = codec_data[4]; + gst::log!(CAT, imp: self, "Using video codec_data profile {profile}"); + + profile + } else { + gst::warning!(CAT, imp: self, "Unexpected VOS startcode in video codec_data. Assuming profile '1'"); + + 1 + }; + + // Use a larger size_len than rtpmp4gpay + // otherwise some large AU can't be payloaded. + // rtpmp4gpay uses bit shifts to have the AU data size + // fit in 13 bits, resulting in an invalid size. + let mode = ModeConfig { + size_len: 16, + index_len: 3, + index_delta_len: 3, + cts_delta_len: 16, + dts_delta_len: 16, + random_access_indication: true, + ..Default::default() + }; + + let caps_builder = mode + .add_to_caps( + caps_builder + .field("media", "video") + .field("streamtype", "4") + .field("mode", "generic") + .field("clock-rate", 90000i32) + .field("profile-level-id", profile as i32), + ) + .expect("invalid video mode"); + + (90000, mode, caps_builder) + } + // TODO handle "application" + _ => unreachable!(), + }; + + self.obj().set_src_caps(&caps_builder.build()); + + let mut state = self.state.borrow_mut(); + state.max_header_bit_len = mode.max_header_bit_len(); + state.min_mtu = rtp_types::RtpPacket::MIN_RTP_PACKET_LEN + + HEADERS_LEN_SIZE + + (state.max_header_bit_len + 7) / 8 + + 1; + state.mode = mode; + state.clock_rate = clock_rate; + + true + } + + fn negotiate(&self, mut src_caps: gst::Caps) { + // Fixate as a first step + src_caps.fixate(); + + let s = src_caps.structure(0).unwrap(); + + // Negotiate ptime/maxptime with downstream and use them in combination with the + // properties. See https://www.iana.org/assignments/media-types/application/mpeg4-generic + let ptime = s + .get::("ptime") + .ok() + .map(u64::from) + .map(gst::ClockTime::from_mseconds); + + let max_ptime = s + .get::("maxptime") + .ok() + .map(u64::from) + .map(gst::ClockTime::from_mseconds); + + self.parent_negotiate(src_caps); + + let mut state = self.state.borrow_mut(); + state.ptime = ptime; + state.max_ptime = max_ptime; + drop(state); + } + + // Encapsulation of MPEG-4 Generic Elementary Streams: + // https://www.rfc-editor.org/rfc/rfc3640 + fn handle_buffer( + &self, + buffer: &gst::Buffer, + id: u64, + ) -> Result { + let mut state = self.state.borrow_mut(); + let mut settings = self.settings.lock().unwrap(); + + gst::trace!(CAT, imp: self, "Handling buffer {id} duration {} pts {} dts {}, len {}", + buffer.duration().display(), buffer.pts().display(), buffer.dts().display(), buffer.size(), + ); + + let maybe_random_access = if state.mode.random_access_indication { + Some(!buffer.flags().contains(gst::BufferFlags::DELTA_UNIT)) + } else { + None + }; + + let dts_delta = ct_delta_to_rtp(buffer.dts(), buffer.pts(), state.clock_rate).and_then(|dts_delta_res| { + if dts_delta_res.is_none() { + gst::warning!(CAT, imp: self, "Overflow computing DTS-delta between pts {} & dts {}", + buffer.dts().display(), buffer.pts().display(), + ); + } + + dts_delta_res + }); + + gst::trace!(CAT, imp: self, + "Pushing AU from buffer {id} dts_delta {dts_delta:?} random access {maybe_random_access:?}", + ); + + state.pending_aus.push_back(AccessUnit { + id, + duration: buffer.duration(), + pts: buffer.pts(), + dts_delta, + buffer: buffer.clone().into_mapped_buffer_readable().map_err(|_| { + gst::error!(CAT, imp: self, "Can't map incoming buffer readable"); + gst::FlowError::Error + })?, + maybe_random_access, + }); + + state.pending_size += buffer.size(); + state.pending_duration.opt_add_assign(buffer.duration()); + + // Make sure we have queried upstream liveness if needed + if settings.aggregate_mode == RtpMpeg4GenericPayAggregateMode::Auto { + self.ensure_upstream_liveness(&mut settings); + } + + self.send_packets(&settings, &mut state, SendPacketMode::WhenReady) + } + + fn drain(&self) -> Result { + let settings = self.settings.lock().unwrap().clone(); + let mut state = self.state.borrow_mut(); + + self.send_packets(&settings, &mut state, SendPacketMode::ForcePending) + } + + fn flush(&self) { + self.state.borrow_mut().flush(); + } + + #[allow(clippy::single_match)] + fn src_query(&self, query: &mut gst::QueryRef) -> bool { + let res = self.parent_src_query(query); + if !res { + return false; + } + + match query.view_mut() { + gst::QueryViewMut::Latency(query) => { + let settings = self.settings.lock().unwrap(); + + let (is_live, mut min, mut max) = query.result(); + + { + let mut live_guard = self.is_live.lock().unwrap(); + + if Some(is_live) != *live_guard { + gst::info!(CAT, imp: self, "Upstream is live: {is_live}"); + *live_guard = Some(is_live); + } + } + + if self.effective_aggregate_mode(&settings) + == RtpMpeg4GenericPayAggregateMode::Aggregate + { + if let Some(max_ptime) = settings.max_ptime { + min += max_ptime; + max.opt_add_assign(max_ptime); + } else if is_live { + gst::warning!(CAT, imp: self, + "Aggregating packets in live mode, but no max_ptime configured. \ + Configured latency may be too low!" + ); + } + query.set(is_live, min, max); + } + } + _ => (), + } + + true + } + + fn start(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + *self.is_live.lock().unwrap() = None; + + self.parent_start() + } + + fn stop(&self) -> Result<(), gst::ErrorMessage> { + *self.state.borrow_mut() = State::default(); + *self.is_live.lock().unwrap() = None; + + self.parent_stop() + } +} + +#[derive(Debug, PartialEq)] +enum SendPacketMode { + WhenReady, + ForcePending, +} + +impl RtpMpeg4GenericPay { + fn send_packets( + &self, + settings: &Settings, + state: &mut State, + send_mode: SendPacketMode, + ) -> Result { + let agg_mode = self.effective_aggregate_mode(settings); + + if (self.obj().mtu() as usize) < state.min_mtu { + gst::error!(CAT, imp: self, "Insufficient mtu {} at least {} bytes needed", self.obj().mtu(), state.min_mtu); + return Err(gst::FlowError::Error); + } + + let max_payload_size = self.obj().max_payload_size() as usize - HEADERS_LEN_SIZE; + + let mut ctx = AuHeaderContext { + config: &state.mode, + prev_index: None, + }; + let mut headers_buf = SmallVec::<[u8; 10 * HEADER_MAX_LEN]>::new(); + let mut au_data_list = SmallVec::<[gst::MappedBuffer; 10]>::new(); + + // https://www.rfc-editor.org/rfc/rfc3640.html#section-3.1 + // The M bit is set to 1 to indicate that the RTP packet payload + // contains either the final fragment of a fragmented Access Unit + // or one or more complete Access Units. + + // Send out packets if there's enough data for one (or more), or if forced. + while let Some(front) = state.pending_aus.front() { + headers_buf.clear(); + ctx.prev_index = None; + + if front.buffer.len() + (state.max_header_bit_len + 7) / 8 > max_payload_size { + // AU needs to be fragmented + let au = state.pending_aus.pop_front().unwrap(); + let mut data = au.buffer.as_slice(); + state.pending_size = state.pending_size.saturating_sub(data.len()); + let mut next_frag_offset = 0; + let mut is_final = false; + + while !is_final { + let header = AuHeader { + // The size of the complete AU for all the fragments + size: Some(au.buffer.len() as u32), + // One AU fragment per packet + index: AccessUnitIndex::ZERO, + // CTS-delta SHOULD not be set for a fragment, see § 3.2.1.1 + dts_delta: au.dts_delta, + maybe_random_access: au.maybe_random_access, + ..Default::default() + }; + + headers_buf.clear(); + let mut w = BitWriter::endian(&mut headers_buf, BigEndian); + let mut res = w.build_with(&header, &ctx); + if res.is_ok() { + // add final padding + res = w.write(7, 0).map_err(Into::into); + } + if let Err(err) = res { + gst::error!(CAT, imp: self, "Failed to write header for AU {} in buffer {}: {err:#}", header.index, au.id); + return Err(gst::FlowError::Error); + } + + // Unfortunately BitWriter doesn't return the size written. + let mut c = BitCounter::::new(); + c.build_with(&header, &ctx).unwrap(); + let header_bit_len = c.written() as u16; + + let left = au.buffer.len() - next_frag_offset; + let bytes_in_this_packet = + std::cmp::min(left, max_payload_size - (header_bit_len as usize + 7) / 8); + + next_frag_offset += bytes_in_this_packet; + is_final = next_frag_offset >= au.buffer.len(); + + self.obj().queue_packet( + au.id.into(), + rtp_types::RtpPacketBuilder::new() + // AU-headers-length: only one 1 AU header here + .payload(header_bit_len.to_be_bytes().as_slice()) + .payload(headers_buf.as_slice()) + .payload(&data[0..bytes_in_this_packet]) + .marker_bit(is_final), + )?; + + data = &data[bytes_in_this_packet..]; + } + + continue; + } + + // Will not fragment this AU + + // We optimistically add average size/duration to send out packets as early as possible + // if we estimate that the next AU would likely overflow our accumulation limits. + let n_aus = state.pending_aus.len(); + let avg_size = state.pending_size / n_aus; + let avg_duration = state.pending_duration.opt_div(n_aus as u64); + + let max_ptime = settings + .max_ptime + .opt_min(state.max_ptime) + .opt_min(state.ptime); + + let is_ready = send_mode == SendPacketMode::ForcePending + || agg_mode != RtpMpeg4GenericPayAggregateMode::Aggregate + || state.pending_size + avg_size + n_aus * (state.max_header_bit_len + 7) / 8 + > max_payload_size + || state + .pending_duration + .opt_add(avg_duration) + .opt_gt(max_ptime) + .unwrap_or(false); + + gst::log!(CAT, imp: self, + "Pending: size {}, duration ~{:.3}, mode: {agg_mode:?} + {send_mode:?} => {}", + state.pending_size, + state.pending_duration.display(), + if is_ready { "ready" } else { "not ready, waiting for more data" }, + ); + + if !is_ready { + break; + } + + gst::trace!(CAT, imp: self, "Creating packet.."); + + let id = front.id; + let mut end_id = front.id; + + let mut acc_duration = gst::ClockTime::ZERO; + let mut acc_size = 0; + + let mut headers_len = 0; + + let mut w = BitWriter::endian(&mut headers_buf, BigEndian); + let mut index = AccessUnitIndex::ZERO; + let mut previous_pts = None; + + au_data_list.clear(); + + while let Some(front) = state.pending_aus.front() { + gst::trace!(CAT, imp: self, "{front:?}, accumulated size {acc_size} duration ~{acc_duration:.3}"); + + // If this AU would overflow the packet, bail out and send out what we have. + // + // Don't take into account the max_ptime for the first AU, since it could be + // lower than the AU duration in which case we would never payload anything. + // + // For the size check in bytes we know that the first AU will fit the mtu, + // because we already checked for the "AU needs to be fragmented" scenario above. + + let cts_delta = if ctx.prev_index.is_none() { + // No CTS-delta for the first AU in the packet + None + } else { + ct_delta_to_rtp(front.pts, previous_pts, state.clock_rate).and_then(|dts_delta_res| { + if dts_delta_res.is_none() { + gst::warning!(CAT, imp: self, "Overflow computing CTS-delta between pts {} & previous pts {}", + front.pts.display(), previous_pts.display(), + ); + } + + dts_delta_res + }) + }; + + previous_pts = front.pts; + + let header = AuHeader { + size: Some(front.buffer.len() as u32), + index, + cts_delta, + dts_delta: front.dts_delta, + maybe_random_access: front.maybe_random_access, + ..Default::default() + }; + + w.build_with(&header, &ctx).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to write header for AU {} in buffer {}: {err:#}", + header.index, front.id, + ); + gst::FlowError::Error + })?; + + // Unfortunately BitWriter doesn't return the size written. + let mut c = BitCounter::::new(); + c.build_with(&header, &ctx).unwrap(); + let header_bit_len = c.written() as u16; + + if acc_size + ((headers_len + header_bit_len) as usize + 7) / 8 + front.buffer.len() + > max_payload_size + || (ctx.prev_index.is_some() + && max_ptime + .opt_lt(acc_duration.opt_add(front.duration)) + .unwrap_or(false)) + { + break; + } + + let au = state.pending_aus.pop_front().unwrap(); + + end_id = au.id; + acc_size += au.buffer.len(); + acc_duration.opt_add_assign(au.duration); + + state.pending_size -= au.buffer.len(); + state.pending_duration.opt_saturating_sub(au.duration); + + headers_len += header_bit_len; + au_data_list.push(au.buffer); + + ctx.prev_index = Some(index); + index += 1; + } + + // add final padding + if let Err(err) = w.write(7, 0) { + gst::error!(CAT, imp: self, "Failed to write padding for final AU {} in buffer {end_id}: {err}", + ctx.prev_index.expect("at least one AU"), + ); + return Err(gst::FlowError::Error); + } + + let headers_len = headers_len.to_be_bytes(); + debug_assert_eq!(headers_len.len(), 2); + + let mut packet = rtp_types::RtpPacketBuilder::new() + .marker_bit(true) + .payload(headers_len.as_slice()) + .payload(headers_buf.as_slice()); + + for au_data in &au_data_list { + packet = packet.payload(au_data.as_slice()); + } + + self.obj() + .queue_packet(PacketToBufferRelation::Ids(id..=end_id), packet)?; + } + + gst::log!(CAT, imp: self, "All done for now, {} pending AUs", state.pending_aus.len()); + + if send_mode == SendPacketMode::ForcePending { + self.obj().finish_pending_packets()?; + } + + Ok(gst::FlowSuccess::Ok) + } + + fn effective_aggregate_mode(&self, settings: &Settings) -> RtpMpeg4GenericPayAggregateMode { + match settings.aggregate_mode { + RtpMpeg4GenericPayAggregateMode::Auto => match self.is_live() { + Some(true) => RtpMpeg4GenericPayAggregateMode::ZeroLatency, + Some(false) => RtpMpeg4GenericPayAggregateMode::Aggregate, + None => RtpMpeg4GenericPayAggregateMode::ZeroLatency, + }, + mode => mode, + } + } + + fn is_live(&self) -> Option { + *self.is_live.lock().unwrap() + } + + // Query upstream live-ness if needed, in case of aggregate-mode=auto + fn ensure_upstream_liveness(&self, settings: &mut Settings) { + if settings.aggregate_mode != RtpMpeg4GenericPayAggregateMode::Auto + || self.is_live().is_some() + { + return; + } + + let mut q = gst::query::Latency::new(); + let is_live = if self.obj().sink_pad().peer_query(&mut q) { + let (is_live, _, _) = q.result(); + is_live + } else { + false + }; + + *self.is_live.lock().unwrap() = Some(is_live); + + gst::info!(CAT, imp: self, "Upstream is live: {is_live}"); + } +} diff --git a/net/rtp/src/mp4g/pay/mod.rs b/net/rtp/src/mp4g/pay/mod.rs new file mode 100644 index 00000000..25b94767 --- /dev/null +++ b/net/rtp/src/mp4g/pay/mod.rs @@ -0,0 +1,58 @@ +// GStreamer RTP MPEG-4 Generic Payloader +// +// Copyright (C) 2023-2024 François Laignel +// +// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0. +// If a copy of the MPL was not distributed with this file, You can obtain one at +// . +// +// SPDX-License-Identifier: MPL-2.0 + +use gst::glib; +use gst::prelude::*; + +pub mod imp; + +#[derive(Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Clone, Copy, glib::Enum)] +#[repr(i32)] +#[enum_type(name = "GstRtpMpeg4GenericPayAggregateMode")] +#[non_exhaustive] +pub(crate) enum RtpMpeg4GenericPayAggregateMode { + #[enum_value( + name = "Automatic: zero-latency if upstream is live, otherwise aggregate elementary streams until packet is full.", + nick = "auto" + )] + Auto = -1, + + #[enum_value( + name = "Zero Latency: always send out elementary streams right away, do not wait for more elementary streams to fill a packet.", + nick = "zero-latency" + )] + ZeroLatency = 0, + + #[enum_value( + name = "Aggregate: collect elementary streams until we have a full packet or the max-ptime limit is hit (if set).", + nick = "aggregate" + )] + Aggregate = 1, +} + +glib::wrapper! { + pub struct RtpMpeg4GenericPay(ObjectSubclass) + @extends crate::basepay::RtpBasePay2, gst::Element, gst::Object; +} + +pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { + #[cfg(feature = "doc")] + { + RtpMpeg4GenericPayAggregateMode::static_type() + .mark_as_plugin_api(gst::PluginAPIFlags::empty()); + } + + gst::Element::register( + Some(plugin), + "rtpmp4gpay2", + gst::Rank::MARGINAL, + RtpMpeg4GenericPay::static_type(), + ) +} diff --git a/net/rtp/src/mp4g/tests.rs b/net/rtp/src/mp4g/tests.rs new file mode 100644 index 00000000..b2020778 --- /dev/null +++ b/net/rtp/src/mp4g/tests.rs @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: MPL-2.0 + +use crate::tests::{run_test_pipeline, ExpectedBuffer, ExpectedPacket, Source}; +use gst::prelude::*; + +fn init() { + use std::sync::Once; + static INIT: Once = Once::new(); + + INIT.call_once(|| { + gst::init().unwrap(); + crate::plugin_register_static().expect("rtpmp4g test"); + }); +} + +#[test] +fn aac_hbr_not_fragmented() { + init(); + + let src = + "audiotestsrc num-buffers=100 ! audio/x-raw,rate=48000,channels=2 ! fdkaacenc ! aacparse"; + let pay = "rtpmp4gpay2"; + let depay = "rtpmp4gdepay2"; + + let mut expected_pay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + expected_pay.push(vec![ExpectedPacket::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::MARKER + }) + .rtp_time((position & 0xffff_ffff) as u32) + .build()]); + } + + let mut expected_depay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} + +#[test] +fn aac_hbr_fragmented() { + init(); + + let src = + "audiotestsrc num-buffers=100 ! audio/x-raw,rate=48000,channels=1 ! fdkaacenc ! aacparse"; + let pay = "rtpmp4gpay2 mtu=288"; + let depay = "rtpmp4gdepay2"; + + let mut expected_pay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + let pts = gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + ); + + let rtp_time = (position & 0xffff_ffff) as u32; + + expected_pay.push(vec![ + ExpectedPacket::builder() + .pts(pts) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .rtp_time(rtp_time) + .marker_bit(false) + .build(), + ExpectedPacket::builder() + .pts(pts) + .flags(gst::BufferFlags::MARKER) + .rtp_time(rtp_time) + .build(), + ]); + } + + let mut expected_depay = Vec::with_capacity(102); + for i in 0..102 { + let position = i * 1024; + + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(gst::ClockTime::from_nseconds( + position + .mul_div_floor(*gst::ClockTime::SECOND, 48_000) + .unwrap(), + )) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::empty() + }) + .build()]); + } + + run_test_pipeline(Source::Bin(src), pay, depay, expected_pay, expected_depay); +} + +#[test] +fn generic_not_fragmented() { + const BUFFER_NB: usize = 4; + const BUFFER_SIZE: usize = 600; + const MTU: usize = 1400; + const PACKETS_PER_BUFFER: usize = MTU / BUFFER_SIZE; + const RTP_CLOCK_RATE: u64 = 90_000; + const FRAME_RATE: u64 = 30; + + init(); + + let codec_data = gst::Buffer::from_slice([0x00, 0x00, 0x01, 0xb0, 0x01]); + let caps = gst::Caps::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false) + .field("codec_data", codec_data) + .build(); + + let pos_to_pts = |pos: usize| { + 1000.hours() + + (pos as u64) + .mul_div_floor(*gst::ClockTime::SECOND, FRAME_RATE) + .map(gst::ClockTime::from_nseconds) + .unwrap() + }; + + let pos_to_rtp = |pos: usize| { + ((pos as u64) + .mul_div_ceil(RTP_CLOCK_RATE, FRAME_RATE) + .unwrap() + & 0xffff_ffff) as u32 + }; + + let duration = + gst::ClockTime::from_nseconds(1.mul_div_ceil(*gst::ClockTime::SECOND, FRAME_RATE).unwrap()); + + let mut buffers = Vec::with_capacity(BUFFER_NB); + for pos in 0..BUFFER_NB { + let mut buffer = gst::Buffer::with_size(BUFFER_SIZE).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + let pts = pos_to_pts(pos); + buffer.set_pts(pts); + buffer.set_dts(match pos { + 0 => pts, + 1 | 2 => pos_to_pts(pos + 1), + 3 => pos_to_pts(pos - 2), + _ => unreachable!(), + }); + buffer.set_duration(duration); + if pos == 0 { + buffer.set_flags(gst::BufferFlags::DISCONT); + } else { + buffer.set_flags(gst::BufferFlags::DELTA_UNIT); + } + } + + buffers.push(buffer); + } + + let pay = format!("rtpmp4gpay2 mtu={MTU}"); + let depay = "rtpmp4gdepay2"; + + let mut expected_pay = Vec::with_capacity(BUFFER_NB); + for i in 0..PACKETS_PER_BUFFER { + expected_pay.push(vec![ExpectedPacket::builder() + .pts(pos_to_pts(i * PACKETS_PER_BUFFER)) + .flags(if i == 0 { + gst::BufferFlags::DISCONT | gst::BufferFlags::MARKER + } else { + gst::BufferFlags::MARKER + }) + .rtp_time(pos_to_rtp(i * PACKETS_PER_BUFFER)) + .build()]); + } + + let mut expected_depay = Vec::with_capacity(BUFFER_NB); + for i in 0..BUFFER_NB { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts( + pos_to_pts(i) + + if i == 3 { + 11110.nseconds() + } else { + 0.nseconds() + }, + ) + .dts(match i { + 0 => pos_to_pts(0), + 1 => pos_to_pts(1 + 1), + 2 => pos_to_pts(2 + 1) + 11110.nseconds(), + 3 => pos_to_pts(3 - 2) + 11111.nseconds(), + _ => unreachable!(), + }) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::DELTA_UNIT + }) + .build()]); + } + + run_test_pipeline( + Source::Buffers(caps, buffers), + &pay, + depay, + expected_pay, + expected_depay, + ); +} + +#[test] +fn generic_fragmented() { + const BUFFER_NB: usize = 4; + const BUFFER_SIZE: usize = 2000; + const MTU: usize = 1400; + // Enough overhead in the MTU to use this approximation: + const FRAGMENTS_PER_BUFFER: usize = (BUFFER_SIZE + MTU - 1) / MTU; + const RTP_CLOCK_RATE: u64 = 90_000; + const LAST_FRAGMENT: usize = FRAGMENTS_PER_BUFFER - 1; + const FRAME_RATE: u64 = 30; + + init(); + + let codec_data = gst::Buffer::from_slice([0x00, 0x00, 0x01, 0xb0, 0x01]); + let caps = gst::Caps::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false) + .field("codec_data", codec_data) + .build(); + + let pos_to_pts = |pos: usize| { + 1000.hours() + + (pos as u64) + .mul_div_floor(*gst::ClockTime::SECOND, FRAME_RATE) + .map(gst::ClockTime::from_nseconds) + .unwrap() + }; + + let pos_to_rtp = |pos: usize| { + ((pos as u64) + .mul_div_ceil(RTP_CLOCK_RATE, FRAME_RATE) + .unwrap() + & 0xffff_ffff) as u32 + }; + + let duration = + gst::ClockTime::from_nseconds(1.mul_div_ceil(*gst::ClockTime::SECOND, FRAME_RATE).unwrap()); + + let mut buffers = Vec::with_capacity(BUFFER_NB); + for pos in 0..BUFFER_NB { + let mut buffer = gst::Buffer::with_size(BUFFER_SIZE).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + let pts = pos_to_pts(pos); + buffer.set_pts(pts); + buffer.set_dts(match pos { + 0 => pts, + 1 | 2 => pos_to_pts(pos + 1), + 3 => pos_to_pts(pos - 2), + _ => unreachable!(), + }); + buffer.set_duration(duration); + if pos == 0 { + buffer.set_flags(gst::BufferFlags::DISCONT); + } else { + buffer.set_flags(gst::BufferFlags::DELTA_UNIT); + } + } + + buffers.push(buffer); + } + + let pay = format!("rtpmp4gpay2 mtu={MTU}"); + let depay = "rtpmp4gdepay2"; + + let mut expected_pay = Vec::with_capacity(BUFFER_NB); + for i in 0..BUFFER_NB { + expected_pay.push({ + let mut packets = Vec::with_capacity(FRAGMENTS_PER_BUFFER); + for frag in 0..FRAGMENTS_PER_BUFFER { + packets.push( + ExpectedPacket::builder() + .pts(pos_to_pts(i)) + .flags(match (i, frag) { + (0, 0) => gst::BufferFlags::DISCONT, + (_, LAST_FRAGMENT) => gst::BufferFlags::MARKER, + _ => gst::BufferFlags::empty(), + }) + .rtp_time(pos_to_rtp(i)) + .marker_bit(frag == LAST_FRAGMENT) + .build(), + ); + } + + packets + }); + } + + let mut expected_depay = Vec::with_capacity(BUFFER_NB); + for i in 0..BUFFER_NB { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(pos_to_pts(i)) + .dts(match i { + 0 => pos_to_pts(0), + 1 => pos_to_pts(1 + 1), + 2 => pos_to_pts(2 + 1) + 11110.nseconds(), + 3 => pos_to_pts(3 - 2) + 1.nseconds(), + _ => unreachable!(), + }) + .size(BUFFER_SIZE) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::DELTA_UNIT + }) + .build()]); + } + + run_test_pipeline( + Source::Buffers(caps, buffers), + &pay, + depay, + expected_pay, + expected_depay, + ); +} + +#[test] +fn generic_variable_au_size() { + const MTU: usize = 1400; + const AU_NB: usize = 5; + const SMALL_AU_SIZE: usize = 500; + const LARGE_AU_SIZE: usize = 2000; + const FRAGMENTS_PER_LARGE_BUFFER: usize = (LARGE_AU_SIZE + MTU - 1) / MTU; + const LAST_FRAGMENT: usize = FRAGMENTS_PER_LARGE_BUFFER - 1; + const RTP_CLOCK_RATE: u64 = 90_000; + const FRAME_RATE: u64 = 30; + + init(); + + let codec_data = gst::Buffer::from_slice([0x00, 0x00, 0x01, 0xb0, 0x01]); + let caps = gst::Caps::builder("video/mpeg") + .field("mpegversion", 4i32) + .field("systemstream", false) + .field("codec_data", codec_data) + .build(); + + let pos_to_pts = |pos: usize| { + 1000.hours() + + (pos as u64) + .mul_div_floor(*gst::ClockTime::SECOND, FRAME_RATE) + .map(gst::ClockTime::from_nseconds) + .unwrap() + }; + + let pos_to_rtp = |pos: usize| { + ((pos as u64) + .mul_div_ceil(RTP_CLOCK_RATE, FRAME_RATE) + .unwrap() + & 0xffff_ffff) as u32 + }; + + let duration = + gst::ClockTime::from_nseconds(1.mul_div_ceil(*gst::ClockTime::SECOND, FRAME_RATE).unwrap()); + + let is_large_au = |pos| pos % 4 == 0; + let au_size = |pos| { + if is_large_au(pos) { + LARGE_AU_SIZE + } else { + SMALL_AU_SIZE + } + }; + + let mut buffers = Vec::with_capacity(AU_NB); + for pos in 0..AU_NB { + let mut buffer = gst::Buffer::with_size(au_size(pos)).unwrap(); + { + let buffer = buffer.get_mut().unwrap(); + let pts = pos_to_pts(pos); + buffer.set_pts(pts); + buffer.set_dts(match pos % 4 { + 0 => pts, + 1 | 2 => pos_to_pts(pos + 1), + 3 => pos_to_pts(pos - 2), + _ => unreachable!(), + }); + buffer.set_duration(duration); + if pos == 0 { + buffer.set_flags(gst::BufferFlags::DISCONT); + } else { + buffer.set_flags(gst::BufferFlags::DELTA_UNIT); + } + } + + buffers.push(buffer); + } + + let pay = format!("rtpmp4gpay2 mtu={MTU}"); + let depay = "rtpmp4gdepay2"; + + let mut expected_pay = Vec::with_capacity(AU_NB); + let mut pending_size = 0; + let mut pending_packet = None; + for i in 0..AU_NB { + let size = au_size(i); + if size > MTU { + // Incoming AU to fragment + let mut packet_list = Vec::with_capacity(3); + + if let Some(pending) = pending_packet.take() { + // and there are pending AUs => push them first + packet_list.push(pending); + pending_size = 0; + } + + // Then push the fragments for current AU + for f in 0..FRAGMENTS_PER_LARGE_BUFFER { + packet_list.push( + ExpectedPacket::builder() + .pts(pos_to_pts(i)) + .flags(match (i, f) { + (0, 0) => gst::BufferFlags::DISCONT, + (_, 0) => gst::BufferFlags::empty(), + (_, LAST_FRAGMENT) => gst::BufferFlags::MARKER, + _ => unreachable!(), + }) + .rtp_time(pos_to_rtp(i)) + .marker_bit(f == LAST_FRAGMENT) + .build(), + ) + } + + expected_pay.push(packet_list); + } else { + let must_push = + if i + 1 < AU_NB && pending_size + size + au_size(i + 1) > MTU || i + 1 == AU_NB { + // Next will overflow => push now + // or last AU and not a fragmented one, will be pushed with time deadline + true + } else { + false + }; + + if must_push { + if let Some(pending) = pending_packet.take() { + expected_pay.push(vec![pending]); + pending_size = 0; + } else { + // Last AU + expected_pay.push(vec![ExpectedPacket::builder() + .pts(pos_to_pts(i)) + .flags(gst::BufferFlags::MARKER) + .rtp_time(pos_to_rtp(i)) + .build()]); + } + } else if pending_packet.is_none() { + // Wait for more payload + pending_packet = Some( + ExpectedPacket::builder() + .pts(pos_to_pts(i)) + .flags(gst::BufferFlags::MARKER) + .rtp_time(pos_to_rtp(i)) + .build(), + ); + pending_size = size; + } else { + // There's already a pending packet + pending_size += size; + } + } + } + + let mut expected_depay = Vec::with_capacity(AU_NB); + for i in 0..AU_NB { + expected_depay.push(vec![ExpectedBuffer::builder() + .pts(pos_to_pts(i)) + .dts(match i % 4 { + 0 => pos_to_pts(0), + 1 => pos_to_pts(1 + 1), + 2 => pos_to_pts(2 + 1) + 11110.nseconds(), + 3 => pos_to_pts(3 - 2) + 11111.nseconds(), + _ => unreachable!(), + }) + .size(au_size(i)) + .flags(if i == 0 { + gst::BufferFlags::DISCONT + } else { + gst::BufferFlags::DELTA_UNIT + }) + .build()]); + } + + run_test_pipeline( + Source::Buffers(caps, buffers), + &pay, + depay, + expected_pay, + expected_depay, + ); +} diff --git a/net/rtp/src/utils.rs b/net/rtp/src/utils.rs new file mode 100644 index 00000000..a1ff9a68 --- /dev/null +++ b/net/rtp/src/utils.rs @@ -0,0 +1,539 @@ +/// Computes the seqnum distance +/// +/// This makes sense if both seqnums are in the same cycle. +pub fn seqnum_distance(seqnum1: u16, seqnum2: u16) -> i16 { + // See http://en.wikipedia.org/wiki/Serial_number_arithmetic + + let seqnum1 = i16::from_ne_bytes(seqnum1.to_ne_bytes()); + let seqnum2 = i16::from_ne_bytes(seqnum2.to_ne_bytes()); + + seqnum1.wrapping_sub(seqnum2) +} + +/// Converts a raw two's complement value of len `bit_len` into an i32. +/// +/// # Panic +/// +/// Panics if `bit_len` > 32. +#[inline] +pub fn raw_2_comp_to_i32(val: u32, bit_len: u8) -> i32 { + assert!(bit_len <= 32); + + if val < 1u32 << (bit_len - 1) as u32 { + // val is positive + val as i32 + } else { + ((0x1_0000_0000 - (1u64 << bit_len)) as u32 + val) as i32 + } +} + +/// Masks the provided `i32` value to be used as a two's complement of len `bit_len`, +/// so the resulting value can be passed to APIs which check the bit range. +/// +/// Returns `None` the `i32` value exceeds the range of a two's complement +/// of len `bit_len`. +/// +/// # Panic +/// +/// Panics if `bit_len` > 32. +#[inline] +pub fn mask_valid_2_comp(val: i32, bit_len: u8) -> Option { + let bit_len = bit_len as u32; + + if bit_len == i32::BITS { + return Some(val); + } + + assert!(bit_len < i32::BITS); + + let overhead = i32::BITS - bit_len; + + let leading_zeros = val.leading_zeros(); + if leading_zeros > 0 && leading_zeros < overhead + || leading_zeros == 0 && val.leading_ones() < overhead + { + return None; + } + + Some(((1 << bit_len) - 1) & val) +} + +/// Defines a comparable new type `$typ` on a `[std::num::Wrapping]::`. +/// +/// The new type will wrap-around on additions and substractions and it comparison +/// operators take the wrapping in consideration. +/// +/// The comparison algorithm uses [serial number arithmetic](serial-number-arithmetic). +/// The limit being that it can't tell whether 0x8000_0000 is greater or less than 0. +/// +/// # Examples +/// +/// ```rust +/// # use gstrsrtp::define_wrapping_comparable_u32; +/// +/// /// Error type to return when comparing 0x8000_0000 to 0. +/// struct RTPTimestampComparisonLimit; +/// +/// /// Define the new type comparable and wrapping `u32` `RTPTimestamp`: +/// define_wrapping_comparable_u32!(RTPTimestamp, RTPTimestampComparisonLimit); +/// +/// let ts0 = RTPTimestamp::ZERO; +/// assert!(ts0.is_zero()); +/// +/// let mut ts = ts0; +/// ts += 1; +/// assert_eq!(*ts, 1); +/// assert_eq!(RTPTimestamp::MAX + ts, ts0); +/// +/// let ts2: RTPTimestamp = 2.into(); +/// assert_eq!(*ts2, 2); +/// assert_eq!(ts - ts2, RTPTimestamp::MAX); +/// ``` +/// +/// [serial-number-arithmetic]: http://en.wikipedia.org/wiki/Serial_number_arithmetic +#[macro_export] +macro_rules! define_wrapping_comparable_u32 { + ($typ:ident) => { + #[derive(Clone, Copy, Debug, Default)] + pub struct $typ(std::num::Wrapping); + + impl $typ { + pub const ZERO: $typ = $typ(std::num::Wrapping(0)); + pub const MIN: $typ = $typ(std::num::Wrapping(u32::MIN)); + pub const MAX: $typ = $typ(std::num::Wrapping(u32::MAX)); + pub const NONE: Option<$typ> = None; + + #[inline] + pub const fn new(val: u32) -> Self { + Self(std::num::Wrapping((val))) + } + + #[inline] + pub fn from_ext(ext_val: u64) -> Self { + Self(std::num::Wrapping((ext_val & 0xffff_ffff) as u32)) + } + + #[inline] + pub fn is_zero(self) -> bool { + self.0 .0 == 0 + } + + #[inline] + pub fn distance(self, other: Self) -> Option { + self.distance_u32(other.0 .0) + } + + #[inline] + pub fn distance_u32(self, other: u32) -> Option { + // See http://en.wikipedia.org/wiki/Serial_number_arithmetic + + let this = i32::from_ne_bytes(self.0 .0.to_ne_bytes()); + let other = i32::from_ne_bytes(other.to_ne_bytes()); + + match this.wrapping_sub(other) { + -0x8000_0000 => { + // This is the limit of the algorithm: + // arguments are too far away to determine the result sign, + // i.e. which one is greater than the other + None + } + delta => Some(delta), + } + } + } + + impl From for $typ { + fn from(value: u32) -> Self { + Self(std::num::Wrapping(value)) + } + } + + impl From<$typ> for u32 { + fn from(value: $typ) -> Self { + value.0 .0 + } + } + + impl std::ops::Deref for $typ { + type Target = u32; + + fn deref(&self) -> &u32 { + &self.0 .0 + } + } + + impl std::ops::Add for $typ { + type Output = Self; + fn add(self, rhs: Self) -> Self { + Self(self.0.add(rhs.0)) + } + } + + impl std::ops::Add for $typ { + type Output = Self; + fn add(self, rhs: u32) -> Self { + Self(self.0.add(std::num::Wrapping(rhs))) + } + } + + impl std::ops::Add for $typ { + type Output = Self; + fn add(self, rhs: i32) -> Self { + // See http://en.wikipedia.org/wiki/Serial_number_arithmetic + + let this = i32::from_ne_bytes(self.0 .0.to_ne_bytes()); + let res = this.wrapping_add(rhs); + + let res = u32::from_ne_bytes(res.to_ne_bytes()); + Self(std::num::Wrapping(res)) + } + } + + impl std::ops::AddAssign for $typ { + fn add_assign(&mut self, rhs: Self) { + self.0.add_assign(rhs.0); + } + } + + impl std::ops::AddAssign for $typ { + fn add_assign(&mut self, rhs: u32) { + self.0.add_assign(std::num::Wrapping(rhs)); + } + } + + impl std::ops::AddAssign for $typ { + fn add_assign(&mut self, rhs: i32) { + *self = *self + rhs; + } + } + + impl std::ops::Sub for $typ { + type Output = Self; + fn sub(self, rhs: Self) -> Self { + self.sub(rhs.0 .0) + } + } + + impl std::ops::Sub for $typ { + type Output = Self; + fn sub(self, rhs: u32) -> Self { + Self(self.0.sub(std::num::Wrapping(rhs))) + } + } + + impl std::ops::SubAssign for $typ { + fn sub_assign(&mut self, rhs: Self) { + self.sub_assign(rhs.0 .0); + } + } + + impl std::ops::SubAssign for $typ { + fn sub_assign(&mut self, rhs: u32) { + self.0.sub_assign(std::num::Wrapping(rhs)); + } + } + + impl std::cmp::PartialEq for $typ { + fn eq(&self, other: &Self) -> bool { + self.0 .0 == other.0 .0 + } + } + + impl std::cmp::PartialEq for $typ { + fn eq(&self, other: &u32) -> bool { + self.0 .0 == *other + } + } + + impl std::cmp::Eq for $typ {} + + impl std::cmp::PartialOrd for $typ { + fn partial_cmp(&self, other: &Self) -> Option { + self.distance(*other).map(|d| d.cmp(&0)) + } + } + + impl gst::prelude::OptionOperations for $typ {} + }; + + ($typ:ident, $comp_err_type:ident) => { + define_wrapping_comparable_u32!($typ); + + impl $typ { + #[inline] + pub fn try_cmp(&self, other: $typ) -> Result { + self.partial_cmp(&other).ok_or($comp_err_type) + } + } + }; + + ($typ:ident, $err_enum:ty, $comp_err_variant:ident) => { + define_wrapping_comparable_u32!($typ); + + impl $typ { + #[inline] + pub fn try_cmp(&self, other: $typ) -> Result { + self.partial_cmp(&other) + .ok_or(<$err_enum>::$comp_err_variant) + } + } + }; +} + +#[macro_export] +macro_rules! define_wrapping_comparable_u32_with_display { + ($typ:ident, impl) => { + impl std::fmt::Display for $typ { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!("{}", self.0 .0)) + } + } + }; + + ($typ:ident) => { + define_wrapping_comparable_u32!($typ); + define_wrapping_comparable_u32_with_display!($typ, impl); + }; + + ($typ:ident, $comp_err_type:ty) => { + define_wrapping_comparable_u32!($typ, $comp_err_type); + define_wrapping_comparable_u32_with_display!($typ, impl); + }; + + ($typ:ident, $err_enum:ty, $comp_err_variant:ident,) => { + define_wrapping_comparable_u32!($typ, $err_enum, $comp_err_variant); + define_wrapping_comparable_u32_with_display!($typ, impl); + }; +} + +#[cfg(test)] +mod tests { + use super::*; + define_wrapping_comparable_u32!(MyWrapper); + + #[test] + fn compare_seqnums() { + assert_eq!(seqnum_distance(0, 1), -1); + assert_eq!(seqnum_distance(1, 1), 0); + assert_eq!(seqnum_distance(1, 0), 1); + + assert_eq!(seqnum_distance(0x7fff, 0), 0x7fff); + assert_eq!(seqnum_distance(0xffff, 0), -1); + + assert_eq!(seqnum_distance(0, 0x7fff), -0x7fff); + assert_eq!(seqnum_distance(0, 0xffff), 1); + + // This is the limit of the algorithm: + assert_eq!(seqnum_distance(0x8000, 0), -0x8000); + assert_eq!(seqnum_distance(0, 0x8000), -0x8000); + } + + #[test] + fn raw_2_comp_12bits_to_i32() { + const BITS: u8 = 12; + assert_eq!(raw_2_comp_to_i32(0, BITS), 0); + assert_eq!(raw_2_comp_to_i32(1, BITS), 1); + assert_eq!(raw_2_comp_to_i32(2, BITS), 2); + assert_eq!(raw_2_comp_to_i32(0xfff, BITS), -1i16 as i32); + assert_eq!(raw_2_comp_to_i32(0xffe, BITS), -2i16 as i32); + assert_eq!(raw_2_comp_to_i32(0x7ff, BITS), (1 << (BITS - 1)) - 1); + assert_eq!(raw_2_comp_to_i32(0x800, BITS), -(1 << (BITS - 1))); + } + + #[test] + fn raw_2_comp_16bits_to_i32() { + const BITS: u8 = i16::BITS as u8; + assert_eq!(raw_2_comp_to_i32(0, BITS), 0); + assert_eq!(raw_2_comp_to_i32(1, BITS), 1); + assert_eq!(raw_2_comp_to_i32(2, BITS), 2); + assert_eq!(raw_2_comp_to_i32(0xffff, BITS), -1i16 as i32); + assert_eq!(raw_2_comp_to_i32(0xfffe, BITS), -2i16 as i32); + assert_eq!(raw_2_comp_to_i32(0x7fff, BITS), i16::MAX as i32); + assert_eq!(raw_2_comp_to_i32(0x8000, BITS), i16::MIN as i32); + } + + #[test] + fn raw_2_comp_32bits_to_i32() { + const BITS: u8 = i32::BITS as u8; + assert_eq!(raw_2_comp_to_i32(0, BITS), 0); + assert_eq!(raw_2_comp_to_i32(1, BITS), 1); + assert_eq!(raw_2_comp_to_i32(2, BITS), 2); + assert_eq!(raw_2_comp_to_i32(0xffff_ffff, BITS), -1i16 as i32); + assert_eq!(raw_2_comp_to_i32(0xffff_fffe, BITS), -2i16 as i32); + assert_eq!(raw_2_comp_to_i32(0x7fff_ffff, BITS), i32::MAX); + assert_eq!(raw_2_comp_to_i32(0x8000_0000, BITS), i32::MIN); + } + + #[test] + fn mask_valid_2_comp_ok() { + const BITS: u8 = i32::BITS as u8; + assert_eq!(mask_valid_2_comp(0, BITS), Some(0)); + assert_eq!(mask_valid_2_comp(-1, BITS), Some(-1)); + assert_eq!(mask_valid_2_comp(i32::MIN, BITS), Some(i32::MIN)); + assert_eq!(mask_valid_2_comp(i32::MAX, BITS), Some(i32::MAX)); + + assert_eq!(mask_valid_2_comp(0, 6), Some(0)); + assert_eq!(mask_valid_2_comp(0x2f, 6), Some(0x2f)); // -1i6 + assert_eq!(mask_valid_2_comp(0x20, 6), Some(0x20)); // i6::MIN + assert_eq!(mask_valid_2_comp(0x1f, 6), Some(0x1f)); // i6::MAX + assert_eq!(mask_valid_2_comp(0x1f, 5), Some(0x1f)); // i6::MAX => -1i5 + } + + #[test] + fn mask_valid_2_comp_ko() { + const BITS: u8 = i32::BITS as u8; + assert_eq!(mask_valid_2_comp(0, BITS), Some(0)); + assert_eq!(mask_valid_2_comp(-1, BITS), Some(-1)); + assert_eq!(mask_valid_2_comp(i32::MIN, BITS), Some(i32::MIN)); + assert_eq!(mask_valid_2_comp(i32::MAX, BITS), Some(i32::MAX)); + + assert_eq!(mask_valid_2_comp(0, 5), Some(0)); + assert!(mask_valid_2_comp(0x2f, 5).is_none()); // -1i6 + assert!(mask_valid_2_comp(0x20, 5).is_none()); // i6::MIN + } + + #[test] + fn wrapping_u32_basics() { + let zero = MyWrapper::ZERO; + let one = MyWrapper::from(1); + let two = MyWrapper::from(2); + + assert_eq!(u32::from(zero), 0); + assert!(zero.is_zero()); + assert_eq!(u32::from(one), 1); + assert_eq!(u32::from(two), 2); + + let max_plus_1_u64 = MyWrapper::from_ext((u32::MAX as u64) + 1); + assert_eq!(max_plus_1_u64, MyWrapper::ZERO); + } + + #[test] + fn add_wrapping_u32() { + let one = MyWrapper::from(1); + let two = MyWrapper::from(2); + + assert_eq!(MyWrapper::ZERO + one, one); + assert_eq!(MyWrapper::ZERO + 1u32, one); + assert_eq!(one + one, two); + assert_eq!(one + 1u32, two); + + assert_eq!(MyWrapper::MAX + MyWrapper::ZERO, MyWrapper::MAX); + assert_eq!(MyWrapper::MAX + one, MyWrapper::ZERO); + assert_eq!(MyWrapper::MAX + two, one); + + let mut var = MyWrapper::ZERO; + assert!(var.is_zero()); + var += 1; + assert_eq!(var, one); + var += one; + assert_eq!(var, two); + + let mut var = MyWrapper::MAX; + var += 1; + assert!(var.is_zero()); + var += one; + assert_eq!(var, one); + } + + #[test] + fn add_wrapping_u32_i32() { + let one = MyWrapper::from(1); + + assert_eq!(MyWrapper::ZERO + 1i32, one); + assert_eq!(MyWrapper::ZERO + -1i32, MyWrapper::MAX); + assert_eq!(MyWrapper::MAX + 1i32, MyWrapper::ZERO); + assert_eq!(MyWrapper::MAX + 2i32, one); + + assert_eq!( + MyWrapper::from(0x8000_0000) + -0i32, + MyWrapper::from(0x8000_0000) + ); + assert_eq!( + MyWrapper::from(0x8000_0000) + 1i32, + MyWrapper::from(0x8000_0001) + ); + assert_eq!( + MyWrapper::from(0x8000_0000) + -1i32, + MyWrapper::from(0x7fff_ffff) + ); + assert_eq!( + MyWrapper::from(0x7fff_ffff) + 1i32, + MyWrapper::from(0x8000_0000) + ); + assert_eq!(MyWrapper::ZERO + i32::MIN, MyWrapper::from(0x8000_0000)); + + let mut var = MyWrapper::ZERO; + var += 1i32; + assert_eq!(var, one); + + let mut var = MyWrapper::ZERO; + var += -1i32; + assert_eq!(var, MyWrapper::MAX); + + let mut var = MyWrapper::MAX; + var += 1; + assert_eq!(var, MyWrapper::ZERO); + } + + #[test] + fn sub_wrapping_u32() { + let one = MyWrapper::from(1); + + assert_eq!(MyWrapper::ZERO - MyWrapper::ZERO, MyWrapper::ZERO); + assert_eq!(MyWrapper::MAX - MyWrapper::MAX, MyWrapper::ZERO); + assert_eq!(MyWrapper::ZERO - one, MyWrapper::MAX); + assert_eq!(MyWrapper::ZERO - MyWrapper::MAX, one); + assert_eq!( + MyWrapper::ZERO - MyWrapper::from(0x8000_0000), + MyWrapper::from(0x8000_0000) + ); + assert_eq!( + MyWrapper::from(0x8000_0000) - MyWrapper::ZERO, + MyWrapper::from(0x8000_0000) + ); + + let mut var = MyWrapper::ZERO; + assert!(var.is_zero()); + var -= 1; + assert_eq!(var, MyWrapper::MAX); + + let mut var = MyWrapper::MAX; + var -= MyWrapper::MAX; + assert!(var.is_zero()); + } + + #[test] + fn compare_wrapping_u32() { + use std::cmp::Ordering::*; + + #[derive(Debug, PartialEq)] + pub struct ComparisonLimit; + define_wrapping_comparable_u32!(MyWrapper, ComparisonLimit); + + let cmp = |a: u32, b: u32| MyWrapper::from(a).partial_cmp(&MyWrapper::from(b)); + let try_cmp = |a: u32, b: u32| MyWrapper::from(a).try_cmp(MyWrapper::from(b)); + + assert_eq!(cmp(0, 1).unwrap(), Less); + assert_eq!(try_cmp(0, 1), Ok(Less)); + assert_eq!(cmp(1, 1).unwrap(), Equal); + assert_eq!(try_cmp(1, 1), Ok(Equal)); + assert_eq!(cmp(1, 0).unwrap(), Greater); + assert_eq!(try_cmp(1, 0), Ok(Greater)); + + assert_eq!(cmp(0x7fff_ffff, 0).unwrap(), Greater); + assert_eq!(try_cmp(0x7fff_ffff, 0), Ok(Greater)); + assert_eq!(cmp(0xffff_ffff, 0).unwrap(), Less); + assert_eq!(try_cmp(0xffff_ffff, 0), Ok(Less)); + + assert_eq!(cmp(0, 0x7fff_ffff).unwrap(), Less); + assert_eq!(try_cmp(0, 0x7fff_ffff), Ok(Less)); + assert_eq!(cmp(0, 0xffff_ffff).unwrap(), Greater); + assert_eq!(try_cmp(0, 0xffff_ffff), Ok(Greater)); + + // This is the limit of the algorithm: + assert!(cmp(0x8000_0000, 0).is_none()); + assert!(cmp(0, 0x8000_0000).is_none()); + assert_eq!(try_cmp(0x8000_0000, 0), Err(ComparisonLimit)); + assert_eq!(try_cmp(0, 0x8000_0000), Err(ComparisonLimit)); + } +}