From e4634ca2fe9ea55b885bf431e94d5b3891729eec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 13 May 2022 11:45:01 +0300 Subject: [PATCH] fmp4mux: Add ONVIF variant with support for additional codecs This variant supports H264/5, JPEG, alaw, mulaw and G726. --- generic/fmp4/src/fmp4mux/boxes.rs | 93 +++++++++++++++++++++---- generic/fmp4/src/fmp4mux/imp.rs | 111 +++++++++++++++++++++++++++++- generic/fmp4/src/fmp4mux/mod.rs | 13 +++- 3 files changed, 202 insertions(+), 15 deletions(-) diff --git a/generic/fmp4/src/fmp4mux/boxes.rs b/generic/fmp4/src/fmp4mux/boxes.rs index 4548ca58..fbae0dfe 100644 --- a/generic/fmp4/src/fmp4mux/boxes.rs +++ b/generic/fmp4/src/fmp4mux/boxes.rs @@ -335,7 +335,7 @@ fn brands_from_variant_and_caps( caps: &[&gst::Caps], ) -> (&'static [u8; 4], Vec<&'static [u8; 4]>) { match variant { - super::Variant::ISO => (b"iso6", vec![b"iso6"]), + super::Variant::ISO | super::Variant::ONVIF => (b"iso6", vec![b"iso6"]), super::Variant::DASH => { // FIXME: `dsms` / `dash` brands, `msix` (b"msdh", vec![b"dums", b"msdh", b"iso6"]) @@ -524,7 +524,9 @@ fn write_tkhd( // Volume let s = caps.structure(0).unwrap(); match s.name() { - "audio/mpeg" => v.extend((1u16 << 8).to_be_bytes()), + "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + v.extend((1u16 << 8).to_be_bytes()) + } _ => v.extend(0u16.to_be_bytes()), } @@ -550,7 +552,7 @@ fn write_tkhd( // Width/height match s.name() { - "video/x-h264" | "video/x-h265" => { + "video/x-h264" | "video/x-h265" | "image/jpeg" => { let width = s.get::("width").context("video caps without width")? as u32; let height = s .get::("height") @@ -642,8 +644,10 @@ fn write_hdlr( let s = caps.structure(0).unwrap(); let (handler_type, name) = match s.name() { - "video/x-h264" | "video/x-h265" => (b"vide", b"VideoHandler\0"), - "audio/mpeg" => (b"soun", b"SoundHandler\0"), + "video/x-h264" | "video/x-h265" | "image/jpeg" => (b"vide", b"VideoHandler\0"), + "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + (b"soun", b"SoundHandler\0") + } _ => unreachable!(), }; @@ -667,13 +671,15 @@ fn write_minf( let s = caps.structure(0).unwrap(); match s.name() { - "video/x-h264" | "video/x-h265" => { + "video/x-h264" | "video/x-h265" | "image/jpeg" => { // Flags are always 1 for unspecified reasons write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))? } - "audio/mpeg" => write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { - write_smhd(v, cfg) - })?, + "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { + write_smhd(v, cfg) + })? + } _ => unreachable!(), } @@ -775,8 +781,10 @@ fn write_stsd( let s = caps.structure(0).unwrap(); match s.name() { - "video/x-h264" | "video/x-h265" => write_visual_sample_entry(v, cfg, caps)?, - "audio/mpeg" => write_audio_sample_entry(v, cfg, caps)?, + "video/x-h264" | "video/x-h265" | "image/jpeg" => write_visual_sample_entry(v, cfg, caps)?, + "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + write_audio_sample_entry(v, cfg, caps)? + } _ => unreachable!(), } @@ -822,6 +830,7 @@ fn write_visual_sample_entry( _ => unreachable!(), } } + "image/jpeg" => b"jpeg", _ => unreachable!(), }; @@ -890,6 +899,9 @@ fn write_visual_sample_entry( Ok(()) })?; } + "image/jpeg" => { + // Nothing to do here + } _ => unreachable!(), } @@ -995,6 +1007,37 @@ fn write_visual_sample_entry( } } + // Write fiel box for codecs that require it + if ["image/jpeg"].contains(&s.name()) { + let interlace_mode = s + .get::<&str>("interlace-mode") + .ok() + .map(gst_video::VideoInterlaceMode::from_string) + .unwrap_or(gst_video::VideoInterlaceMode::Progressive); + let field_order = s + .get::<&str>("field-order") + .ok() + .map(gst_video::VideoFieldOrder::from_string) + .unwrap_or(gst_video::VideoFieldOrder::Unknown); + + write_box(v, b"fiel", move |v| { + let (interlace, field_order) = match interlace_mode { + gst_video::VideoInterlaceMode::Progressive => (1, 0), + gst_video::VideoInterlaceMode::Interleaved + if field_order == gst_video::VideoFieldOrder::TopFieldFirst => + { + (2, 9) + } + gst_video::VideoInterlaceMode::Interleaved => (2, 14), + _ => (0, 0), + }; + + v.push(interlace); + v.push(field_order); + Ok(()) + })?; + } + // TODO: write btrt bitrate box based on tags Ok(()) @@ -1011,9 +1054,27 @@ fn write_audio_sample_entry( let s = caps.structure(0).unwrap(); let fourcc = match s.name() { "audio/mpeg" => b"mp4a", + "audio/x-alaw" => b"alaw", + "audio/x-mulaw" => b"ulaw", + "audio/x-adpcm" => { + let layout = s.get::<&str>("layout").context("no ADPCM layout field")?; + + match layout { + "g726" => b"ms\x00\x45", + _ => unreachable!(), + } + } _ => unreachable!(), }; + let sample_size = match s.name() { + "audio/x-adpcm" => { + let bitrate = s.get::("bitrate").context("no ADPCM bitrate field")?; + (bitrate / 8000) as u16 + } + _ => 16u16, + }; + write_sample_entry_box(v, fourcc, move |v| { // Reserved v.extend([0u8; 2 * 4]); @@ -1024,7 +1085,7 @@ fn write_audio_sample_entry( v.extend(channels.to_be_bytes()); // Sample size - v.extend(16u16.to_be_bytes()); + v.extend(sample_size.to_be_bytes()); // Pre-defined v.extend([0u8; 2]); @@ -1050,6 +1111,9 @@ fn write_audio_sample_entry( } write_esds_aac(v, &map)?; } + "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + // Nothing to do here + } _ => unreachable!(), } @@ -1615,7 +1679,10 @@ fn write_traf( let timescale = caps_to_timescale(caps); let check_dts = matches!(s.name(), "video/x-h264" | "video/x-h265"); - let intra_only = matches!(s.name(), "audio/mpeg"); + let intra_only = matches!( + s.name(), + "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" | "image/jpeg" + ); // Analyze all buffers to know what values can be put into the tfhd for all samples and what // has to be stored for every single sample diff --git a/generic/fmp4/src/fmp4mux/imp.rs b/generic/fmp4/src/fmp4mux/imp.rs index 5fc0e91b..d5774cc5 100644 --- a/generic/fmp4/src/fmp4mux/imp.rs +++ b/generic/fmp4/src/fmp4mux/imp.rs @@ -844,6 +844,9 @@ impl FMP4Mux { return Err(gst::FlowError::NotNegotiated); } } + "image/jpeg" => { + intra_only = true; + } "audio/mpeg" => { if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) { gst::error!(CAT, obj: &pad, "Received caps without codec_data"); @@ -851,6 +854,12 @@ impl FMP4Mux { } intra_only = true; } + "audio/x-alaw" | "audio/x-mulaw" => { + intra_only = true; + } + "audio/x-adpcm" => { + intra_only = true; + } _ => unreachable!(), } @@ -950,7 +959,7 @@ impl FMP4Mux { state.stream_header = Some(buffer.clone()); let variant = match variant { - super::Variant::ISO | super::Variant::DASH => "iso-fragmented", + super::Variant::ISO | super::Variant::DASH | super::Variant::ONVIF => "iso-fragmented", super::Variant::CMAF => "cmaf", }; let caps = gst::Caps::builder("video/quicktime") @@ -1872,3 +1881,103 @@ impl AggregatorImpl for DASHMP4Mux {} impl FMP4MuxImpl for DASHMP4Mux { const VARIANT: super::Variant = super::Variant::DASH; } + +#[derive(Default)] +pub(crate) struct ONVIFFMP4Mux; + +#[glib::object_subclass] +impl ObjectSubclass for ONVIFFMP4Mux { + const NAME: &'static str = "GstONVIFFMP4Mux"; + type Type = super::ONVIFFMP4Mux; + type ParentType = super::FMP4Mux; +} + +impl ObjectImpl for ONVIFFMP4Mux {} + +impl GstObjectImpl for ONVIFFMP4Mux {} + +impl ElementImpl for ONVIFFMP4Mux { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static ELEMENT_METADATA: Lazy = Lazy::new(|| { + gst::subclass::ElementMetadata::new( + "ONVIFFMP4Mux", + "Codec/Muxer", + "ONVIF fragmented MP4 muxer", + "Sebastian Dröge ", + ) + }); + + Some(&*ELEMENT_METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static PAD_TEMPLATES: Lazy> = Lazy::new(|| { + let src_pad_template = gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &gst::Caps::builder("video/quicktime") + .field("variant", "iso-fragmented") + .build(), + ) + .unwrap(); + + let sink_pad_template = gst::PadTemplate::new( + "sink_%u", + gst::PadDirection::Sink, + gst::PadPresence::Request, + &[ + gst::Structure::builder("video/x-h264") + .field("stream-format", gst::List::new(&[&"avc", &"avc3"])) + .field("alignment", "au") + .field("width", gst::IntRange::::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("video/x-h265") + .field("stream-format", gst::List::new(&[&"hvc1", &"hev1"])) + .field("alignment", "au") + .field("width", gst::IntRange::::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("image/jpeg") + .field("width", gst::IntRange::::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::::new(1, u16::MAX as i32)) + .build(), + gst::Structure::builder("audio/mpeg") + .field("mpegversion", 4i32) + .field("stream-format", "raw") + .field("channels", gst::IntRange::::new(1, u16::MAX as i32)) + .field("rate", gst::IntRange::::new(1, i32::MAX)) + .build(), + gst::Structure::builder("audio/x-alaw") + .field("channels", gst::IntRange::::new(1, 2)) + .field("rate", gst::IntRange::::new(1, i32::MAX)) + .build(), + gst::Structure::builder("audio/x-mulaw") + .field("channels", gst::IntRange::::new(1, 2)) + .field("rate", gst::IntRange::::new(1, i32::MAX)) + .build(), + gst::Structure::builder("audio/x-adpcm") + .field("layout", "g726") + .field("channels", 1i32) + .field("rate", 8000i32) + .field("bitrate", gst::List::new([16000i32, 24000, 32000, 40000])) + .build(), + ] + .into_iter() + .collect::(), + ) + .unwrap(); + + vec![src_pad_template, sink_pad_template] + }); + + PAD_TEMPLATES.as_ref() + } +} + +impl AggregatorImpl for ONVIFFMP4Mux {} + +impl FMP4MuxImpl for ONVIFFMP4Mux { + const VARIANT: super::Variant = super::Variant::ONVIF; +} diff --git a/generic/fmp4/src/fmp4mux/mod.rs b/generic/fmp4/src/fmp4mux/mod.rs index 98e328a3..a88db6bf 100644 --- a/generic/fmp4/src/fmp4mux/mod.rs +++ b/generic/fmp4/src/fmp4mux/mod.rs @@ -28,6 +28,10 @@ glib::wrapper! { pub(crate) struct DASHMP4Mux(ObjectSubclass) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object; } +glib::wrapper! { + pub(crate) struct ONVIFFMP4Mux(ObjectSubclass) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object; +} + pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { gst::Element::register( Some(plugin), @@ -47,6 +51,12 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { gst::Rank::Primary, DASHMP4Mux::static_type(), )?; + gst::Element::register( + Some(plugin), + "onviffmp4mux", + gst::Rank::Primary, + ONVIFFMP4Mux::static_type(), + )?; Ok(()) } @@ -97,12 +107,13 @@ pub(crate) enum Variant { ISO, CMAF, DASH, + ONVIF, } impl Variant { pub(crate) fn is_single_stream(self) -> bool { match self { - Variant::ISO => false, + Variant::ISO | Variant::ONVIF => false, Variant::CMAF | Variant::DASH => true, } }