fmp4mux: Add ONVIF variant with support for additional codecs

This variant supports H264/5, JPEG, alaw, mulaw and G726.

generic/fmp4/src/fmp4mux/boxes.rs

@@ -335,7 +335,7 @@ fn brands_from_variant_and_caps(
     caps: &[&gst::Caps],
 ) -> (&'static [u8; 4], Vec<&'static [u8; 4]>) {
     match variant {
-        super::Variant::ISO => (b"iso6", vec![b"iso6"]),
+        super::Variant::ISO | super::Variant::ONVIF => (b"iso6", vec![b"iso6"]),
         super::Variant::DASH => {
             // FIXME: `dsms` / `dash` brands, `msix`
             (b"msdh", vec![b"dums", b"msdh", b"iso6"])
@@ -524,7 +524,9 @@ fn write_tkhd(
     // Volume
     let s = caps.structure(0).unwrap();
     match s.name() {
-        "audio/mpeg" => v.extend((1u16 << 8).to_be_bytes()),
+        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+            v.extend((1u16 << 8).to_be_bytes())
+        }
         _ => v.extend(0u16.to_be_bytes()),
     }
 
@@ -550,7 +552,7 @@ fn write_tkhd(
 
     // Width/height
     match s.name() {
-        "video/x-h264" | "video/x-h265" => {
+        "video/x-h264" | "video/x-h265" | "image/jpeg" => {
             let width = s.get::<i32>("width").context("video caps without width")? as u32;
             let height = s
                 .get::<i32>("height")
@@ -642,8 +644,10 @@ fn write_hdlr(
 
     let s = caps.structure(0).unwrap();
     let (handler_type, name) = match s.name() {
-        "video/x-h264" | "video/x-h265" => (b"vide", b"VideoHandler\0"),
+        "video/x-h264" | "video/x-h265" | "image/jpeg" => (b"vide", b"VideoHandler\0"),
-        "audio/mpeg" => (b"soun", b"SoundHandler\0"),
+        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+            (b"soun", b"SoundHandler\0")
+        }
         _ => unreachable!(),
     };
 
@@ -667,13 +671,15 @@ fn write_minf(
     let s = caps.structure(0).unwrap();
 
     match s.name() {
-        "video/x-h264" | "video/x-h265" => {
+        "video/x-h264" | "video/x-h265" | "image/jpeg" => {
             // Flags are always 1 for unspecified reasons
             write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))?
         }
-        "audio/mpeg" => write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
+        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
-            write_smhd(v, cfg)
+            write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
-        })?,
+                write_smhd(v, cfg)
+            })?
+        }
         _ => unreachable!(),
     }
 
@@ -775,8 +781,10 @@ fn write_stsd(
 
     let s = caps.structure(0).unwrap();
     match s.name() {
-        "video/x-h264" | "video/x-h265" => write_visual_sample_entry(v, cfg, caps)?,
+        "video/x-h264" | "video/x-h265" | "image/jpeg" => write_visual_sample_entry(v, cfg, caps)?,
-        "audio/mpeg" => write_audio_sample_entry(v, cfg, caps)?,
+        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+            write_audio_sample_entry(v, cfg, caps)?
+        }
         _ => unreachable!(),
     }
 
@@ -822,6 +830,7 @@ fn write_visual_sample_entry(
                 _ => unreachable!(),
             }
         }
+        "image/jpeg" => b"jpeg",
         _ => unreachable!(),
     };
 
@@ -890,6 +899,9 @@ fn write_visual_sample_entry(
                     Ok(())
                 })?;
             }
+            "image/jpeg" => {
+                // Nothing to do here
+            }
             _ => unreachable!(),
         }
 
@@ -995,6 +1007,37 @@ fn write_visual_sample_entry(
             }
         }
 
+        // Write fiel box for codecs that require it
+        if ["image/jpeg"].contains(&s.name()) {
+            let interlace_mode = s
+                .get::<&str>("interlace-mode")
+                .ok()
+                .map(gst_video::VideoInterlaceMode::from_string)
+                .unwrap_or(gst_video::VideoInterlaceMode::Progressive);
+            let field_order = s
+                .get::<&str>("field-order")
+                .ok()
+                .map(gst_video::VideoFieldOrder::from_string)
+                .unwrap_or(gst_video::VideoFieldOrder::Unknown);
+
+            write_box(v, b"fiel", move |v| {
+                let (interlace, field_order) = match interlace_mode {
+                    gst_video::VideoInterlaceMode::Progressive => (1, 0),
+                    gst_video::VideoInterlaceMode::Interleaved
+                        if field_order == gst_video::VideoFieldOrder::TopFieldFirst =>
+                    {
+                        (2, 9)
+                    }
+                    gst_video::VideoInterlaceMode::Interleaved => (2, 14),
+                    _ => (0, 0),
+                };
+
+                v.push(interlace);
+                v.push(field_order);
+                Ok(())
+            })?;
+        }
+
         // TODO: write btrt bitrate box based on tags
 
         Ok(())
@@ -1011,9 +1054,27 @@ fn write_audio_sample_entry(
     let s = caps.structure(0).unwrap();
     let fourcc = match s.name() {
         "audio/mpeg" => b"mp4a",
+        "audio/x-alaw" => b"alaw",
+        "audio/x-mulaw" => b"ulaw",
+        "audio/x-adpcm" => {
+            let layout = s.get::<&str>("layout").context("no ADPCM layout field")?;
+
+            match layout {
+                "g726" => b"ms\x00\x45",
+                _ => unreachable!(),
+            }
+        }
         _ => unreachable!(),
     };
 
+    let sample_size = match s.name() {
+        "audio/x-adpcm" => {
+            let bitrate = s.get::<i32>("bitrate").context("no ADPCM bitrate field")?;
+            (bitrate / 8000) as u16
+        }
+        _ => 16u16,
+    };
+
     write_sample_entry_box(v, fourcc, move |v| {
         // Reserved
         v.extend([0u8; 2 * 4]);
@@ -1024,7 +1085,7 @@ fn write_audio_sample_entry(
         v.extend(channels.to_be_bytes());
 
         // Sample size
-        v.extend(16u16.to_be_bytes());
+        v.extend(sample_size.to_be_bytes());
 
         // Pre-defined
         v.extend([0u8; 2]);
@@ -1050,6 +1111,9 @@ fn write_audio_sample_entry(
                 }
                 write_esds_aac(v, &map)?;
             }
+            "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+                // Nothing to do here
+            }
             _ => unreachable!(),
         }
 
@@ -1615,7 +1679,10 @@ fn write_traf(
     let timescale = caps_to_timescale(caps);
 
     let check_dts = matches!(s.name(), "video/x-h264" | "video/x-h265");
-    let intra_only = matches!(s.name(), "audio/mpeg");
+    let intra_only = matches!(
+        s.name(),
+        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" | "image/jpeg"
+    );
 
     // Analyze all buffers to know what values can be put into the tfhd for all samples and what
     // has to be stored for every single sample

generic/fmp4/src/fmp4mux/imp.rs

@@ -844,6 +844,9 @@ impl FMP4Mux {
                         return Err(gst::FlowError::NotNegotiated);
                     }
                 }
+                "image/jpeg" => {
+                    intra_only = true;
+                }
                 "audio/mpeg" => {
                     if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) {
                         gst::error!(CAT, obj: &pad, "Received caps without codec_data");
@@ -851,6 +854,12 @@ impl FMP4Mux {
                     }
                     intra_only = true;
                 }
+                "audio/x-alaw" | "audio/x-mulaw" => {
+                    intra_only = true;
+                }
+                "audio/x-adpcm" => {
+                    intra_only = true;
+                }
                 _ => unreachable!(),
             }
 
@@ -950,7 +959,7 @@ impl FMP4Mux {
         state.stream_header = Some(buffer.clone());
 
         let variant = match variant {
-            super::Variant::ISO | super::Variant::DASH => "iso-fragmented",
+            super::Variant::ISO | super::Variant::DASH | super::Variant::ONVIF => "iso-fragmented",
             super::Variant::CMAF => "cmaf",
         };
         let caps = gst::Caps::builder("video/quicktime")
@@ -1872,3 +1881,103 @@ impl AggregatorImpl for DASHMP4Mux {}
 impl FMP4MuxImpl for DASHMP4Mux {
     const VARIANT: super::Variant = super::Variant::DASH;
 }
+
+#[derive(Default)]
+pub(crate) struct ONVIFFMP4Mux;
+
+#[glib::object_subclass]
+impl ObjectSubclass for ONVIFFMP4Mux {
+    const NAME: &'static str = "GstONVIFFMP4Mux";
+    type Type = super::ONVIFFMP4Mux;
+    type ParentType = super::FMP4Mux;
+}
+
+impl ObjectImpl for ONVIFFMP4Mux {}
+
+impl GstObjectImpl for ONVIFFMP4Mux {}
+
+impl ElementImpl for ONVIFFMP4Mux {
+    fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
+        static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
+            gst::subclass::ElementMetadata::new(
+                "ONVIFFMP4Mux",
+                "Codec/Muxer",
+                "ONVIF fragmented MP4 muxer",
+                "Sebastian Dröge <sebastian@centricular.com>",
+            )
+        });
+
+        Some(&*ELEMENT_METADATA)
+    }
+
+    fn pad_templates() -> &'static [gst::PadTemplate] {
+        static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
+            let src_pad_template = gst::PadTemplate::new(
+                "src",
+                gst::PadDirection::Src,
+                gst::PadPresence::Always,
+                &gst::Caps::builder("video/quicktime")
+                    .field("variant", "iso-fragmented")
+                    .build(),
+            )
+            .unwrap();
+
+            let sink_pad_template = gst::PadTemplate::new(
+                "sink_%u",
+                gst::PadDirection::Sink,
+                gst::PadPresence::Request,
+                &[
+                    gst::Structure::builder("video/x-h264")
+                        .field("stream-format", gst::List::new(&[&"avc", &"avc3"]))
+                        .field("alignment", "au")
+                        .field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .build(),
+                    gst::Structure::builder("video/x-h265")
+                        .field("stream-format", gst::List::new(&[&"hvc1", &"hev1"]))
+                        .field("alignment", "au")
+                        .field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .build(),
+                    gst::Structure::builder("image/jpeg")
+                        .field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .build(),
+                    gst::Structure::builder("audio/mpeg")
+                        .field("mpegversion", 4i32)
+                        .field("stream-format", "raw")
+                        .field("channels", gst::IntRange::<i32>::new(1, u16::MAX as i32))
+                        .field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
+                        .build(),
+                    gst::Structure::builder("audio/x-alaw")
+                        .field("channels", gst::IntRange::<i32>::new(1, 2))
+                        .field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
+                        .build(),
+                    gst::Structure::builder("audio/x-mulaw")
+                        .field("channels", gst::IntRange::<i32>::new(1, 2))
+                        .field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
+                        .build(),
+                    gst::Structure::builder("audio/x-adpcm")
+                        .field("layout", "g726")
+                        .field("channels", 1i32)
+                        .field("rate", 8000i32)
+                        .field("bitrate", gst::List::new([16000i32, 24000, 32000, 40000]))
+                        .build(),
+                ]
+                .into_iter()
+                .collect::<gst::Caps>(),
+            )
+            .unwrap();
+
+            vec![src_pad_template, sink_pad_template]
+        });
+
+        PAD_TEMPLATES.as_ref()
+    }
+}
+
+impl AggregatorImpl for ONVIFFMP4Mux {}
+
+impl FMP4MuxImpl for ONVIFFMP4Mux {
+    const VARIANT: super::Variant = super::Variant::ONVIF;
+}

generic/fmp4/src/fmp4mux/mod.rs

@@ -28,6 +28,10 @@ glib::wrapper! {
     pub(crate) struct DASHMP4Mux(ObjectSubclass<imp::DASHMP4Mux>) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object;
 }
 
+glib::wrapper! {
+    pub(crate) struct ONVIFFMP4Mux(ObjectSubclass<imp::ONVIFFMP4Mux>) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object;
+}
+
 pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
     gst::Element::register(
         Some(plugin),
@@ -47,6 +51,12 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
         gst::Rank::Primary,
         DASHMP4Mux::static_type(),
     )?;
+    gst::Element::register(
+        Some(plugin),
+        "onviffmp4mux",
+        gst::Rank::Primary,
+        ONVIFFMP4Mux::static_type(),
+    )?;
 
     Ok(())
 }
@@ -97,12 +107,13 @@ pub(crate) enum Variant {
     ISO,
     CMAF,
     DASH,
+    ONVIF,
 }
 
 impl Variant {
     pub(crate) fn is_single_stream(self) -> bool {
         match self {
-            Variant::ISO => false,
+            Variant::ISO | Variant::ONVIF => false,
             Variant::CMAF | Variant::DASH => true,
         }
     }