fmp4mux: Add initial Opus support

Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/239

docs/plugins/gst_plugins_cache.json

@@ -1587,7 +1587,7 @@
                 "long-name": "CMAFMux",
                 "pad-templates": {
                     "sink": {
-                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\n",
+                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\n",
                         "direction": "sink",
                         "presence": "always"
                     },
@@ -1615,7 +1615,7 @@
                 "long-name": "DASHMP4Mux",
                 "pad-templates": {
                     "sink": {
-                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\n",
+                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n       channels: [ 1, 8 ]\n           rate: [ 1, 2147483647 ]\n",
                         "direction": "sink",
                         "presence": "always"
                     },
@@ -1643,7 +1643,7 @@
                 "long-name": "ISOFMP4Mux",
                 "pad-templates": {
                     "sink_%%u": {
-                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\n",
+                        "caps": "video/x-h264:\n  stream-format: { (string)avc, (string)avc3 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-h265:\n  stream-format: { (string)hvc1, (string)hev1 }\n      alignment: au\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\nvideo/x-vp9:\n        profile: { (string)0, (string)1, (string)2, (string)3 }\n  chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n          width: [ 1, 65535 ]\n         height: [ 1, 65535 ]\naudio/mpeg:\n    mpegversion: 4\n  stream-format: raw\n       channels: [ 1, 65535 ]\n           rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n       channels: [ 1, 8 ]\n           rate: [ 1, 2147483647 ]\n",
                         "direction": "sink",
                         "presence": "request"
                     },

mux/fmp4/Cargo.toml

@@ -14,6 +14,7 @@ gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/g
 gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
 gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
 gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst-pbutils = { package = "gstreamer-pbutils", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
 once_cell = "1.0"
 
 [lib]

mux/fmp4/src/fmp4mux/boxes.rs

@@ -593,7 +593,7 @@ fn write_tkhd(
     // Volume
     let s = caps.structure(0).unwrap();
     match s.name() {
-        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+        "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
             v.extend((1u16 << 8).to_be_bytes())
         }
         _ => v.extend(0u16.to_be_bytes()),
@@ -734,7 +734,7 @@ fn write_hdlr(
         "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
             (b"vide", b"VideoHandler\0".as_slice())
         }
-        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+        "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
             (b"soun", b"SoundHandler\0".as_slice())
         }
         "application/x-onvif-metadata" => (b"meta", b"MetadataHandler\0".as_slice()),
@@ -765,7 +765,7 @@ fn write_minf(
             // Flags are always 1 for unspecified reasons
             write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))?
         }
-        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+        "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
             write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
                 write_smhd(v, cfg)
             })?
@@ -879,7 +879,7 @@ fn write_stsd(
         "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => {
             write_visual_sample_entry(v, cfg, caps)?
         }
-        "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
+        "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
             write_audio_sample_entry(v, cfg, caps)?
         }
         "application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, caps)?,
@@ -1216,6 +1216,7 @@ fn write_audio_sample_entry(
     let s = caps.structure(0).unwrap();
     let fourcc = match s.name() {
         "audio/mpeg" => b"mp4a",
+        "audio/x-opus" => b"Opus",
         "audio/x-alaw" => b"alaw",
         "audio/x-mulaw" => b"ulaw",
         "audio/x-adpcm" => {
@@ -1273,6 +1274,9 @@ fn write_audio_sample_entry(
                 }
                 write_esds_aac(v, &map)?;
             }
+            "audio/x-opus" => {
+                write_dops(v, caps)?;
+            }
             "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
                 // Nothing to do here
             }
@@ -1408,6 +1412,70 @@ fn write_esds_aac(v: &mut Vec<u8>, codec_data: &[u8]) -> Result<(), Error> {
     )
 }
 
+fn write_dops(v: &mut Vec<u8>, caps: &gst::CapsRef) -> Result<(), Error> {
+    let rate;
+    let channels;
+    let channel_mapping_family;
+    let stream_count;
+    let coupled_count;
+    let pre_skip;
+    let output_gain;
+    let mut channel_mapping = [0; 256];
+
+    // TODO: Use audio clipping meta to calculate pre_skip
+
+    if let Some(header) = caps
+        .structure(0)
+        .unwrap()
+        .get::<gst::ArrayRef>("streamheader")
+        .ok()
+        .and_then(|a| a.get(0).and_then(|v| v.get::<gst::Buffer>().ok()))
+    {
+        (
+            rate,
+            channels,
+            channel_mapping_family,
+            stream_count,
+            coupled_count,
+            pre_skip,
+            output_gain,
+        ) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping))
+            .unwrap();
+    } else {
+        // FIXME: Workaround for below function taking a &Caps instead of &CapsRef
+        // SAFETY: This is OK because we only get an immutable reference and don't
+        // clone it, so nobody will be able to get a mutable reference to the caps.
+        let caps = unsafe { &*(&caps as *const &gst::CapsRef as *const gst::Caps) };
+
+        (
+            rate,
+            channels,
+            channel_mapping_family,
+            stream_count,
+            coupled_count,
+        ) = gst_pbutils::codec_utils_opus_parse_caps(caps, Some(&mut channel_mapping)).unwrap();
+        output_gain = 0;
+        pre_skip = 0;
+    }
+
+    write_box(v, b"dOps", move |v| {
+        // Version number
+        v.push(0);
+        v.push(channels);
+        v.extend(pre_skip.to_le_bytes());
+        v.extend(rate.to_le_bytes());
+        v.extend(output_gain.to_le_bytes());
+        v.push(channel_mapping_family);
+        if channel_mapping_family > 0 {
+            v.push(stream_count);
+            v.push(coupled_count);
+            v.extend(&channel_mapping[..channels as usize]);
+        }
+
+        Ok(())
+    })
+}
+
 fn write_xml_meta_data_sample_entry(
     v: &mut Vec<u8>,
     _cfg: &super::HeaderConfiguration,

mux/fmp4/src/fmp4mux/imp.rs

@@ -1499,6 +1499,21 @@ impl FMP4Mux {
                         return Err(gst::FlowError::NotNegotiated);
                     }
                 }
+                "audio/x-opus" => {
+                    if let Some(header) = s
+                        .get::<gst::ArrayRef>("streamheader")
+                        .ok()
+                        .and_then(|a| a.get(0).and_then(|v| v.get::<gst::Buffer>().ok()))
+                    {
+                        if gst_pbutils::codec_utils_opus_parse_header(&header, None).is_err() {
+                            gst::error!(CAT, obj: pad, "Received invalid Opus header");
+                            return Err(gst::FlowError::NotNegotiated);
+                        }
+                    } else if gst_pbutils::codec_utils_opus_parse_caps(&caps, None).is_err() {
+                        gst::error!(CAT, obj: pad, "Received invalid Opus caps");
+                        return Err(gst::FlowError::NotNegotiated);
+                    }
+                }
                 "audio/x-alaw" | "audio/x-mulaw" => (),
                 "audio/x-adpcm" => (),
                 "application/x-onvif-metadata" => (),
@@ -2362,6 +2377,11 @@ impl ElementImpl for ISOFMP4Mux {
                         .field("channels", gst::IntRange::new(1, u16::MAX as i32))
                         .field("rate", gst::IntRange::new(1, i32::MAX))
                         .build(),
+                    gst::Structure::builder("audio/x-opus")
+                        .field("channel-mapping-family", gst::IntRange::new(0i32, 255))
+                        .field("channels", gst::IntRange::new(1i32, 8))
+                        .field("rate", gst::IntRange::new(1, i32::MAX))
+                        .build(),
                 ]
                 .into_iter()
                 .collect::<gst::Caps>(),
@@ -2542,6 +2562,11 @@ impl ElementImpl for DASHMP4Mux {
                         .field("channels", gst::IntRange::<i32>::new(1, u16::MAX as i32))
                         .field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
                         .build(),
+                    gst::Structure::builder("audio/x-opus")
+                        .field("channel-mapping-family", gst::IntRange::new(0i32, 255))
+                        .field("channels", gst::IntRange::new(1i32, 8))
+                        .field("rate", gst::IntRange::new(1, i32::MAX))
+                        .build(),
                 ]
                 .into_iter()
                 .collect::<gst::Caps>(),