From 6706f3a4b4d0aa0e167f48cd18c4384d06f6e13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 3 Nov 2022 16:01:35 +0200 Subject: [PATCH] fmp4mux: Add initial Opus support Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/239 --- docs/plugins/gst_plugins_cache.json | 6 +-- mux/fmp4/Cargo.toml | 1 + mux/fmp4/src/fmp4mux/boxes.rs | 76 +++++++++++++++++++++++++++-- mux/fmp4/src/fmp4mux/imp.rs | 25 ++++++++++ 4 files changed, 101 insertions(+), 7 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 6ba6624a..4ebd6c25 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -1587,7 +1587,7 @@ "long-name": "CMAFMux", "pad-templates": { "sink": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "always" }, @@ -1615,7 +1615,7 @@ "long-name": "DASHMP4Mux", "pad-templates": { "sink": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "always" }, @@ -1643,7 +1643,7 @@ "long-name": "ISOFMP4Mux", "pad-templates": { "sink_%%u": { - "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\n", + "caps": "video/x-h264:\n stream-format: { (string)avc, (string)avc3 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-h265:\n stream-format: { (string)hvc1, (string)hev1 }\n alignment: au\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\nvideo/x-vp9:\n profile: { (string)0, (string)1, (string)2, (string)3 }\n chroma-format: { (string)4:2:0, (string)4:2:2, (string)4:4:4 }\n bit-depth-luma: { (uint)8, (uint)10, (uint)12 }\nbit-depth-chroma: { (uint)8, (uint)10, (uint)12 }\n width: [ 1, 65535 ]\n height: [ 1, 65535 ]\naudio/mpeg:\n mpegversion: 4\n stream-format: raw\n channels: [ 1, 65535 ]\n rate: [ 1, 2147483647 ]\naudio/x-opus:\nchannel-mapping-family: [ 0, 255 ]\n channels: [ 1, 8 ]\n rate: [ 1, 2147483647 ]\n", "direction": "sink", "presence": "request" }, diff --git a/mux/fmp4/Cargo.toml b/mux/fmp4/Cargo.toml index aebb2f80..7de120c1 100644 --- a/mux/fmp4/Cargo.toml +++ b/mux/fmp4/Cargo.toml @@ -14,6 +14,7 @@ gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/g gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } gst-video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } +gst-pbutils = { package = "gstreamer-pbutils", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" } once_cell = "1.0" [lib] diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 2a6cc9a1..23546a35 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -593,7 +593,7 @@ fn write_tkhd( // Volume let s = caps.structure(0).unwrap(); match s.name() { - "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { v.extend((1u16 << 8).to_be_bytes()) } _ => v.extend(0u16.to_be_bytes()), @@ -734,7 +734,7 @@ fn write_hdlr( "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { (b"vide", b"VideoHandler\0".as_slice()) } - "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { (b"soun", b"SoundHandler\0".as_slice()) } "application/x-onvif-metadata" => (b"meta", b"MetadataHandler\0".as_slice()), @@ -765,7 +765,7 @@ fn write_minf( // Flags are always 1 for unspecified reasons write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))? } - "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| { write_smhd(v, cfg) })? @@ -879,7 +879,7 @@ fn write_stsd( "video/x-h264" | "video/x-h265" | "video/x-vp9" | "image/jpeg" => { write_visual_sample_entry(v, cfg, caps)? } - "audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { + "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { write_audio_sample_entry(v, cfg, caps)? } "application/x-onvif-metadata" => write_xml_meta_data_sample_entry(v, cfg, caps)?, @@ -1216,6 +1216,7 @@ fn write_audio_sample_entry( let s = caps.structure(0).unwrap(); let fourcc = match s.name() { "audio/mpeg" => b"mp4a", + "audio/x-opus" => b"Opus", "audio/x-alaw" => b"alaw", "audio/x-mulaw" => b"ulaw", "audio/x-adpcm" => { @@ -1273,6 +1274,9 @@ fn write_audio_sample_entry( } write_esds_aac(v, &map)?; } + "audio/x-opus" => { + write_dops(v, caps)?; + } "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { // Nothing to do here } @@ -1408,6 +1412,70 @@ fn write_esds_aac(v: &mut Vec, codec_data: &[u8]) -> Result<(), Error> { ) } +fn write_dops(v: &mut Vec, caps: &gst::CapsRef) -> Result<(), Error> { + let rate; + let channels; + let channel_mapping_family; + let stream_count; + let coupled_count; + let pre_skip; + let output_gain; + let mut channel_mapping = [0; 256]; + + // TODO: Use audio clipping meta to calculate pre_skip + + if let Some(header) = caps + .structure(0) + .unwrap() + .get::("streamheader") + .ok() + .and_then(|a| a.get(0).and_then(|v| v.get::().ok())) + { + ( + rate, + channels, + channel_mapping_family, + stream_count, + coupled_count, + pre_skip, + output_gain, + ) = gst_pbutils::codec_utils_opus_parse_header(&header, Some(&mut channel_mapping)) + .unwrap(); + } else { + // FIXME: Workaround for below function taking a &Caps instead of &CapsRef + // SAFETY: This is OK because we only get an immutable reference and don't + // clone it, so nobody will be able to get a mutable reference to the caps. + let caps = unsafe { &*(&caps as *const &gst::CapsRef as *const gst::Caps) }; + + ( + rate, + channels, + channel_mapping_family, + stream_count, + coupled_count, + ) = gst_pbutils::codec_utils_opus_parse_caps(caps, Some(&mut channel_mapping)).unwrap(); + output_gain = 0; + pre_skip = 0; + } + + write_box(v, b"dOps", move |v| { + // Version number + v.push(0); + v.push(channels); + v.extend(pre_skip.to_le_bytes()); + v.extend(rate.to_le_bytes()); + v.extend(output_gain.to_le_bytes()); + v.push(channel_mapping_family); + if channel_mapping_family > 0 { + v.push(stream_count); + v.push(coupled_count); + v.extend(&channel_mapping[..channels as usize]); + } + + Ok(()) + }) +} + fn write_xml_meta_data_sample_entry( v: &mut Vec, _cfg: &super::HeaderConfiguration, diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index 788d4d5e..56dcd8ad 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -1499,6 +1499,21 @@ impl FMP4Mux { return Err(gst::FlowError::NotNegotiated); } } + "audio/x-opus" => { + if let Some(header) = s + .get::("streamheader") + .ok() + .and_then(|a| a.get(0).and_then(|v| v.get::().ok())) + { + if gst_pbutils::codec_utils_opus_parse_header(&header, None).is_err() { + gst::error!(CAT, obj: pad, "Received invalid Opus header"); + return Err(gst::FlowError::NotNegotiated); + } + } else if gst_pbutils::codec_utils_opus_parse_caps(&caps, None).is_err() { + gst::error!(CAT, obj: pad, "Received invalid Opus caps"); + return Err(gst::FlowError::NotNegotiated); + } + } "audio/x-alaw" | "audio/x-mulaw" => (), "audio/x-adpcm" => (), "application/x-onvif-metadata" => (), @@ -2362,6 +2377,11 @@ impl ElementImpl for ISOFMP4Mux { .field("channels", gst::IntRange::new(1, u16::MAX as i32)) .field("rate", gst::IntRange::new(1, i32::MAX)) .build(), + gst::Structure::builder("audio/x-opus") + .field("channel-mapping-family", gst::IntRange::new(0i32, 255)) + .field("channels", gst::IntRange::new(1i32, 8)) + .field("rate", gst::IntRange::new(1, i32::MAX)) + .build(), ] .into_iter() .collect::(), @@ -2542,6 +2562,11 @@ impl ElementImpl for DASHMP4Mux { .field("channels", gst::IntRange::::new(1, u16::MAX as i32)) .field("rate", gst::IntRange::::new(1, i32::MAX)) .build(), + gst::Structure::builder("audio/x-opus") + .field("channel-mapping-family", gst::IntRange::new(0i32, 255)) + .field("channels", gst::IntRange::new(1i32, 8)) + .field("rate", gst::IntRange::new(1, i32::MAX)) + .build(), ] .into_iter() .collect::(),