mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-11-22 03:21:00 +00:00
Code Issues Projects Releases Wiki Activity

fmp4mux: Add ONVIF variant with support for additional codecs

Browse source 
This variant supports H264/5, JPEG, alaw, mulaw and G726.
This commit is contained in:
Sebastian Dröge 2022-05-13 11:45:01 +03:00
parent 0206178279
commit e4634ca2fe
3 changed files with 202 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
generic/fmp4/src/fmp4mux/boxes.rs
View file

@ -335,7 +335,7 @@ fn brands_from_variant_and_caps(
caps: &[&gst::Caps],
) -> (&'static [u8; 4], Vec<&'static [u8; 4]>) {
match variant {
super::Variant::ISO => (b"iso6", vec![b"iso6"]),
super::Variant::ISO | super::Variant::ONVIF => (b"iso6", vec![b"iso6"]),
super::Variant::DASH => {
// FIXME: `dsms` / `dash` brands, `msix`
(b"msdh", vec![b"dums", b"msdh", b"iso6"])
@ -524,7 +524,9 @@ fn write_tkhd(
// Volume
let s = caps.structure(0).unwrap();
match s.name() {
"audio/mpeg" => v.extend((1u16 << 8).to_be_bytes()),
"audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
v.extend((1u16 << 8).to_be_bytes())
}
_ => v.extend(0u16.to_be_bytes()),
}
@ -550,7 +552,7 @@ fn write_tkhd(
// Width/height
match s.name() {
"video/x-h264" | "video/x-h265" => {
"video/x-h264" | "video/x-h265" | "image/jpeg" => {
let width = s.get::<i32>("width").context("video caps without width")? as u32;
let height = s
.get::<i32>("height")
@ -642,8 +644,10 @@ fn write_hdlr(
let s = caps.structure(0).unwrap();
let (handler_type, name) = match s.name() {
"video/x-h264" | "video/x-h265" => (b"vide", b"VideoHandler\0"),
"audio/mpeg" => (b"soun", b"SoundHandler\0"),
"video/x-h264" | "video/x-h265" | "image/jpeg" => (b"vide", b"VideoHandler\0"),
"audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
(b"soun", b"SoundHandler\0")
}
_ => unreachable!(),
};
@ -667,13 +671,15 @@ fn write_minf(
let s = caps.structure(0).unwrap();
match s.name() {
"video/x-h264" | "video/x-h265" => {
"video/x-h264" | "video/x-h265" | "image/jpeg" => {
// Flags are always 1 for unspecified reasons
write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))?
}
"audio/mpeg" => write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_smhd(v, cfg)
})?,
"audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
write_full_box(v, b"smhd", FULL_BOX_VERSION_0, FULL_BOX_FLAGS_NONE, |v| {
write_smhd(v, cfg)
})?
}
_ => unreachable!(),
}
@ -775,8 +781,10 @@ fn write_stsd(
let s = caps.structure(0).unwrap();
match s.name() {
"video/x-h264" | "video/x-h265" => write_visual_sample_entry(v, cfg, caps)?,
"audio/mpeg" => write_audio_sample_entry(v, cfg, caps)?,
"video/x-h264" | "video/x-h265" | "image/jpeg" => write_visual_sample_entry(v, cfg, caps)?,
"audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
write_audio_sample_entry(v, cfg, caps)?
}
_ => unreachable!(),
}
@ -822,6 +830,7 @@ fn write_visual_sample_entry(
_ => unreachable!(),
}
}
"image/jpeg" => b"jpeg",
_ => unreachable!(),
};
@ -890,6 +899,9 @@ fn write_visual_sample_entry(
Ok(())
})?;
}
"image/jpeg" => {
// Nothing to do here
}
_ => unreachable!(),
}
@ -995,6 +1007,37 @@ fn write_visual_sample_entry(
}
}
// Write fiel box for codecs that require it
if ["image/jpeg"].contains(&s.name()) {
let interlace_mode = s
.get::<&str>("interlace-mode")
.ok()
.map(gst_video::VideoInterlaceMode::from_string)
.unwrap_or(gst_video::VideoInterlaceMode::Progressive);
let field_order = s
.get::<&str>("field-order")
.ok()
.map(gst_video::VideoFieldOrder::from_string)
.unwrap_or(gst_video::VideoFieldOrder::Unknown);
write_box(v, b"fiel", move |v| {
let (interlace, field_order) = match interlace_mode {
gst_video::VideoInterlaceMode::Progressive => (1, 0),
gst_video::VideoInterlaceMode::Interleaved
if field_order == gst_video::VideoFieldOrder::TopFieldFirst =>
{
(2, 9)
}
gst_video::VideoInterlaceMode::Interleaved => (2, 14),
_ => (0, 0),
};
v.push(interlace);
v.push(field_order);
Ok(())
})?;
}
// TODO: write btrt bitrate box based on tags
Ok(())
@ -1011,9 +1054,27 @@ fn write_audio_sample_entry(
let s = caps.structure(0).unwrap();
let fourcc = match s.name() {
"audio/mpeg" => b"mp4a",
"audio/x-alaw" => b"alaw",
"audio/x-mulaw" => b"ulaw",
"audio/x-adpcm" => {
let layout = s.get::<&str>("layout").context("no ADPCM layout field")?;
match layout {
"g726" => b"ms\x00\x45",
_ => unreachable!(),
}
}
_ => unreachable!(),
};
let sample_size = match s.name() {
"audio/x-adpcm" => {
let bitrate = s.get::<i32>("bitrate").context("no ADPCM bitrate field")?;
(bitrate / 8000) as u16
}
_ => 16u16,
};
write_sample_entry_box(v, fourcc, move |v| {
// Reserved
v.extend([0u8; 2 * 4]);
@ -1024,7 +1085,7 @@ fn write_audio_sample_entry(
v.extend(channels.to_be_bytes());
// Sample size
v.extend(16u16.to_be_bytes());
v.extend(sample_size.to_be_bytes());
// Pre-defined
v.extend([0u8; 2]);
@ -1050,6 +1111,9 @@ fn write_audio_sample_entry(
}
write_esds_aac(v, &map)?;
}
"audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => {
// Nothing to do here
}
_ => unreachable!(),
}
@ -1615,7 +1679,10 @@ fn write_traf(
let timescale = caps_to_timescale(caps);
let check_dts = matches!(s.name(), "video/x-h264" | "video/x-h265");
let intra_only = matches!(s.name(), "audio/mpeg");
let intra_only = matches!(
s.name(),
"audio/mpeg" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" | "image/jpeg"
);
// Analyze all buffers to know what values can be put into the tfhd for all samples and what
// has to be stored for every single sample

111
generic/fmp4/src/fmp4mux/imp.rs
View file

@ -844,6 +844,9 @@ impl FMP4Mux {
return Err(gst::FlowError::NotNegotiated);
}
}
"image/jpeg" => {
intra_only = true;
}
"audio/mpeg" => {
if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) {
gst::error!(CAT, obj: &pad, "Received caps without codec_data");
@ -851,6 +854,12 @@ impl FMP4Mux {
}
intra_only = true;
}
"audio/x-alaw" | "audio/x-mulaw" => {
intra_only = true;
}
"audio/x-adpcm" => {
intra_only = true;
}
_ => unreachable!(),
}
@ -950,7 +959,7 @@ impl FMP4Mux {
state.stream_header = Some(buffer.clone());
let variant = match variant {
super::Variant::ISO | super::Variant::DASH => "iso-fragmented",
super::Variant::ISO | super::Variant::DASH | super::Variant::ONVIF => "iso-fragmented",
super::Variant::CMAF => "cmaf",
};
let caps = gst::Caps::builder("video/quicktime")
@ -1872,3 +1881,103 @@ impl AggregatorImpl for DASHMP4Mux {}
impl FMP4MuxImpl for DASHMP4Mux {
const VARIANT: super::Variant = super::Variant::DASH;
}
#[derive(Default)]
pub(crate) struct ONVIFFMP4Mux;
#[glib::object_subclass]
impl ObjectSubclass for ONVIFFMP4Mux {
const NAME: &'static str = "GstONVIFFMP4Mux";
type Type = super::ONVIFFMP4Mux;
type ParentType = super::FMP4Mux;
}
impl ObjectImpl for ONVIFFMP4Mux {}
impl GstObjectImpl for ONVIFFMP4Mux {}
impl ElementImpl for ONVIFFMP4Mux {
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
gst::subclass::ElementMetadata::new(
"ONVIFFMP4Mux",
"Codec/Muxer",
"ONVIF fragmented MP4 muxer",
"Sebastian Dröge <sebastian@centricular.com>",
)
});
Some(&*ELEMENT_METADATA)
}
fn pad_templates() -> &'static [gst::PadTemplate] {
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
let src_pad_template = gst::PadTemplate::new(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&gst::Caps::builder("video/quicktime")
.field("variant", "iso-fragmented")
.build(),
)
.unwrap();
let sink_pad_template = gst::PadTemplate::new(
"sink_%u",
gst::PadDirection::Sink,
gst::PadPresence::Request,
&[
gst::Structure::builder("video/x-h264")
.field("stream-format", gst::List::new(&[&"avc", &"avc3"]))
.field("alignment", "au")
.field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.build(),
gst::Structure::builder("video/x-h265")
.field("stream-format", gst::List::new(&[&"hvc1", &"hev1"]))
.field("alignment", "au")
.field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.build(),
gst::Structure::builder("image/jpeg")
.field("width", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.field("height", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.build(),
gst::Structure::builder("audio/mpeg")
.field("mpegversion", 4i32)
.field("stream-format", "raw")
.field("channels", gst::IntRange::<i32>::new(1, u16::MAX as i32))
.field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
.build(),
gst::Structure::builder("audio/x-alaw")
.field("channels", gst::IntRange::<i32>::new(1, 2))
.field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
.build(),
gst::Structure::builder("audio/x-mulaw")
.field("channels", gst::IntRange::<i32>::new(1, 2))
.field("rate", gst::IntRange::<i32>::new(1, i32::MAX))
.build(),
gst::Structure::builder("audio/x-adpcm")
.field("layout", "g726")
.field("channels", 1i32)
.field("rate", 8000i32)
.field("bitrate", gst::List::new([16000i32, 24000, 32000, 40000]))
.build(),
]
.into_iter()
.collect::<gst::Caps>(),
)
.unwrap();
vec![src_pad_template, sink_pad_template]
});
PAD_TEMPLATES.as_ref()
}
}
impl AggregatorImpl for ONVIFFMP4Mux {}
impl FMP4MuxImpl for ONVIFFMP4Mux {
const VARIANT: super::Variant = super::Variant::ONVIF;
}

13
generic/fmp4/src/fmp4mux/mod.rs
View file

@ -28,6 +28,10 @@ glib::wrapper! {
pub(crate) struct DASHMP4Mux(ObjectSubclass<imp::DASHMP4Mux>) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object;
}
glib::wrapper! {
pub(crate) struct ONVIFFMP4Mux(ObjectSubclass<imp::ONVIFFMP4Mux>) @extends FMP4Mux, gst_base::Aggregator, gst::Element, gst::Object;
}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Element::register(
Some(plugin),
@ -47,6 +51,12 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Rank::Primary,
DASHMP4Mux::static_type(),
)?;
gst::Element::register(
Some(plugin),
"onviffmp4mux",
gst::Rank::Primary,
ONVIFFMP4Mux::static_type(),
)?;
Ok(())
}
@ -97,12 +107,13 @@ pub(crate) enum Variant {
ISO,
CMAF,
DASH,
ONVIF,
}
impl Variant {
pub(crate) fn is_single_stream(self) -> bool {
match self {
Variant::ISO => false,
Variant::ISO | Variant::ONVIF => false,
Variant::CMAF | Variant::DASH => true,
}
}