From 4c3ae6f8ce53c27afbd58764b3110ade4f5b1e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 2 Feb 2023 16:12:31 +0200 Subject: [PATCH] fmp4: Add support for AV1 Part-of: --- mux/fmp4/src/fmp4mux/boxes.rs | 95 ++++++++++++++++++++++++++++++++--- mux/fmp4/src/fmp4mux/imp.rs | 29 +++++++++++ 2 files changed, 116 insertions(+), 8 deletions(-) diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index 4d8575cd..720c823e 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -632,7 +632,8 @@ fn write_tkhd( // Width/height match s.name().as_str() { - "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "image/jpeg" => { + "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "video/x-av1" + | "image/jpeg" => { let width = s.get::("width").context("video caps without width")? as u32; let height = s .get::("height") @@ -742,9 +743,8 @@ fn write_hdlr( let s = stream.caps.structure(0).unwrap(); let (handler_type, name) = match s.name().as_str() { - "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "image/jpeg" => { - (b"vide", b"VideoHandler\0".as_slice()) - } + "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "video/x-av1" + | "image/jpeg" => (b"vide", b"VideoHandler\0".as_slice()), "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { (b"soun", b"SoundHandler\0".as_slice()) } @@ -772,7 +772,8 @@ fn write_minf( let s = stream.caps.structure(0).unwrap(); match s.name().as_str() { - "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "image/jpeg" => { + "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "video/x-av1" + | "image/jpeg" => { // Flags are always 1 for unspecified reasons write_full_box(v, b"vmhd", FULL_BOX_VERSION_0, 1, |v| write_vmhd(v, cfg))? } @@ -883,9 +884,8 @@ fn write_stsd( let s = stream.caps.structure(0).unwrap(); match s.name().as_str() { - "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "image/jpeg" => { - write_visual_sample_entry(v, cfg, stream)? - } + "video/x-h264" | "video/x-h265" | "video/x-vp8" | "video/x-vp9" | "video/x-av1" + | "image/jpeg" => write_visual_sample_entry(v, cfg, stream)?, "audio/mpeg" | "audio/x-opus" | "audio/x-alaw" | "audio/x-mulaw" | "audio/x-adpcm" => { write_audio_sample_entry(v, cfg, stream)? } @@ -938,6 +938,7 @@ fn write_visual_sample_entry( "image/jpeg" => b"jpeg", "video/x-vp8" => b"vp08", "video/x-vp9" => b"vp09", + "video/x-av1" => b"av01", _ => unreachable!(), }; @@ -1069,6 +1070,84 @@ fn write_visual_sample_entry( Ok(()) })?; } + "video/x-av1" => { + write_box(v, b"av1C", move |v| { + if let Ok(codec_data) = s.get::<&gst::BufferRef>("codec_data") { + let map = codec_data + .map_readable() + .context("codec_data not mappable")?; + + v.extend_from_slice(&map); + } else { + let presentation_delay_minus_one = + if let Ok(presentation_delay) = s.get::("presentation-delay") { + Some( + (1u8 << 5) + | std::cmp::max( + 0xF, + (presentation_delay.saturating_sub(1) & 0xF) as u8, + ), + ) + } else { + None + }; + + let profile = match s.get::<&str>("profile").unwrap() { + "main" => 0, + "high" => 1, + "professional" => 2, + _ => unreachable!(), + }; + + let level = 1; // FIXME + let tier = 0; // FIXME + let (high_bitdepth, twelve_bit) = + match s.get::("bit-depth-luma").unwrap() { + 8 => (false, false), + 10 => (true, false), + 12 => (true, true), + _ => unreachable!(), + }; + let (monochrome, chroma_sub_x, chroma_sub_y) = + match s.get::<&str>("chroma-format").unwrap() { + "4:0:0" => (true, true, true), + "4:2:0" => (false, true, true), + "4:2:2" => (false, true, false), + "4:4:4" => (false, false, false), + _ => unreachable!(), + }; + + let chrome_sample_position = match s.get::<&str>("chroma-site") { + Ok("v-cosited") => 1, + Ok("v-cosited+h-cosited") => 2, + _ => 0, + }; + + let codec_data = [ + 0x80 | 0x01, // marker | version + (profile << 5) | level, // profile | level + (tier << 7) + | ((high_bitdepth as u8) << 6) + | ((twelve_bit as u8) << 5) + | ((monochrome as u8) << 4) + | ((chroma_sub_x as u8) << 3) + | ((chroma_sub_y as u8) << 2) + | chrome_sample_position, // tier | high bitdepth | twelve bit | monochrome | chroma sub x | + // chroma sub y | chroma sample position + if let Some(presentation_delay_minus_one) = presentation_delay_minus_one + { + 0x10 | presentation_delay_minus_one // reserved | presentation delay present | presentation delay + } else { + 0 + }, + ]; + + v.extend_from_slice(&codec_data); + } + + Ok(()) + })?; + } "video/x-vp8" | "image/jpeg" => { // Nothing to do here } diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index 2842a5f1..c3886158 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -2418,6 +2418,9 @@ impl FMP4Mux { } delta_frames = DeltaFrames::PredictiveOnly; } + "video/x-av1" => { + delta_frames = DeltaFrames::PredictiveOnly; + } "image/jpeg" => (), "audio/mpeg" => { if !s.has_field_with_type("codec_data", gst::Buffer::static_type()) { @@ -3279,6 +3282,19 @@ impl ElementImpl for ISOFMP4Mux { .field("width", gst::IntRange::new(1, u16::MAX as i32)) .field("height", gst::IntRange::new(1, u16::MAX as i32)) .build(), + gst::Structure::builder("video/x-av1") + .field("stream-format", "obu-stream") + .field("alignment", "tu") + .field("profile", gst::List::new(["main", "high", "professional"])) + .field( + "chroma-format", + gst::List::new(["4:0:0", "4:2:0", "4:2:2", "4:4:4"]), + ) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), gst::Structure::builder("audio/mpeg") .field("mpegversion", 4i32) .field("stream-format", "raw") @@ -3462,6 +3478,19 @@ impl ElementImpl for DASHMP4Mux { .field("width", gst::IntRange::new(1, u16::MAX as i32)) .field("height", gst::IntRange::new(1, u16::MAX as i32)) .build(), + gst::Structure::builder("video/x-av1") + .field("stream-format", "obu-stream") + .field("alignment", "tu") + .field("profile", gst::List::new(["main", "high", "professional"])) + .field( + "chroma-format", + gst::List::new(["4:0:0", "4:2:0", "4:2:2", "4:4:4"]), + ) + .field("bit-depth-luma", gst::List::new([8u32, 10u32, 12u32])) + .field("bit-depth-chroma", gst::List::new([8u32, 10u32, 12u32])) + .field("width", gst::IntRange::new(1, u16::MAX as i32)) + .field("height", gst::IntRange::new(1, u16::MAX as i32)) + .build(), gst::Structure::builder("audio/mpeg") .field("mpegversion", 4i32) .field("stream-format", "raw")