From 9006a47e9b656acea177dc43a310311fe4d5303e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 7 Aug 2024 11:16:25 +0300 Subject: [PATCH] mp4mux: added image orientation tag support Based on a patch by sergey radionov Part-of: --- mux/mp4/src/mp4mux/boxes.rs | 25 +++++------- mux/mp4/src/mp4mux/imp.rs | 34 +++++++++++++++- mux/mp4/src/mp4mux/mod.rs | 77 +++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 16 deletions(-) diff --git a/mux/mp4/src/mp4mux/boxes.rs b/mux/mp4/src/mp4mux/boxes.rs index 7fd830ba..1d53c248 100644 --- a/mux/mp4/src/mp4mux/boxes.rs +++ b/mux/mp4/src/mp4mux/boxes.rs @@ -12,6 +12,8 @@ use anyhow::{anyhow, bail, Context, Error}; use std::convert::TryFrom; use std::str::FromStr; +use super::{ImageOrientation, IDENTITY_MATRIX}; + fn write_box) -> Result>( vec: &mut Vec, fourcc: impl std::borrow::Borrow<[u8; 4]>, @@ -404,21 +406,14 @@ fn write_tkhd( v.extend([0u8; 2]); // Matrix - v.extend( - [ - (1u32 << 16).to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - (1u32 << 16).to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - (16384u32 << 16).to_be_bytes(), - ] - .into_iter() - .flatten(), - ); + let matrix = match s.name().as_str() { + x if x.starts_with("video/") || x.starts_with("image/") => stream + .orientation + .unwrap_or(ImageOrientation::Rotate0) + .transform_matrix(), + _ => &IDENTITY_MATRIX, + }; + v.extend(matrix.iter().flatten()); // Width/height match s.name().as_str() { diff --git a/mux/mp4/src/mp4mux/imp.rs b/mux/mp4/src/mp4mux/imp.rs index bd19ba7b..8ba5448f 100644 --- a/mux/mp4/src/mp4mux/imp.rs +++ b/mux/mp4/src/mp4mux/imp.rs @@ -18,7 +18,7 @@ use std::sync::Mutex; use crate::mp4mux::obu::read_seq_header_obu_bytes; use once_cell::sync::Lazy; -use super::boxes; +use super::{boxes, ImageOrientation}; /// Offset between NTP and UNIX epoch in seconds. /// NTP = UNIX + NTP_UNIX_OFFSET. @@ -138,6 +138,9 @@ struct Stream { running_time_utc_time_mapping: Option<(gst::Signed, gst::ClockTime)>, extra_header_data: Option>, + + /// Orientation from tags + orientation: Option, } #[derive(Default)] @@ -1018,6 +1021,7 @@ impl MP4Mux { end_pts: None, running_time_utc_time_mapping: None, extra_header_data: None, + orientation: None, }); } @@ -1250,6 +1254,33 @@ impl AggregatorImpl for MP4Mux { } state.language_code = Some(language_code); } + } else if let Some(tag_value) = ev.tag().get::() { + let orientation = tag_value.get(); + gst::trace!( + CAT, + obj = aggregator_pad, + "Received image orientation from tags: {:?}", + orientation + ); + + let mut state = self.state.lock().unwrap(); + for stream in &mut state.streams { + if &stream.sinkpad == aggregator_pad { + stream.orientation = match orientation { + "rotate-0" => Some(ImageOrientation::Rotate0), + "rotate-90" => Some(ImageOrientation::Rotate90), + "rotate-180" => Some(ImageOrientation::Rotate180), + "rotate-270" => Some(ImageOrientation::Rotate270), + // TODO: + // "flip-rotate-0" => Some(ImageOrientation::FlipRotate0), + // "flip-rotate-90" => Some(ImageOrientation::FlipRotate90), + // "flip-rotate-180" => Some(ImageOrientation::FlipRotate180), + // "flip-rotate-270" => Some(ImageOrientation::FlipRotate270), + _ => None, + }; + break; + } + } } self.parent_sink_event_pre_queue(aggregator_pad, event) @@ -1455,6 +1486,7 @@ impl AggregatorImpl for MP4Mux { end_pts, chunks: stream.chunks, extra_header_data: stream.extra_header_data.clone(), + orientation: stream.orientation, }); } diff --git a/mux/mp4/src/mp4mux/mod.rs b/mux/mp4/src/mp4mux/mod.rs index ff667e67..29f5409e 100644 --- a/mux/mp4/src/mp4mux/mod.rs +++ b/mux/mp4/src/mp4mux/mod.rs @@ -51,6 +51,80 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { Ok(()) } +#[derive(Debug, Copy, Clone)] +pub(crate) enum ImageOrientation { + Rotate0, + Rotate90, + Rotate180, + Rotate270, + // TODO: + // FlipRotate0, + // FlipRotate90, + // FlipRotate180, + // FlipRotate270, +} + +type TransformMatrix = [[u8; 4]; 9]; + +const IDENTITY_MATRIX: TransformMatrix = [ + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_90_MATRIX: TransformMatrix = [ + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_180_MATRIX: TransformMatrix = [ + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_270_MATRIX: TransformMatrix = [ + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +impl ImageOrientation { + pub(crate) fn transform_matrix(&self) -> &'static TransformMatrix { + match self { + ImageOrientation::Rotate0 => &IDENTITY_MATRIX, + ImageOrientation::Rotate90 => &ROTATE_90_MATRIX, + ImageOrientation::Rotate180 => &ROTATE_180_MATRIX, + ImageOrientation::Rotate270 => &ROTATE_270_MATRIX, + } + } +} + #[derive(Debug, Copy, Clone)] pub(crate) enum DeltaFrames { /// Only single completely decodable frames @@ -130,6 +204,9 @@ pub(crate) struct Stream { // More data to be included in the fragmented stream header extra_header_data: Option>, + + /// Orientation from tags + orientation: Option, } #[derive(Debug)]