From fdfa3a33d90563645c49ccccf2d106d520616218 Mon Sep 17 00:00:00 2001 From: sergey radionov Date: Fri, 12 Jul 2024 19:55:46 +0700 Subject: [PATCH] fmp4mux: added image orientation tag support fix #565 Part-of: --- mux/fmp4/src/fmp4mux/boxes.rs | 26 +++++------- mux/fmp4/src/fmp4mux/imp.rs | 26 ++++++++++++ mux/fmp4/src/fmp4mux/mod.rs | 75 +++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 16 deletions(-) diff --git a/mux/fmp4/src/fmp4mux/boxes.rs b/mux/fmp4/src/fmp4mux/boxes.rs index a9a50ff1..27baecf9 100644 --- a/mux/fmp4/src/fmp4mux/boxes.rs +++ b/mux/fmp4/src/fmp4mux/boxes.rs @@ -11,7 +11,7 @@ use gst::prelude::*; use anyhow::{anyhow, bail, Context, Error}; use std::convert::TryFrom; -use super::Buffer; +use super::{Buffer, ImageOrientation, IDENTITY_MATRIX}; fn write_box) -> Result>( vec: &mut Vec, @@ -618,21 +618,15 @@ fn write_tkhd( v.extend([0u8; 2]); // Matrix - v.extend( - [ - (1u32 << 16).to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - (1u32 << 16).to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - 0u32.to_be_bytes(), - (16384u32 << 16).to_be_bytes(), - ] - .into_iter() - .flatten(), - ); + let matrix = match s.name().as_str() { + x if x.starts_with("video/") || x.starts_with("image/") => _cfg + .orientation + .unwrap_or(ImageOrientation::Rotate0) + .transform_matrix(), + _ => &IDENTITY_MATRIX, + }; + + v.extend(matrix.iter().flatten()); // Width/height match s.name().as_str() { diff --git a/mux/fmp4/src/fmp4mux/imp.rs b/mux/fmp4/src/fmp4mux/imp.rs index d1062dcf..e9fb8cfd 100644 --- a/mux/fmp4/src/fmp4mux/imp.rs +++ b/mux/fmp4/src/fmp4mux/imp.rs @@ -17,6 +17,7 @@ use std::mem; use std::sync::Mutex; use crate::fmp4mux::obu::read_seq_header_obu_bytes; +use crate::fmp4mux::ImageOrientation; use once_cell::sync::Lazy; use super::boxes; @@ -255,6 +256,8 @@ struct State { start_dts: Option, /// Language code from tags language_code: Option<[u8; 3]>, + /// Orientation from tags + orientation: Option, /// Start PTS of the current fragment fragment_start_pts: Option, @@ -2788,6 +2791,7 @@ impl FMP4Mux { write_mehd: settings.write_mehd, duration: if at_eos { duration } else { None }, language_code: state.language_code, + orientation: state.orientation, start_utc_time: if variant == super::Variant::ONVIF { state .earliest_pts @@ -3241,6 +3245,28 @@ impl AggregatorImpl for FMP4Mux { } state.language_code = Some(language_code); } + } else if let Some(tag_value) = ev.tag().get::() { + let orientation = tag_value.get(); + gst::trace!( + CAT, + imp = self, + "Received image orientation from tags: {:?}", + orientation + ); + + let mut state = self.state.lock().unwrap(); + state.orientation = match orientation { + "rotate-0" => Some(ImageOrientation::Rotate0), + "rotate-90" => Some(ImageOrientation::Rotate90), + "rotate-180" => Some(ImageOrientation::Rotate180), + "rotate-270" => Some(ImageOrientation::Rotate270), + // TODO: + // "flip-rotate-0" => Some(ImageOrientation::FlipRotate0), + // "flip-rotate-90" => Some(ImageOrientation::FlipRotate90), + // "flip-rotate-180" => Some(ImageOrientation::FlipRotate180), + // "flip-rotate-270" => Some(ImageOrientation::FlipRotate270), + _ => None, + } } self.parent_sink_event(aggregator_pad, event) diff --git a/mux/fmp4/src/fmp4mux/mod.rs b/mux/fmp4/src/fmp4mux/mod.rs index 99304cd0..407debe7 100644 --- a/mux/fmp4/src/fmp4mux/mod.rs +++ b/mux/fmp4/src/fmp4mux/mod.rs @@ -73,6 +73,80 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { Ok(()) } +#[derive(Debug, Copy, Clone)] +pub(crate) enum ImageOrientation { + Rotate0, + Rotate90, + Rotate180, + Rotate270, + // TODO: + // FlipRotate0, + // FlipRotate90, + // FlipRotate180, + // FlipRotate270, +} + +type TransformMatrix = [[u8; 4]; 9]; + +const IDENTITY_MATRIX: TransformMatrix = [ + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_90_MATRIX: TransformMatrix = [ + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_180_MATRIX: TransformMatrix = [ + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +const ROTATE_270_MATRIX: TransformMatrix = [ + 0u32.to_be_bytes(), + (-1i32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 16).to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + 0u32.to_be_bytes(), + (1u32 << 30).to_be_bytes(), +]; + +impl ImageOrientation { + pub(crate) fn transform_matrix(&self) -> &'static TransformMatrix { + match self { + ImageOrientation::Rotate0 => &IDENTITY_MATRIX, + ImageOrientation::Rotate90 => &ROTATE_90_MATRIX, + ImageOrientation::Rotate180 => &ROTATE_180_MATRIX, + ImageOrientation::Rotate270 => &ROTATE_270_MATRIX, + } + } +} + #[derive(Debug)] pub(crate) struct HeaderConfiguration { variant: Variant, @@ -88,6 +162,7 @@ pub(crate) struct HeaderConfiguration { write_mehd: bool, duration: Option, language_code: Option<[u8; 3]>, + orientation: Option, /// Start UTC time in ONVIF mode. /// Since Jan 1 1601 in 100ns units.