mp4mux: added image orientation tag support

Based on a patch by sergey radionov <rsatom@gmail.com>

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1694>

mux/mp4/src/mp4mux/boxes.rs

@@ -12,6 +12,8 @@ use anyhow::{anyhow, bail, Context, Error};
 use std::convert::TryFrom;
 use std::str::FromStr;
 
+use super::{ImageOrientation, IDENTITY_MATRIX};
+
 fn write_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>(
     vec: &mut Vec<u8>,
     fourcc: impl std::borrow::Borrow<[u8; 4]>,
@@ -404,21 +406,14 @@ fn write_tkhd(
     v.extend([0u8; 2]);
 
     // Matrix
-    v.extend(
+    let matrix = match s.name().as_str() {
-        [
+        x if x.starts_with("video/") || x.starts_with("image/") => stream
-            (1u32 << 16).to_be_bytes(),
+            .orientation
-            0u32.to_be_bytes(),
+            .unwrap_or(ImageOrientation::Rotate0)
-            0u32.to_be_bytes(),
+            .transform_matrix(),
-            0u32.to_be_bytes(),
+        _ => &IDENTITY_MATRIX,
-            (1u32 << 16).to_be_bytes(),
+    };
-            0u32.to_be_bytes(),
+    v.extend(matrix.iter().flatten());
-            0u32.to_be_bytes(),
-            0u32.to_be_bytes(),
-            (16384u32 << 16).to_be_bytes(),
-        ]
-        .into_iter()
-        .flatten(),
-    );
 
     // Width/height
     match s.name().as_str() {

mux/mp4/src/mp4mux/imp.rs

@@ -18,7 +18,7 @@ use std::sync::Mutex;
 use crate::mp4mux::obu::read_seq_header_obu_bytes;
 use once_cell::sync::Lazy;
 
-use super::boxes;
+use super::{boxes, ImageOrientation};
 
 /// Offset between NTP and UNIX epoch in seconds.
 /// NTP = UNIX + NTP_UNIX_OFFSET.
@@ -138,6 +138,9 @@ struct Stream {
     running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>,
 
     extra_header_data: Option<Vec<u8>>,
+
+    /// Orientation from tags
+    orientation: Option<ImageOrientation>,
 }
 
 #[derive(Default)]
@@ -1018,6 +1021,7 @@ impl MP4Mux {
                 end_pts: None,
                 running_time_utc_time_mapping: None,
                 extra_header_data: None,
+                orientation: None,
             });
         }
 
@@ -1250,6 +1254,33 @@ impl AggregatorImpl for MP4Mux {
                         }
                         state.language_code = Some(language_code);
                     }
+                } else if let Some(tag_value) = ev.tag().get::<gst::tags::ImageOrientation>() {
+                    let orientation = tag_value.get();
+                    gst::trace!(
+                        CAT,
+                        obj = aggregator_pad,
+                        "Received image orientation from tags: {:?}",
+                        orientation
+                    );
+
+                    let mut state = self.state.lock().unwrap();
+                    for stream in &mut state.streams {
+                        if &stream.sinkpad == aggregator_pad {
+                            stream.orientation = match orientation {
+                                "rotate-0" => Some(ImageOrientation::Rotate0),
+                                "rotate-90" => Some(ImageOrientation::Rotate90),
+                                "rotate-180" => Some(ImageOrientation::Rotate180),
+                                "rotate-270" => Some(ImageOrientation::Rotate270),
+                                // TODO:
+                                // "flip-rotate-0" => Some(ImageOrientation::FlipRotate0),
+                                // "flip-rotate-90" => Some(ImageOrientation::FlipRotate90),
+                                // "flip-rotate-180" => Some(ImageOrientation::FlipRotate180),
+                                // "flip-rotate-270" => Some(ImageOrientation::FlipRotate270),
+                                _ => None,
+                            };
+                            break;
+                        }
+                    }
                 }
 
                 self.parent_sink_event_pre_queue(aggregator_pad, event)
@@ -1455,6 +1486,7 @@ impl AggregatorImpl for MP4Mux {
                     end_pts,
                     chunks: stream.chunks,
                     extra_header_data: stream.extra_header_data.clone(),
+                    orientation: stream.orientation,
                 });
             }
 

mux/mp4/src/mp4mux/mod.rs

@@ -51,6 +51,80 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
     Ok(())
 }
 
+#[derive(Debug, Copy, Clone)]
+pub(crate) enum ImageOrientation {
+    Rotate0,
+    Rotate90,
+    Rotate180,
+    Rotate270,
+    // TODO:
+    // FlipRotate0,
+    // FlipRotate90,
+    // FlipRotate180,
+    // FlipRotate270,
+}
+
+type TransformMatrix = [[u8; 4]; 9];
+
+const IDENTITY_MATRIX: TransformMatrix = [
+    (1u32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 30).to_be_bytes(),
+];
+
+const ROTATE_90_MATRIX: TransformMatrix = [
+    0u32.to_be_bytes(),
+    (1u32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    (-1i32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 30).to_be_bytes(),
+];
+
+const ROTATE_180_MATRIX: TransformMatrix = [
+    (-1i32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (-1i32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 30).to_be_bytes(),
+];
+
+const ROTATE_270_MATRIX: TransformMatrix = [
+    0u32.to_be_bytes(),
+    (-1i32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 16).to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    0u32.to_be_bytes(),
+    (1u32 << 30).to_be_bytes(),
+];
+
+impl ImageOrientation {
+    pub(crate) fn transform_matrix(&self) -> &'static TransformMatrix {
+        match self {
+            ImageOrientation::Rotate0 => &IDENTITY_MATRIX,
+            ImageOrientation::Rotate90 => &ROTATE_90_MATRIX,
+            ImageOrientation::Rotate180 => &ROTATE_180_MATRIX,
+            ImageOrientation::Rotate270 => &ROTATE_270_MATRIX,
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone)]
 pub(crate) enum DeltaFrames {
     /// Only single completely decodable frames
@@ -130,6 +204,9 @@ pub(crate) struct Stream {
 
     // More data to be included in the fragmented stream header
     extra_header_data: Option<Vec<u8>>,
+
+    /// Orientation from tags
+    orientation: Option<ImageOrientation>,
 }
 
 #[derive(Debug)]