mp4mux: added image orientation tag support

Based on a patch by sergey radionov <rsatom@gmail.com>

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1694>
This commit is contained in:
Sebastian Dröge 2024-08-07 11:16:25 +03:00
parent cfe9968a77
commit 9006a47e9b
3 changed files with 120 additions and 16 deletions

View file

@ -12,6 +12,8 @@ use anyhow::{anyhow, bail, Context, Error};
use std::convert::TryFrom; use std::convert::TryFrom;
use std::str::FromStr; use std::str::FromStr;
use super::{ImageOrientation, IDENTITY_MATRIX};
fn write_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>( fn write_box<T, F: FnOnce(&mut Vec<u8>) -> Result<T, Error>>(
vec: &mut Vec<u8>, vec: &mut Vec<u8>,
fourcc: impl std::borrow::Borrow<[u8; 4]>, fourcc: impl std::borrow::Borrow<[u8; 4]>,
@ -404,21 +406,14 @@ fn write_tkhd(
v.extend([0u8; 2]); v.extend([0u8; 2]);
// Matrix // Matrix
v.extend( let matrix = match s.name().as_str() {
[ x if x.starts_with("video/") || x.starts_with("image/") => stream
(1u32 << 16).to_be_bytes(), .orientation
0u32.to_be_bytes(), .unwrap_or(ImageOrientation::Rotate0)
0u32.to_be_bytes(), .transform_matrix(),
0u32.to_be_bytes(), _ => &IDENTITY_MATRIX,
(1u32 << 16).to_be_bytes(), };
0u32.to_be_bytes(), v.extend(matrix.iter().flatten());
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(16384u32 << 16).to_be_bytes(),
]
.into_iter()
.flatten(),
);
// Width/height // Width/height
match s.name().as_str() { match s.name().as_str() {

View file

@ -18,7 +18,7 @@ use std::sync::Mutex;
use crate::mp4mux::obu::read_seq_header_obu_bytes; use crate::mp4mux::obu::read_seq_header_obu_bytes;
use once_cell::sync::Lazy; use once_cell::sync::Lazy;
use super::boxes; use super::{boxes, ImageOrientation};
/// Offset between NTP and UNIX epoch in seconds. /// Offset between NTP and UNIX epoch in seconds.
/// NTP = UNIX + NTP_UNIX_OFFSET. /// NTP = UNIX + NTP_UNIX_OFFSET.
@ -138,6 +138,9 @@ struct Stream {
running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>, running_time_utc_time_mapping: Option<(gst::Signed<gst::ClockTime>, gst::ClockTime)>,
extra_header_data: Option<Vec<u8>>, extra_header_data: Option<Vec<u8>>,
/// Orientation from tags
orientation: Option<ImageOrientation>,
} }
#[derive(Default)] #[derive(Default)]
@ -1018,6 +1021,7 @@ impl MP4Mux {
end_pts: None, end_pts: None,
running_time_utc_time_mapping: None, running_time_utc_time_mapping: None,
extra_header_data: None, extra_header_data: None,
orientation: None,
}); });
} }
@ -1250,6 +1254,33 @@ impl AggregatorImpl for MP4Mux {
} }
state.language_code = Some(language_code); state.language_code = Some(language_code);
} }
} else if let Some(tag_value) = ev.tag().get::<gst::tags::ImageOrientation>() {
let orientation = tag_value.get();
gst::trace!(
CAT,
obj = aggregator_pad,
"Received image orientation from tags: {:?}",
orientation
);
let mut state = self.state.lock().unwrap();
for stream in &mut state.streams {
if &stream.sinkpad == aggregator_pad {
stream.orientation = match orientation {
"rotate-0" => Some(ImageOrientation::Rotate0),
"rotate-90" => Some(ImageOrientation::Rotate90),
"rotate-180" => Some(ImageOrientation::Rotate180),
"rotate-270" => Some(ImageOrientation::Rotate270),
// TODO:
// "flip-rotate-0" => Some(ImageOrientation::FlipRotate0),
// "flip-rotate-90" => Some(ImageOrientation::FlipRotate90),
// "flip-rotate-180" => Some(ImageOrientation::FlipRotate180),
// "flip-rotate-270" => Some(ImageOrientation::FlipRotate270),
_ => None,
};
break;
}
}
} }
self.parent_sink_event_pre_queue(aggregator_pad, event) self.parent_sink_event_pre_queue(aggregator_pad, event)
@ -1455,6 +1486,7 @@ impl AggregatorImpl for MP4Mux {
end_pts, end_pts,
chunks: stream.chunks, chunks: stream.chunks,
extra_header_data: stream.extra_header_data.clone(), extra_header_data: stream.extra_header_data.clone(),
orientation: stream.orientation,
}); });
} }

View file

@ -51,6 +51,80 @@ pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
Ok(()) Ok(())
} }
#[derive(Debug, Copy, Clone)]
pub(crate) enum ImageOrientation {
Rotate0,
Rotate90,
Rotate180,
Rotate270,
// TODO:
// FlipRotate0,
// FlipRotate90,
// FlipRotate180,
// FlipRotate270,
}
type TransformMatrix = [[u8; 4]; 9];
const IDENTITY_MATRIX: TransformMatrix = [
(1u32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 30).to_be_bytes(),
];
const ROTATE_90_MATRIX: TransformMatrix = [
0u32.to_be_bytes(),
(1u32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
(-1i32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 30).to_be_bytes(),
];
const ROTATE_180_MATRIX: TransformMatrix = [
(-1i32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(-1i32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 30).to_be_bytes(),
];
const ROTATE_270_MATRIX: TransformMatrix = [
0u32.to_be_bytes(),
(-1i32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 16).to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
0u32.to_be_bytes(),
(1u32 << 30).to_be_bytes(),
];
impl ImageOrientation {
pub(crate) fn transform_matrix(&self) -> &'static TransformMatrix {
match self {
ImageOrientation::Rotate0 => &IDENTITY_MATRIX,
ImageOrientation::Rotate90 => &ROTATE_90_MATRIX,
ImageOrientation::Rotate180 => &ROTATE_180_MATRIX,
ImageOrientation::Rotate270 => &ROTATE_270_MATRIX,
}
}
}
#[derive(Debug, Copy, Clone)] #[derive(Debug, Copy, Clone)]
pub(crate) enum DeltaFrames { pub(crate) enum DeltaFrames {
/// Only single completely decodable frames /// Only single completely decodable frames
@ -130,6 +204,9 @@ pub(crate) struct Stream {
// More data to be included in the fragmented stream header // More data to be included in the fragmented stream header
extra_header_data: Option<Vec<u8>>, extra_header_data: Option<Vec<u8>>,
/// Orientation from tags
orientation: Option<ImageOrientation>,
} }
#[derive(Debug)] #[derive(Debug)]