From 8522c8a445080392f0663ee5d121eb143d6fdde7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 28 May 2024 16:26:33 +0300 Subject: [PATCH] gtk4: Add support for rotations / flipping Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/284 Part-of: --- docs/plugins/gst_plugins_cache.json | 62 ++++++++++ video/gtk4/src/lib.rs | 2 + video/gtk4/src/sink/frame.rs | 89 ++++++++++++-- video/gtk4/src/sink/imp.rs | 67 +++++++++-- video/gtk4/src/sink/mod.rs | 2 +- video/gtk4/src/sink/paintable/imp.rs | 168 +++++++++++++++++++++++++-- 6 files changed, 359 insertions(+), 31 deletions(-) diff --git a/docs/plugins/gst_plugins_cache.json b/docs/plugins/gst_plugins_cache.json index 6ee2c592..95eeb2ec 100644 --- a/docs/plugins/gst_plugins_cache.json +++ b/docs/plugins/gst_plugins_cache.json @@ -2622,6 +2622,18 @@ "type": "GdkGLContext", "writable": true }, + "orientation": { + "blurb": "Orientation of the video frames", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "auto (0)", + "mutable": "null", + "readable": true, + "type": "GstGtk4PaintableSinkOrientation", + "writable": true + }, "scaling-filter": { "blurb": "Scaling filter to use for rendering", "conditionally-available": false, @@ -2647,6 +2659,56 @@ "writable": true } } + }, + "GstGtk4PaintableSinkOrientation": { + "kind": "enum", + "values": [ + { + "desc": "Auto", + "name": "auto", + "value": "0" + }, + { + "desc": "Rotate0", + "name": "rotate0", + "value": "1" + }, + { + "desc": "Rotate90", + "name": "rotate90", + "value": "2" + }, + { + "desc": "Rotate180", + "name": "rotate180", + "value": "3" + }, + { + "desc": "Rotate270", + "name": "rotate270", + "value": "4" + }, + { + "desc": "FlipRotate0", + "name": "flip-rotate0", + "value": "5" + }, + { + "desc": "FlipRotate90", + "name": "flip-rotate90", + "value": "6" + }, + { + "desc": "FlipRotate180", + "name": "flip-rotate180", + "value": "7" + }, + { + "desc": "FlipRotate270", + "name": "flip-rotate270", + "value": "8" + } + ] } }, "package": "gst-plugin-gtk4", diff --git a/video/gtk4/src/lib.rs b/video/gtk4/src/lib.rs index e0eecf1f..12c8dff4 100644 --- a/video/gtk4/src/lib.rs +++ b/video/gtk4/src/lib.rs @@ -19,6 +19,7 @@ use gst::glib; mod sink; mod utils; +pub use sink::frame::Orientation; pub use sink::paintable::Paintable; pub use sink::PaintableSink; @@ -28,6 +29,7 @@ fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> { use gst::prelude::*; sink::paintable::Paintable::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); + sink::frame::Orientation::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty()); } #[cfg(not(feature = "gtk_v4_10"))] diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs index 09e6ea19..2c4f3332 100644 --- a/video/gtk4/src/sink/frame.rs +++ b/video/gtk4/src/sink/frame.rs @@ -72,11 +72,15 @@ pub enum TextureCacheId { #[derive(Debug)] enum MappedFrame { - SysMem(gst_video::VideoFrame), + SysMem { + frame: gst_video::VideoFrame, + orientation: Orientation, + }, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] GL { frame: gst_gl::GLVideoFrame, wrapped_context: gst_gl::GLContext, + orientation: Orientation, }, #[cfg(all(target_os = "linux", feature = "dmabuf"))] DmaBuf { @@ -88,13 +92,14 @@ enum MappedFrame { strides: [usize; 4], width: u32, height: u32, + orientation: Orientation, }, } impl MappedFrame { fn buffer(&self) -> &gst::BufferRef { match self { - MappedFrame::SysMem(frame) => frame.buffer(), + MappedFrame::SysMem { frame, .. } => frame.buffer(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.buffer(), #[cfg(all(target_os = "linux", feature = "dmabuf"))] @@ -104,7 +109,7 @@ impl MappedFrame { fn width(&self) -> u32 { match self { - MappedFrame::SysMem(frame) => frame.width(), + MappedFrame::SysMem { frame, .. } => frame.width(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.width(), #[cfg(all(target_os = "linux", feature = "dmabuf"))] @@ -114,7 +119,7 @@ impl MappedFrame { fn height(&self) -> u32 { match self { - MappedFrame::SysMem(frame) => frame.height(), + MappedFrame::SysMem { frame, .. } => frame.height(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.height(), #[cfg(all(target_os = "linux", feature = "dmabuf"))] @@ -124,13 +129,23 @@ impl MappedFrame { fn format_info(&self) -> gst_video::VideoFormatInfo { match self { - MappedFrame::SysMem(frame) => frame.format_info(), + MappedFrame::SysMem { frame, .. } => frame.format_info(), #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, .. } => frame.format_info(), #[cfg(all(target_os = "linux", feature = "dmabuf"))] MappedFrame::DmaBuf { info, .. } => info.format_info(), } } + + fn orientation(&self) -> Orientation { + match self { + MappedFrame::SysMem { orientation, .. } => *orientation, + #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] + MappedFrame::GL { orientation, .. } => *orientation, + #[cfg(all(target_os = "linux", feature = "dmabuf"))] + MappedFrame::DmaBuf { orientation, .. } => *orientation, + } + } } #[derive(Debug)] @@ -139,6 +154,52 @@ pub(crate) struct Frame { overlays: Vec, } +#[derive(Debug, Default, glib::Enum, PartialEq, Eq, Copy, Clone)] +#[repr(C)] +#[enum_type(name = "GstGtk4PaintableSinkOrientation")] +pub enum Orientation { + #[default] + Auto, + Rotate0, + Rotate90, + Rotate180, + Rotate270, + FlipRotate0, + FlipRotate90, + FlipRotate180, + FlipRotate270, +} + +impl Orientation { + pub fn from_tags(tags: &gst::TagListRef) -> Option { + let orientation = tags + .generic("image-orientation") + .and_then(|v| v.get::().ok())?; + + Some(match orientation.as_str() { + "rotate-0" => Orientation::Rotate0, + "rotate-90" => Orientation::Rotate90, + "rotate-180" => Orientation::Rotate180, + "rotate-270" => Orientation::Rotate270, + "flip-rotate-0" => Orientation::FlipRotate0, + "flip-rotate-90" => Orientation::FlipRotate90, + "flip-rotate-180" => Orientation::FlipRotate180, + "flip-rotate-270" => Orientation::FlipRotate270, + _ => return None, + }) + } + + pub fn is_flip_width_height(self) -> bool { + matches!( + self, + Orientation::Rotate90 + | Orientation::Rotate270 + | Orientation::FlipRotate90 + | Orientation::FlipRotate270 + ) + } +} + #[derive(Debug)] struct Overlay { frame: gst_video::VideoFrame, @@ -158,6 +219,7 @@ pub(crate) struct Texture { pub height: f32, pub global_alpha: f32, pub has_alpha: bool, + pub orientation: Orientation, } struct FrameWrapper(gst_video::VideoFrame); @@ -374,14 +436,16 @@ impl Frame { let width = self.frame.width(); let height = self.frame.height(); let has_alpha = self.frame.format_info().has_alpha(); + let orientation = self.frame.orientation(); let (texture, pixel_aspect_ratio) = match self.frame { - MappedFrame::SysMem(frame) => { + MappedFrame::SysMem { frame, .. } => { video_frame_to_memory_texture(frame, cached_textures, &mut used_textures) } #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] MappedFrame::GL { frame, wrapped_context, + .. } => { let Some(gdk_context) = gdk_context else { // This will fail badly if the video frame was actually mapped as GL texture @@ -407,6 +471,7 @@ impl Frame { strides, width, height, + .. } => video_frame_to_dmabuf_texture( buffer, cached_textures, @@ -429,6 +494,7 @@ impl Frame { height: height as f32, global_alpha: 1.0, has_alpha, + orientation, }); for overlay in self.overlays { @@ -444,6 +510,7 @@ impl Frame { height: overlay.height as f32, global_alpha: overlay.global_alpha, has_alpha, + orientation: Orientation::Rotate0, }); } @@ -458,6 +525,7 @@ impl Frame { pub(crate) fn new( buffer: &gst::Buffer, info: &VideoInfo, + orientation: Orientation, #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option< &gst_gl::GLContext, >, @@ -521,6 +589,7 @@ impl Frame { strides, width: vmeta.width(), height: vmeta.height(), + orientation, }); } } @@ -571,6 +640,7 @@ impl Frame { frame = Some(MappedFrame::GL { frame: mapped_frame, wrapped_context: wrapped_context.unwrap().clone(), + orientation, }); } } @@ -580,10 +650,11 @@ impl Frame { let mut frame = Self { frame: match frame { Some(frame) => frame, - None => MappedFrame::SysMem( - gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) + None => MappedFrame::SysMem { + frame: gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info) .map_err(|_| gst::FlowError::Error)?, - ), + orientation, + }, }, overlays: vec![], }; diff --git a/video/gtk4/src/sink/imp.rs b/video/gtk4/src/sink/imp.rs index 54e47bc4..53848a09 100644 --- a/video/gtk4/src/sink/imp.rs +++ b/video/gtk4/src/sink/imp.rs @@ -9,7 +9,7 @@ // // SPDX-License-Identifier: MPL-2.0 -use super::SinkEvent; +use super::{frame, SinkEvent}; use crate::sink::frame::Frame; use crate::sink::paintable::Paintable; @@ -58,11 +58,29 @@ pub(crate) static CAT: Lazy = Lazy::new(|| { ) }); +struct StreamConfig { + info: Option, + /// Orientation from a global scope tag + global_orientation: frame::Orientation, + /// Orientation from a stream scope tag + stream_orientation: Option, +} + +impl Default for StreamConfig { + fn default() -> Self { + StreamConfig { + info: None, + global_orientation: frame::Orientation::Rotate0, + stream_orientation: None, + } + } +} + #[derive(Default)] pub struct PaintableSink { paintable: Mutex>>, window: Mutex>>, - info: Mutex>, + config: Mutex, sender: Mutex>>, pending_frame: Mutex>, cached_caps: Mutex>, @@ -376,7 +394,7 @@ impl ElementImpl for PaintableSink { match transition { gst::StateChange::PausedToReady => { - let _ = self.info.lock().unwrap().take(); + *self.config.lock().unwrap() = StreamConfig::default(); let _ = self.pending_frame.lock().unwrap().take(); // Flush frames from the GDK paintable but don't wait @@ -455,7 +473,7 @@ impl BaseSinkImpl for PaintableSink { .into(), }; - self.info.lock().unwrap().replace(video_info); + self.config.lock().unwrap().info = Some(video_info); Ok(()) } @@ -526,6 +544,31 @@ impl BaseSinkImpl for PaintableSink { _ => BaseSinkImplExt::parent_query(self, query), } } + + fn event(&self, event: gst::Event) -> bool { + match event.view() { + gst::EventView::StreamStart(_) => { + let mut config = self.config.lock().unwrap(); + config.global_orientation = frame::Orientation::Rotate0; + config.stream_orientation = None; + } + gst::EventView::Tag(ev) => { + let mut config = self.config.lock().unwrap(); + let tags = ev.tag(); + let scope = tags.scope(); + let orientation = frame::Orientation::from_tags(tags); + + if scope == gst::TagScope::Global { + config.global_orientation = orientation.unwrap_or(frame::Orientation::Rotate0); + } else { + config.stream_orientation = orientation; + } + } + _ => (), + } + + self.parent_event(event) + } } impl VideoSinkImpl for PaintableSink { @@ -542,11 +585,14 @@ impl VideoSinkImpl for PaintableSink { return Ok(gst::FlowSuccess::Ok); }; - let info = self.info.lock().unwrap(); - let info = info.as_ref().ok_or_else(|| { + let config = self.config.lock().unwrap(); + let info = config.info.as_ref().ok_or_else(|| { gst::error!(CAT, imp: self, "Received no caps yet"); gst::FlowError::NotNegotiated })?; + let orientation = config + .stream_orientation + .unwrap_or(config.global_orientation); let wrapped_context = { #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))] @@ -566,10 +612,11 @@ impl VideoSinkImpl for PaintableSink { } } }; - let frame = Frame::new(buffer, info, wrapped_context.as_ref()).map_err(|err| { - gst::error!(CAT, imp: self, "Failed to map video frame"); - err - })?; + let frame = + Frame::new(buffer, info, orientation, wrapped_context.as_ref()).map_err(|err| { + gst::error!(CAT, imp: self, "Failed to map video frame"); + err + })?; self.pending_frame.lock().unwrap().replace(frame); let sender = self.sender.lock().unwrap(); diff --git a/video/gtk4/src/sink/mod.rs b/video/gtk4/src/sink/mod.rs index f64e985d..42ff12ae 100644 --- a/video/gtk4/src/sink/mod.rs +++ b/video/gtk4/src/sink/mod.rs @@ -38,7 +38,7 @@ use gtk::glib; use gtk::glib::prelude::*; -mod frame; +pub(super) mod frame; pub(super) mod imp; pub(super) mod paintable; diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs index a2b996d9..5a4b4391 100644 --- a/video/gtk4/src/sink/paintable/imp.rs +++ b/video/gtk4/src/sink/paintable/imp.rs @@ -13,7 +13,7 @@ use gtk::prelude::*; use gtk::subclass::prelude::*; use gtk::{gdk, glib, graphene, gsk}; -use crate::sink::frame::{Frame, Texture}; +use crate::sink::frame::{self, Frame, Texture}; use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -38,6 +38,7 @@ pub struct Paintable { scaling_filter: Cell, use_scaling_filter: Cell, force_aspect_ratio: Cell, + orientation: Cell, #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader, } @@ -53,6 +54,7 @@ impl Default for Paintable { scaling_filter: Cell::new(gsk::ScalingFilter::Linear), use_scaling_filter: Cell::new(false), force_aspect_ratio: Cell::new(false), + orientation: Cell::new(frame::Orientation::Auto), #[cfg(not(feature = "gtk_v4_10"))] premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!( "premult.glsl" @@ -101,6 +103,10 @@ impl ObjectImpl for Paintable { .blurb("When enabled, scaling will respect original aspect ratio") .default_value(false) .build(), + glib::ParamSpecEnum::builder::("orientation") + .nick("Orientation") + .blurb("Orientation of the video frames") + .build(), ] }); @@ -125,6 +131,7 @@ impl ObjectImpl for Paintable { #[cfg(feature = "gtk_v4_10")] "use-scaling-filter" => self.use_scaling_filter.get().to_value(), "force-aspect-ratio" => self.force_aspect_ratio.get().to_value(), + "orientation" => self.orientation.get().to_value(), _ => unimplemented!(), } } @@ -148,6 +155,7 @@ impl ObjectImpl for Paintable { #[cfg(feature = "gtk_v4_10")] "use-scaling-filter" => self.use_scaling_filter.set(value.get().unwrap()), "force-aspect-ratio" => self.force_aspect_ratio.set(value.get().unwrap()), + "orientation" => self.orientation.set(value.get().unwrap()), _ => unimplemented!(), } } @@ -156,7 +164,14 @@ impl ObjectImpl for Paintable { impl PaintableImpl for Paintable { fn intrinsic_height(&self) -> i32 { if let Some(paintable) = self.paintables.borrow().first() { - f32::round(paintable.height) as i32 + if self + .effective_orientation(paintable.orientation) + .is_flip_width_height() + { + f32::round(paintable.width) as i32 + } else { + f32::round(paintable.height) as i32 + } } else { 0 } @@ -164,7 +179,14 @@ impl PaintableImpl for Paintable { fn intrinsic_width(&self) -> i32 { if let Some(paintable) = self.paintables.borrow().first() { - f32::round(paintable.width) as i32 + if self + .effective_orientation(paintable.orientation) + .is_flip_width_height() + { + f32::round(paintable.height) as i32 + } else { + f32::round(paintable.width) as i32 + } } else { 0 } @@ -172,7 +194,14 @@ impl PaintableImpl for Paintable { fn intrinsic_aspect_ratio(&self) -> f64 { if let Some(paintable) = self.paintables.borrow().first() { - paintable.width as f64 / paintable.height as f64 + if self + .effective_orientation(paintable.orientation) + .is_flip_width_height() + { + paintable.height as f64 / paintable.width as f64 + } else { + paintable.width as f64 / paintable.height as f64 + } } else { 0.0 } @@ -180,7 +209,6 @@ impl PaintableImpl for Paintable { fn snapshot(&self, snapshot: &gdk::Snapshot, width: f64, height: f64) { let snapshot = snapshot.downcast_ref::().unwrap(); - let background_color = self.background_color.get(); let force_aspect_ratio = self.force_aspect_ratio.get(); let paintables = self.paintables.borrow(); @@ -201,8 +229,72 @@ impl PaintableImpl for Paintable { // // Based on its size relative to the snapshot width/height, all other paintables are // scaled accordingly. - let (frame_width, frame_height) = (first_paintable.width, first_paintable.height); + // + // We also only consider the orientation of the first paintable for now and rotate all + // overlays consistently with that to follow the behaviour of glvideoflip. + let effective_orientation = self.effective_orientation(first_paintable.orientation); + // First do the rotation around the center of the whole snapshot area + if effective_orientation != frame::Orientation::Rotate0 { + snapshot.translate(&graphene::Point::new( + width as f32 / 2.0, + height as f32 / 2.0, + )); + } + match effective_orientation { + frame::Orientation::Rotate0 => {} + frame::Orientation::Rotate90 => { + snapshot.rotate(90.0); + } + frame::Orientation::Rotate180 => { + snapshot.rotate(180.0); + } + frame::Orientation::Rotate270 => { + snapshot.rotate(270.0); + } + frame::Orientation::FlipRotate0 => { + snapshot.rotate_3d(180.0, >k::graphene::Vec3::y_axis()); + } + frame::Orientation::FlipRotate90 => { + snapshot.rotate(90.0); + snapshot.rotate_3d(180.0, >k::graphene::Vec3::y_axis()); + } + frame::Orientation::FlipRotate180 => { + snapshot.rotate(180.0); + snapshot.rotate_3d(180.0, >k::graphene::Vec3::y_axis()); + } + frame::Orientation::FlipRotate270 => { + snapshot.rotate(270.0); + snapshot.rotate_3d(180.0, >k::graphene::Vec3::y_axis()); + } + frame::Orientation::Auto => unreachable!(), + } + if effective_orientation != frame::Orientation::Rotate0 { + if effective_orientation.is_flip_width_height() { + snapshot.translate(&graphene::Point::new( + -height as f32 / 2.0, + -width as f32 / 2.0, + )); + } else { + snapshot.translate(&graphene::Point::new( + -width as f32 / 2.0, + -height as f32 / 2.0, + )); + } + } + + // The rotation is applied now and we're back at the origin at this point + + // Width / height of the overall frame that we're drawing. This has to be flipped + // if a 90/270 degree rotation is applied. + let (frame_width, frame_height) = if effective_orientation.is_flip_width_height() { + (first_paintable.height, first_paintable.width) + } else { + (first_paintable.width, first_paintable.height) + }; + + // Amount of scaling that has to be applied to the main frame and all overlays to fill the + // available area let mut scale_x = width / frame_width as f64; let mut scale_y = height / frame_height as f64; @@ -227,14 +319,26 @@ impl PaintableImpl for Paintable { } if !background_color.is_clear() && (trans_x > f64::EPSILON || trans_y > f64::EPSILON) { + // Clamping for the bounds below has to be flipped over for 90/270 degree rotations. + let (width, height) = if effective_orientation.is_flip_width_height() { + (height, width) + } else { + (width, height) + }; + snapshot.append_color( &background_color, &graphene::Rect::new(0f32, 0f32, width as f32, height as f32), ); } + if effective_orientation.is_flip_width_height() { + std::mem::swap(&mut trans_x, &mut trans_y); + } snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32)); } + // At this point we're at the origin of the area into which the actual video frame is drawn + // Make immutable let scale_x = scale_x; let scale_y = scale_y; @@ -249,11 +353,19 @@ impl PaintableImpl for Paintable { height: paintable_height, global_alpha, has_alpha, + orientation: _orientation, }, ) in paintables.iter().enumerate() { snapshot.push_opacity(*global_alpha as f64); + // Clamping for the bounds below has to be flipped over for 90/270 degree rotations. + let (width, height) = if effective_orientation.is_flip_width_height() { + (height, width) + } else { + (width, height) + }; + let bounds = if !force_aspect_ratio && idx == 0 { // While this should end up with width again, be explicit in this case to avoid // rounding errors and fill the whole area with the video frame. @@ -261,11 +373,32 @@ impl PaintableImpl for Paintable { } else { // Scale texture position and size with the same scale factor as the main video // frame, and make sure to not render outside (0, 0, width, height). - let x = f32::clamp(*x * scale_x as f32, 0.0, width as f32); - let y = f32::clamp(*y * scale_y as f32, 0.0, height as f32); - let texture_width = f32::min(*paintable_width * scale_x as f32, width as f32); - let texture_height = f32::min(*paintable_height * scale_y as f32, height as f32); - graphene::Rect::new(x, y, texture_width, texture_height) + let (rect_x, rect_y) = if effective_orientation.is_flip_width_height() { + ( + f32::clamp(*y * scale_y as f32, 0.0, width as f32), + f32::clamp(*x * scale_x as f32, 0.0, height as f32), + ) + } else { + ( + f32::clamp(*x * scale_x as f32, 0.0, width as f32), + f32::clamp(*y * scale_y as f32, 0.0, height as f32), + ) + }; + + let (texture_width, texture_height) = + if effective_orientation.is_flip_width_height() { + ( + f32::min(*paintable_width * scale_y as f32, width as f32), + f32::min(*paintable_height * scale_x as f32, height as f32), + ) + } else { + ( + f32::min(*paintable_width * scale_x as f32, width as f32), + f32::min(*paintable_height * scale_y as f32, height as f32), + ) + }; + + graphene::Rect::new(rect_x, rect_y, texture_width, texture_height) }; // Only premultiply GL textures that expect to be in premultiplied RGBA format. @@ -364,6 +497,19 @@ impl PaintableImpl for Paintable { } impl Paintable { + fn effective_orientation( + &self, + paintable_orientation: frame::Orientation, + ) -> frame::Orientation { + let orientation = self.orientation.get(); + if orientation != frame::Orientation::Auto { + return orientation; + } + + assert_ne!(paintable_orientation, frame::Orientation::Auto); + paintable_orientation + } + pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) { let context = self.gl_context.borrow();