mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-11-10 20:31:10 +00:00
Code Issues Projects Releases Wiki Activity

gtk4: Add support for rotations / flipping

Browse source 
Fixes https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/issues/284

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1590>
This commit is contained in:
Sebastian Dröge 2024-05-28 16:26:33 +03:00 committed by GStreamer Marge Bot
parent 2fe852166e
commit 8522c8a445
6 changed files with 359 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
docs/plugins/gst_plugins_cache.json
View file

@ -2622,6 +2622,18 @@
"type": "GdkGLContext",
"writable": true
},
"orientation": {
"blurb": "Orientation of the video frames",
"conditionally-available": false,
"construct": false,
"construct-only": false,
"controllable": false,
"default": "auto (0)",
"mutable": "null",
"readable": true,
"type": "GstGtk4PaintableSinkOrientation",
"writable": true
},
"scaling-filter": {
"blurb": "Scaling filter to use for rendering",
"conditionally-available": false,
@ -2647,6 +2659,56 @@
"writable": true
}
}
},
"GstGtk4PaintableSinkOrientation": {
"kind": "enum",
"values": [
{
"desc": "Auto",
"name": "auto",
"value": "0"
},
{
"desc": "Rotate0",
"name": "rotate0",
"value": "1"
},
{
"desc": "Rotate90",
"name": "rotate90",
"value": "2"
},
{
"desc": "Rotate180",
"name": "rotate180",
"value": "3"
},
{
"desc": "Rotate270",
"name": "rotate270",
"value": "4"
},
{
"desc": "FlipRotate0",
"name": "flip-rotate0",
"value": "5"
},
{
"desc": "FlipRotate90",
"name": "flip-rotate90",
"value": "6"
},
{
"desc": "FlipRotate180",
"name": "flip-rotate180",
"value": "7"
},
{
"desc": "FlipRotate270",
"name": "flip-rotate270",
"value": "8"
}
]
}
},
"package": "gst-plugin-gtk4",

2
video/gtk4/src/lib.rs
View file

@ -19,6 +19,7 @@ use gst::glib;
mod sink;
mod utils;
pub use sink::frame::Orientation;
pub use sink::paintable::Paintable;
pub use sink::PaintableSink;
@ -28,6 +29,7 @@ fn plugin_init(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
use gst::prelude::*;
sink::paintable::Paintable::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
sink::frame::Orientation::static_type().mark_as_plugin_api(gst::PluginAPIFlags::empty());
}
#[cfg(not(feature = "gtk_v4_10"))]

89
video/gtk4/src/sink/frame.rs
View file

@ -72,11 +72,15 @@ pub enum TextureCacheId {
#[derive(Debug)]
enum MappedFrame {
SysMem(gst_video::VideoFrame<gst_video::video_frame::Readable>),
SysMem {
frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
orientation: Orientation,
},
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
GL {
frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
wrapped_context: gst_gl::GLContext,
orientation: Orientation,
},
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
DmaBuf {
@ -88,13 +92,14 @@ enum MappedFrame {
strides: [usize; 4],
width: u32,
height: u32,
orientation: Orientation,
},
}
impl MappedFrame {
fn buffer(&self) -> &gst::BufferRef {
match self {
MappedFrame::SysMem(frame) => frame.buffer(),
MappedFrame::SysMem { frame, .. } => frame.buffer(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.buffer(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
@ -104,7 +109,7 @@ impl MappedFrame {
fn width(&self) -> u32 {
match self {
MappedFrame::SysMem(frame) => frame.width(),
MappedFrame::SysMem { frame, .. } => frame.width(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.width(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
@ -114,7 +119,7 @@ impl MappedFrame {
fn height(&self) -> u32 {
match self {
MappedFrame::SysMem(frame) => frame.height(),
MappedFrame::SysMem { frame, .. } => frame.height(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.height(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
@ -124,13 +129,23 @@ impl MappedFrame {
fn format_info(&self) -> gst_video::VideoFormatInfo {
match self {
MappedFrame::SysMem(frame) => frame.format_info(),
MappedFrame::SysMem { frame, .. } => frame.format_info(),
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { frame, .. } => frame.format_info(),
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { info, .. } => info.format_info(),
}
}
fn orientation(&self) -> Orientation {
match self {
MappedFrame::SysMem { orientation, .. } => *orientation,
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL { orientation, .. } => *orientation,
#[cfg(all(target_os = "linux", feature = "dmabuf"))]
MappedFrame::DmaBuf { orientation, .. } => *orientation,
}
}
}
#[derive(Debug)]
@ -139,6 +154,52 @@ pub(crate) struct Frame {
overlays: Vec<Overlay>,
}
#[derive(Debug, Default, glib::Enum, PartialEq, Eq, Copy, Clone)]
#[repr(C)]
#[enum_type(name = "GstGtk4PaintableSinkOrientation")]
pub enum Orientation {
#[default]
Auto,
Rotate0,
Rotate90,
Rotate180,
Rotate270,
FlipRotate0,
FlipRotate90,
FlipRotate180,
FlipRotate270,
}
impl Orientation {
pub fn from_tags(tags: &gst::TagListRef) -> Option<Orientation> {
let orientation = tags
.generic("image-orientation")
.and_then(|v| v.get::<String>().ok())?;
Some(match orientation.as_str() {
"rotate-0" => Orientation::Rotate0,
"rotate-90" => Orientation::Rotate90,
"rotate-180" => Orientation::Rotate180,
"rotate-270" => Orientation::Rotate270,
"flip-rotate-0" => Orientation::FlipRotate0,
"flip-rotate-90" => Orientation::FlipRotate90,
"flip-rotate-180" => Orientation::FlipRotate180,
"flip-rotate-270" => Orientation::FlipRotate270,
_ => return None,
})
}
pub fn is_flip_width_height(self) -> bool {
matches!(
self,
Orientation::Rotate90
| Orientation::Rotate270
| Orientation::FlipRotate90
| Orientation::FlipRotate270
)
}
}
#[derive(Debug)]
struct Overlay {
frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
@ -158,6 +219,7 @@ pub(crate) struct Texture {
pub height: f32,
pub global_alpha: f32,
pub has_alpha: bool,
pub orientation: Orientation,
}
struct FrameWrapper(gst_video::VideoFrame<gst_video::video_frame::Readable>);
@ -374,14 +436,16 @@ impl Frame {
let width = self.frame.width();
let height = self.frame.height();
let has_alpha = self.frame.format_info().has_alpha();
let orientation = self.frame.orientation();
let (texture, pixel_aspect_ratio) = match self.frame {
MappedFrame::SysMem(frame) => {
MappedFrame::SysMem { frame, .. } => {
video_frame_to_memory_texture(frame, cached_textures, &mut used_textures)
}
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
MappedFrame::GL {
frame,
wrapped_context,
..
} => {
let Some(gdk_context) = gdk_context else {
// This will fail badly if the video frame was actually mapped as GL texture
@ -407,6 +471,7 @@ impl Frame {
strides,
width,
height,
..
} => video_frame_to_dmabuf_texture(
buffer,
cached_textures,
@ -429,6 +494,7 @@ impl Frame {
height: height as f32,
global_alpha: 1.0,
has_alpha,
orientation,
});
for overlay in self.overlays {
@ -444,6 +510,7 @@ impl Frame {
height: overlay.height as f32,
global_alpha: overlay.global_alpha,
has_alpha,
orientation: Orientation::Rotate0,
});
}
@ -458,6 +525,7 @@ impl Frame {
pub(crate) fn new(
buffer: &gst::Buffer,
info: &VideoInfo,
orientation: Orientation,
#[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option<
&gst_gl::GLContext,
>,
@ -521,6 +589,7 @@ impl Frame {
strides,
width: vmeta.width(),
height: vmeta.height(),
orientation,
});
}
}
@ -571,6 +640,7 @@ impl Frame {
frame = Some(MappedFrame::GL {
frame: mapped_frame,
wrapped_context: wrapped_context.unwrap().clone(),
orientation,
});
}
}
@ -580,10 +650,11 @@ impl Frame {
let mut frame = Self {
frame: match frame {
Some(frame) => frame,
None => MappedFrame::SysMem(
gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
None => MappedFrame::SysMem {
frame: gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
.map_err(|_| gst::FlowError::Error)?,
),
orientation,
},
},
overlays: vec![],
};

61
video/gtk4/src/sink/imp.rs
View file

@ -9,7 +9,7 @@
//
// SPDX-License-Identifier: MPL-2.0
use super::SinkEvent;
use super::{frame, SinkEvent};
use crate::sink::frame::Frame;
use crate::sink::paintable::Paintable;
@ -58,11 +58,29 @@ pub(crate) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
)
});
struct StreamConfig {
info: Option<super::frame::VideoInfo>,
/// Orientation from a global scope tag
global_orientation: frame::Orientation,
/// Orientation from a stream scope tag
stream_orientation: Option<frame::Orientation>,
}
impl Default for StreamConfig {
fn default() -> Self {
StreamConfig {
info: None,
global_orientation: frame::Orientation::Rotate0,
stream_orientation: None,
}
}
}
#[derive(Default)]
pub struct PaintableSink {
paintable: Mutex<Option<ThreadGuard<Paintable>>>,
window: Mutex<Option<ThreadGuard<gtk::Window>>>,
info: Mutex<Option<super::frame::VideoInfo>>,
config: Mutex<StreamConfig>,
sender: Mutex<Option<async_channel::Sender<SinkEvent>>>,
pending_frame: Mutex<Option<Frame>>,
cached_caps: Mutex<Option<gst::Caps>>,
@ -376,7 +394,7 @@ impl ElementImpl for PaintableSink {
match transition {
gst::StateChange::PausedToReady => {
let _ = self.info.lock().unwrap().take();
*self.config.lock().unwrap() = StreamConfig::default();
let _ = self.pending_frame.lock().unwrap().take();
// Flush frames from the GDK paintable but don't wait
@ -455,7 +473,7 @@ impl BaseSinkImpl for PaintableSink {
.into(),
};
self.info.lock().unwrap().replace(video_info);
self.config.lock().unwrap().info = Some(video_info);
Ok(())
}
@ -526,6 +544,31 @@ impl BaseSinkImpl for PaintableSink {
_ => BaseSinkImplExt::parent_query(self, query),
}
}
fn event(&self, event: gst::Event) -> bool {
match event.view() {
gst::EventView::StreamStart(_) => {
let mut config = self.config.lock().unwrap();
config.global_orientation = frame::Orientation::Rotate0;
config.stream_orientation = None;
}
gst::EventView::Tag(ev) => {
let mut config = self.config.lock().unwrap();
let tags = ev.tag();
let scope = tags.scope();
let orientation = frame::Orientation::from_tags(tags);
if scope == gst::TagScope::Global {
config.global_orientation = orientation.unwrap_or(frame::Orientation::Rotate0);
} else {
config.stream_orientation = orientation;
}
}
_ => (),
}
self.parent_event(event)
}
}
impl VideoSinkImpl for PaintableSink {
@ -542,11 +585,14 @@ impl VideoSinkImpl for PaintableSink {
return Ok(gst::FlowSuccess::Ok);
};
let info = self.info.lock().unwrap();
let info = info.as_ref().ok_or_else(|| {
let config = self.config.lock().unwrap();
let info = config.info.as_ref().ok_or_else(|| {
gst::error!(CAT, imp: self, "Received no caps yet");
gst::FlowError::NotNegotiated
})?;
let orientation = config
.stream_orientation
.unwrap_or(config.global_orientation);
let wrapped_context = {
#[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))]
@ -566,7 +612,8 @@ impl VideoSinkImpl for PaintableSink {
}
}
};
let frame = Frame::new(buffer, info, wrapped_context.as_ref()).map_err(|err| {
let frame =
Frame::new(buffer, info, orientation, wrapped_context.as_ref()).map_err(|err| {
gst::error!(CAT, imp: self, "Failed to map video frame");
err
})?;

2
video/gtk4/src/sink/mod.rs
View file

@ -38,7 +38,7 @@
use gtk::glib;
use gtk::glib::prelude::*;
mod frame;
pub(super) mod frame;
pub(super) mod imp;
pub(super) mod paintable;

162
video/gtk4/src/sink/paintable/imp.rs
View file

@ -13,7 +13,7 @@ use gtk::prelude::*;
use gtk::subclass::prelude::*;
use gtk::{gdk, glib, graphene, gsk};
use crate::sink::frame::{Frame, Texture};
use crate::sink::frame::{self, Frame, Texture};
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
@ -38,6 +38,7 @@ pub struct Paintable {
scaling_filter: Cell<gsk::ScalingFilter>,
use_scaling_filter: Cell<bool>,
force_aspect_ratio: Cell<bool>,
orientation: Cell<frame::Orientation>,
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader,
}
@ -53,6 +54,7 @@ impl Default for Paintable {
scaling_filter: Cell::new(gsk::ScalingFilter::Linear),
use_scaling_filter: Cell::new(false),
force_aspect_ratio: Cell::new(false),
orientation: Cell::new(frame::Orientation::Auto),
#[cfg(not(feature = "gtk_v4_10"))]
premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
"premult.glsl"
@ -101,6 +103,10 @@ impl ObjectImpl for Paintable {
.blurb("When enabled, scaling will respect original aspect ratio")
.default_value(false)
.build(),
glib::ParamSpecEnum::builder::<frame::Orientation>("orientation")
.nick("Orientation")
.blurb("Orientation of the video frames")
.build(),
]
});
@ -125,6 +131,7 @@ impl ObjectImpl for Paintable {
#[cfg(feature = "gtk_v4_10")]
"use-scaling-filter" => self.use_scaling_filter.get().to_value(),
"force-aspect-ratio" => self.force_aspect_ratio.get().to_value(),
"orientation" => self.orientation.get().to_value(),
_ => unimplemented!(),
}
}
@ -148,6 +155,7 @@ impl ObjectImpl for Paintable {
#[cfg(feature = "gtk_v4_10")]
"use-scaling-filter" => self.use_scaling_filter.set(value.get().unwrap()),
"force-aspect-ratio" => self.force_aspect_ratio.set(value.get().unwrap()),
"orientation" => self.orientation.set(value.get().unwrap()),
_ => unimplemented!(),
}
}
@ -156,7 +164,14 @@ impl ObjectImpl for Paintable {
impl PaintableImpl for Paintable {
fn intrinsic_height(&self) -> i32 {
if let Some(paintable) = self.paintables.borrow().first() {
if self
.effective_orientation(paintable.orientation)
.is_flip_width_height()
{
f32::round(paintable.width) as i32
} else {
f32::round(paintable.height) as i32
}
} else {
0
}
@ -164,7 +179,14 @@ impl PaintableImpl for Paintable {
fn intrinsic_width(&self) -> i32 {
if let Some(paintable) = self.paintables.borrow().first() {
if self
.effective_orientation(paintable.orientation)
.is_flip_width_height()
{
f32::round(paintable.height) as i32
} else {
f32::round(paintable.width) as i32
}
} else {
0
}
@ -172,7 +194,14 @@ impl PaintableImpl for Paintable {
fn intrinsic_aspect_ratio(&self) -> f64 {
if let Some(paintable) = self.paintables.borrow().first() {
if self
.effective_orientation(paintable.orientation)
.is_flip_width_height()
{
paintable.height as f64 / paintable.width as f64
} else {
paintable.width as f64 / paintable.height as f64
}
} else {
0.0
}
@ -180,7 +209,6 @@ impl PaintableImpl for Paintable {
fn snapshot(&self, snapshot: &gdk::Snapshot, width: f64, height: f64) {
let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap();
let background_color = self.background_color.get();
let force_aspect_ratio = self.force_aspect_ratio.get();
let paintables = self.paintables.borrow();
@ -201,8 +229,72 @@ impl PaintableImpl for Paintable {
//
// Based on its size relative to the snapshot width/height, all other paintables are
// scaled accordingly.
let (frame_width, frame_height) = (first_paintable.width, first_paintable.height);
//
// We also only consider the orientation of the first paintable for now and rotate all
// overlays consistently with that to follow the behaviour of glvideoflip.
let effective_orientation = self.effective_orientation(first_paintable.orientation);
// First do the rotation around the center of the whole snapshot area
if effective_orientation != frame::Orientation::Rotate0 {
snapshot.translate(&graphene::Point::new(
width as f32 / 2.0,
height as f32 / 2.0,
));
}
match effective_orientation {
frame::Orientation::Rotate0 => {}
frame::Orientation::Rotate90 => {
snapshot.rotate(90.0);
}
frame::Orientation::Rotate180 => {
snapshot.rotate(180.0);
}
frame::Orientation::Rotate270 => {
snapshot.rotate(270.0);
}
frame::Orientation::FlipRotate0 => {
snapshot.rotate_3d(180.0, &gtk::graphene::Vec3::y_axis());
}
frame::Orientation::FlipRotate90 => {
snapshot.rotate(90.0);
snapshot.rotate_3d(180.0, &gtk::graphene::Vec3::y_axis());
}
frame::Orientation::FlipRotate180 => {
snapshot.rotate(180.0);
snapshot.rotate_3d(180.0, &gtk::graphene::Vec3::y_axis());
}
frame::Orientation::FlipRotate270 => {
snapshot.rotate(270.0);
snapshot.rotate_3d(180.0, &gtk::graphene::Vec3::y_axis());
}
frame::Orientation::Auto => unreachable!(),
}
if effective_orientation != frame::Orientation::Rotate0 {
if effective_orientation.is_flip_width_height() {
snapshot.translate(&graphene::Point::new(
-height as f32 / 2.0,
-width as f32 / 2.0,
));
} else {
snapshot.translate(&graphene::Point::new(
-width as f32 / 2.0,
-height as f32 / 2.0,
));
}
}
// The rotation is applied now and we're back at the origin at this point
// Width / height of the overall frame that we're drawing. This has to be flipped
// if a 90/270 degree rotation is applied.
let (frame_width, frame_height) = if effective_orientation.is_flip_width_height() {
(first_paintable.height, first_paintable.width)
} else {
(first_paintable.width, first_paintable.height)
};
// Amount of scaling that has to be applied to the main frame and all overlays to fill the
// available area
let mut scale_x = width / frame_width as f64;
let mut scale_y = height / frame_height as f64;
@ -227,14 +319,26 @@ impl PaintableImpl for Paintable {
}
if !background_color.is_clear() && (trans_x > f64::EPSILON || trans_y > f64::EPSILON) {
// Clamping for the bounds below has to be flipped over for 90/270 degree rotations.
let (width, height) = if effective_orientation.is_flip_width_height() {
(height, width)
} else {
(width, height)
};
snapshot.append_color(
&background_color,
&graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
);
}
if effective_orientation.is_flip_width_height() {
std::mem::swap(&mut trans_x, &mut trans_y);
}
snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
}
// At this point we're at the origin of the area into which the actual video frame is drawn
// Make immutable
let scale_x = scale_x;
let scale_y = scale_y;
@ -249,11 +353,19 @@ impl PaintableImpl for Paintable {
height: paintable_height,
global_alpha,
has_alpha,
orientation: _orientation,
},
) in paintables.iter().enumerate()
{
snapshot.push_opacity(*global_alpha as f64);
// Clamping for the bounds below has to be flipped over for 90/270 degree rotations.
let (width, height) = if effective_orientation.is_flip_width_height() {
(height, width)
} else {
(width, height)
};
let bounds = if !force_aspect_ratio && idx == 0 {
// While this should end up with width again, be explicit in this case to avoid
// rounding errors and fill the whole area with the video frame.
@ -261,11 +373,32 @@ impl PaintableImpl for Paintable {
} else {
// Scale texture position and size with the same scale factor as the main video
// frame, and make sure to not render outside (0, 0, width, height).
let x = f32::clamp(*x * scale_x as f32, 0.0, width as f32);
let y = f32::clamp(*y * scale_y as f32, 0.0, height as f32);
let texture_width = f32::min(*paintable_width * scale_x as f32, width as f32);
let texture_height = f32::min(*paintable_height * scale_y as f32, height as f32);
graphene::Rect::new(x, y, texture_width, texture_height)
let (rect_x, rect_y) = if effective_orientation.is_flip_width_height() {
(
f32::clamp(*y * scale_y as f32, 0.0, width as f32),
f32::clamp(*x * scale_x as f32, 0.0, height as f32),
)
} else {
(
f32::clamp(*x * scale_x as f32, 0.0, width as f32),
f32::clamp(*y * scale_y as f32, 0.0, height as f32),
)
};
let (texture_width, texture_height) =
if effective_orientation.is_flip_width_height() {
(
f32::min(*paintable_width * scale_y as f32, width as f32),
f32::min(*paintable_height * scale_x as f32, height as f32),
)
} else {
(
f32::min(*paintable_width * scale_x as f32, width as f32),
f32::min(*paintable_height * scale_y as f32, height as f32),
)
};
graphene::Rect::new(rect_x, rect_y, texture_width, texture_height)
};
// Only premultiply GL textures that expect to be in premultiplied RGBA format.
@ -364,6 +497,19 @@ impl PaintableImpl for Paintable {
}
impl Paintable {
fn effective_orientation(
&self,
paintable_orientation: frame::Orientation,
) -> frame::Orientation {
let orientation = self.orientation.get();
if orientation != frame::Orientation::Auto {
return orientation;
}
assert_ne!(paintable_orientation, frame::Orientation::Auto);
paintable_orientation
}
pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) {
let context = self.gl_context.borrow();