gtk4: Add support for rendering overlay composition rectangles directly via GTK

video/gtk4/Cargo.toml

@@ -10,9 +10,9 @@ description = "GTK 4 Sink element and Paintable widget"
 [dependencies]
 gtk = { package = "gtk4", git = "https://github.com/gtk-rs/gtk4-rs" }
 
-gst_video = {package="gstreamer-video", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"}
-gst_base = {package="gstreamer-base", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"}
-gst = {package="gstreamer", git="https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"}
+gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", features = ["v1_16"] }
+gst_base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
+gst_video = { package = "gstreamer-video", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs" }
 
 once_cell = "1.0"
 fragile = "1.0.0"

video/gtk4/examples/gtksink.rs

@@ -9,6 +9,9 @@ fn create_ui(app: &gtk::Application) {
     let pipeline = gst::Pipeline::new(None);
     let src = gst::ElementFactory::make("videotestsrc", None).unwrap();
 
+    let overlay = gst::ElementFactory::make("clockoverlay", None).unwrap();
+    overlay.set_property("font-desc", "Monospace 42").unwrap();
+
     let sink = gst::ElementFactory::make("gtk4paintablesink", None).unwrap();
     let paintable = sink
         .property("paintable")
@@ -16,8 +19,16 @@ fn create_ui(app: &gtk::Application) {
         .get::<gdk::Paintable>()
         .unwrap();
 
-    pipeline.add_many(&[&src, &sink]).unwrap();
-    src.link(&sink).unwrap();
+    pipeline.add_many(&[&src, &overlay, &sink]).unwrap();
+    src.link_filtered(
+        &overlay,
+        &gst::Caps::builder("video/x-raw")
+            .field("width", 640)
+            .field("height", 480)
+            .build(),
+    )
+    .unwrap();
+    overlay.link(&sink).unwrap();
 
     let window = gtk::ApplicationWindow::new(app);
     window.set_default_size(640, 480);

video/gtk4/src/sink/frame.rs

@@ -11,77 +11,167 @@
 
 use gtk::prelude::*;
 use gtk::{gdk, glib};
+use std::collections::{HashMap, HashSet};
 use std::convert::AsRef;
 
 #[derive(Debug)]
-pub struct Frame(pub gst_video::VideoFrame<gst_video::video_frame::Readable>);
+pub struct Frame {
+    pub frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
+    pub overlays: Vec<Overlay>,
+}
 
 #[derive(Debug)]
-pub struct Paintable {
-    pub paintable: gdk::Paintable,
-    pub pixel_aspect_ratio: f64,
+pub struct Overlay {
+    pub frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
+    pub x: i32,
+    pub y: i32,
+    pub width: u32,
+    pub height: u32,
+    pub global_alpha: f32,
 }
 
-impl Paintable {
-    pub fn width(&self) -> i32 {
-        f64::round(self.paintable.intrinsic_width() as f64 * self.pixel_aspect_ratio) as i32
-    }
-
-    pub fn height(&self) -> i32 {
-        self.paintable.intrinsic_height()
-    }
+#[derive(Debug)]
+pub struct Texture {
+    pub texture: gdk::Texture,
+    pub x: f32,
+    pub y: f32,
+    pub width: f32,
+    pub height: f32,
+    pub global_alpha: f32,
 }
 
-impl AsRef<[u8]> for Frame {
+struct FrameWrapper(gst_video::VideoFrame<gst_video::video_frame::Readable>);
+impl AsRef<[u8]> for FrameWrapper {
     fn as_ref(&self) -> &[u8] {
         self.0.plane_data(0).unwrap()
     }
 }
 
-impl From<Frame> for Paintable {
-    fn from(f: Frame) -> Paintable {
-        let format = match f.0.format() {
-            gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
-            gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
-            gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
-            gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
-            gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
-            gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
-            _ => unreachable!(),
-        };
-        let width = f.0.width() as i32;
-        let height = f.0.height() as i32;
-        let rowstride = f.0.plane_stride()[0] as usize;
+fn video_frame_to_memory_texture(
+    frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
+    cached_textures: &mut HashMap<usize, gdk::Texture>,
+    used_textures: &mut HashSet<usize>,
+) -> (gdk::Texture, f64) {
+    let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize;
 
-        let pixel_aspect_ratio =
-            (*f.0.info().par().numer() as f64) / (*f.0.info().par().denom() as f64);
+    let pixel_aspect_ratio =
+        (*frame.info().par().numer() as f64) / (*frame.info().par().denom() as f64);
 
-        Paintable {
-            paintable: gdk::MemoryTexture::new(
-                width,
-                height,
-                format,
-                &glib::Bytes::from_owned(f),
-                rowstride,
-            )
-            .upcast(),
-            pixel_aspect_ratio,
+    if let Some(texture) = cached_textures.get(&texture_id) {
+        used_textures.insert(texture_id);
+        return (texture.clone(), pixel_aspect_ratio);
+    }
+
+    let format = match frame.format() {
+        gst_video::VideoFormat::Bgra => gdk::MemoryFormat::B8g8r8a8,
+        gst_video::VideoFormat::Argb => gdk::MemoryFormat::A8r8g8b8,
+        gst_video::VideoFormat::Rgba => gdk::MemoryFormat::R8g8b8a8,
+        gst_video::VideoFormat::Abgr => gdk::MemoryFormat::A8b8g8r8,
+        gst_video::VideoFormat::Rgb => gdk::MemoryFormat::R8g8b8,
+        gst_video::VideoFormat::Bgr => gdk::MemoryFormat::B8g8r8,
+        _ => unreachable!(),
+    };
+    let width = frame.width();
+    let height = frame.height();
+    let rowstride = frame.plane_stride()[0] as usize;
+
+    let texture = gdk::MemoryTexture::new(
+        width as i32,
+        height as i32,
+        format,
+        &glib::Bytes::from_owned(FrameWrapper(frame)),
+        rowstride,
+    )
+    .upcast::<gdk::Texture>();
+
+    cached_textures.insert(texture_id, texture.clone());
+    used_textures.insert(texture_id);
+
+    (texture, pixel_aspect_ratio)
+}
+
+impl Frame {
+    pub fn into_textures(self, cached_textures: &mut HashMap<usize, gdk::Texture>) -> Vec<Texture> {
+        let mut textures = Vec::with_capacity(1 + self.overlays.len());
+        let mut used_textures = HashSet::with_capacity(1 + self.overlays.len());
+
+        let width = self.frame.width();
+        let height = self.frame.height();
+        let (texture, pixel_aspect_ratio) =
+            video_frame_to_memory_texture(self.frame, cached_textures, &mut used_textures);
+
+        textures.push(Texture {
+            texture,
+            x: 0.0,
+            y: 0.0,
+            width: width as f32 * pixel_aspect_ratio as f32,
+            height: height as f32,
+            global_alpha: 1.0,
+        });
+
+        for overlay in self.overlays {
+            let (texture, _pixel_aspect_ratio) =
+                video_frame_to_memory_texture(overlay.frame, cached_textures, &mut used_textures);
+
+            textures.push(Texture {
+                texture,
+                x: overlay.x as f32,
+                y: overlay.y as f32,
+                width: overlay.width as f32,
+                height: overlay.height as f32,
+                global_alpha: overlay.global_alpha,
+            });
         }
+
+        // Remove textures that were not used this time
+        cached_textures.retain(|id, _| used_textures.contains(id));
+
+        textures
     }
 }
 
 impl Frame {
-    pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Self {
-        let video_frame =
-            gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info).unwrap();
-        Self(video_frame)
-    }
+    pub fn new(buffer: &gst::Buffer, info: &gst_video::VideoInfo) -> Result<Self, gst::FlowError> {
+        let frame = gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
+            .map_err(|_| gst::FlowError::Error)?;
 
-    pub fn width(&self) -> u32 {
-        self.0.width()
-    }
+        let overlays = frame
+            .buffer()
+            .iter_meta::<gst_video::VideoOverlayCompositionMeta>()
+            .map(|meta| {
+                meta.overlay()
+                    .iter()
+                    .filter_map(|rect| {
+                        let buffer = rect
+                            .pixels_unscaled_argb(gst_video::VideoOverlayFormatFlags::GLOBAL_ALPHA);
+                        let (x, y, width, height) = rect.render_rectangle();
+                        let global_alpha = rect.global_alpha();
 
-    pub fn height(&self) -> u32 {
-        self.0.height()
+                        let vmeta = buffer.meta::<gst_video::VideoMeta>().unwrap();
+                        let info = gst_video::VideoInfo::builder(
+                            vmeta.format(),
+                            vmeta.width(),
+                            vmeta.height(),
+                        )
+                        .build()
+                        .unwrap();
+                        let frame =
+                            gst_video::VideoFrame::from_buffer_readable(buffer, &info).ok()?;
+
+                        Some(Overlay {
+                            frame,
+                            x,
+                            y,
+                            width,
+                            height,
+                            global_alpha,
+                        })
+                    })
+                    .collect::<Vec<_>>()
+            })
+            .flatten()
+            .collect();
+
+        Ok(Self { frame, overlays })
     }
 }

video/gtk4/src/sink/imp.rs

@@ -132,16 +132,34 @@ impl ElementImpl for PaintableSink {
     fn pad_templates() -> &'static [gst::PadTemplate] {
         static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
             // Those are the supported formats by a gdk::Texture
-            let caps = gst_video::video_make_raw_caps(&[
-                gst_video::VideoFormat::Bgra,
-                gst_video::VideoFormat::Argb,
-                gst_video::VideoFormat::Rgba,
-                gst_video::VideoFormat::Abgr,
-                gst_video::VideoFormat::Rgb,
-                gst_video::VideoFormat::Bgr,
-            ])
-            .any_features()
-            .build();
+            let mut caps = gst::Caps::new_empty();
+            {
+                let caps = caps.get_mut().unwrap();
+
+                for features in [
+                    None,
+                    Some(&["memory:SystemMemory", "meta:GstVideoOverlayComposition"][..]),
+                    Some(&["meta:GstVideoOverlayComposition"][..]),
+                ] {
+                    let mut c = gst_video::video_make_raw_caps(&[
+                        gst_video::VideoFormat::Bgra,
+                        gst_video::VideoFormat::Argb,
+                        gst_video::VideoFormat::Rgba,
+                        gst_video::VideoFormat::Abgr,
+                        gst_video::VideoFormat::Rgb,
+                        gst_video::VideoFormat::Bgr,
+                    ])
+                    .build();
+
+                    if let Some(features) = features {
+                        c.get_mut()
+                            .unwrap()
+                            .set_features_simple(Some(gst::CapsFeatures::new(features)));
+                    }
+
+                    caps.append(c);
+                }
+            }
 
             vec![gst::PadTemplate::new(
                 "sink",
@@ -209,6 +227,9 @@ impl BaseSinkImpl for PaintableSink {
     ) -> Result<(), gst::ErrorMessage> {
         query.add_allocation_meta::<gst_video::VideoMeta>(None);
 
+        // TODO: Provide a preferred "window size" here for higher-resolution rendering
+        query.add_allocation_meta::<gst_video::VideoOverlayCompositionMeta>(None);
+
         self.parent_propose_allocation(element, query)
     }
 }
@@ -227,7 +248,10 @@ impl VideoSinkImpl for PaintableSink {
             gst::FlowError::NotNegotiated
         })?;
 
-        let frame = Frame::new(buffer, info);
+        let frame = Frame::new(buffer, info).map_err(|err| {
+            gst_error!(CAT, obj: element, "Failed to map video frame");
+            err
+        })?;
         self.pending_frame.lock().unwrap().replace(frame);
 
         let sender = self.sender.lock().unwrap();

video/gtk4/src/sink/paintable/imp.rs

@@ -13,11 +13,12 @@ use gtk::prelude::*;
 use gtk::subclass::prelude::*;
 use gtk::{gdk, glib, graphene};
 
-use gst::gst_trace;
+use gst::{gst_debug, gst_trace};
 
-use crate::sink::frame::Paintable;
+use crate::sink::frame::{Frame, Texture};
 
 use std::cell::RefCell;
+use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
 
@@ -31,7 +32,8 @@ pub(super) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
 
 #[derive(Default)]
 pub struct SinkPaintable {
-    pub paintable: RefCell<Option<Paintable>>,
+    paintables: RefCell<Vec<Texture>>,
+    cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
 }
 
 #[glib::object_subclass]
@@ -46,52 +48,86 @@ impl ObjectImpl for SinkPaintable {}
 
 impl PaintableImpl for SinkPaintable {
     fn intrinsic_height(&self, _paintable: &Self::Type) -> i32 {
-        if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() {
-            paintable.intrinsic_height()
+        if let Some(paintable) = self.paintables.borrow().first() {
+            f32::round(paintable.height) as i32
         } else {
             0
         }
     }
 
     fn intrinsic_width(&self, _paintable: &Self::Type) -> i32 {
-        if let Some(Paintable {
-            ref paintable,
-            pixel_aspect_ratio,
-        }) = *self.paintable.borrow()
-        {
-            f64::round(paintable.intrinsic_width() as f64 * pixel_aspect_ratio) as i32
+        if let Some(paintable) = self.paintables.borrow().first() {
+            f32::round(paintable.width) as i32
         } else {
             0
         }
     }
 
     fn intrinsic_aspect_ratio(&self, _paintable: &Self::Type) -> f64 {
-        if let Some(Paintable {
-            ref paintable,
-            pixel_aspect_ratio,
-        }) = *self.paintable.borrow()
-        {
-            paintable.intrinsic_aspect_ratio() * pixel_aspect_ratio
+        if let Some(paintable) = self.paintables.borrow().first() {
+            paintable.width as f64 / paintable.height as f64
         } else {
             0.0
         }
     }
 
-    fn current_image(&self, _paintable: &Self::Type) -> gdk::Paintable {
-        if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() {
-            paintable.clone()
-        } else {
-            gdk::Paintable::new_empty(0, 0).expect("Couldn't create empty paintable")
-        }
-    }
-
     fn snapshot(&self, paintable: &Self::Type, snapshot: &gdk::Snapshot, width: f64, height: f64) {
-        if let Some(Paintable { ref paintable, .. }) = *self.paintable.borrow() {
+        let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap();
+
+        let paintables = self.paintables.borrow();
+
+        if !paintables.is_empty() {
             gst_trace!(CAT, obj: paintable, "Snapshotting frame");
-            paintable.snapshot(snapshot, width, height);
+
+            let (frame_width, frame_height) =
+                paintables.first().map(|p| (p.width, p.height)).unwrap();
+
+            let mut scale_x = width / frame_width as f64;
+            let mut scale_y = height / frame_height as f64;
+            let mut trans_x = 0.0;
+            let mut trans_y = 0.0;
+
+            // TODO: Property for keeping aspect ratio or not
+            if (scale_x - scale_y).abs() > f64::EPSILON {
+                if scale_x > scale_y {
+                    trans_x =
+                        ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0;
+                    scale_x = scale_y;
+                } else {
+                    trans_y =
+                        ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0;
+                    scale_y = scale_x;
+                }
+            }
+
+            if trans_x != 0.0 || trans_y != 0.0 {
+                snapshot.append_color(
+                    &gdk::RGBA::BLACK,
+                    &graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
+                );
+            }
+
+            snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
+            snapshot.scale(scale_x as f32, scale_y as f32);
+
+            for Texture {
+                texture,
+                x,
+                y,
+                width: paintable_width,
+                height: paintable_height,
+                global_alpha,
+            } in &*paintables
+            {
+                snapshot.push_opacity(*global_alpha as f64);
+                snapshot.append_texture(
+                    texture,
+                    &graphene::Rect::new(*x, *y, *paintable_width, *paintable_height),
+                );
+                snapshot.pop();
+            }
         } else {
             gst_trace!(CAT, obj: paintable, "Snapshotting black frame");
-            let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap();
             snapshot.append_color(
                 &gdk::RGBA::BLACK,
                 &graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
@@ -99,3 +135,35 @@ impl PaintableImpl for SinkPaintable {
         }
     }
 }
+
+impl SinkPaintable {
+    pub(super) fn handle_frame_changed(&self, obj: &super::SinkPaintable, frame: Option<Frame>) {
+        if let Some(frame) = frame {
+            gst_trace!(CAT, obj: obj, "Received new frame");
+
+            let new_paintables = frame.into_textures(&mut *self.cached_textures.borrow_mut());
+            let new_size = new_paintables
+                .first()
+                .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))
+                .unwrap();
+
+            let old_paintables = self.paintables.replace(new_paintables);
+            let old_size = old_paintables
+                .first()
+                .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32));
+
+            if Some(new_size) != old_size {
+                gst_debug!(
+                    CAT,
+                    obj: obj,
+                    "Size changed from {:?} to {:?}",
+                    old_size,
+                    new_size,
+                );
+                obj.invalidate_size();
+            }
+
+            obj.invalidate_contents();
+        }
+    }
+}

video/gtk4/src/sink/paintable/mod.rs

@@ -9,14 +9,11 @@
 //
 // SPDX-License-Identifier: MPL-2.0
 
-use crate::sink::frame::{Frame, Paintable};
+use crate::sink::frame::Frame;
 
-use gtk::prelude::*;
 use gtk::subclass::prelude::*;
 use gtk::{gdk, glib};
 
-use gst::{gst_debug, gst_trace};
-
 mod imp;
 
 glib::wrapper! {
@@ -39,27 +36,6 @@ impl Default for SinkPaintable {
 impl SinkPaintable {
     pub(crate) fn handle_frame_changed(&self, frame: Option<Frame>) {
         let self_ = imp::SinkPaintable::from_instance(self);
-        if let Some(frame) = frame {
-            gst_trace!(imp::CAT, obj: self, "Received new frame");
-
-            let paintable: Paintable = frame.into();
-            let new_size = (paintable.width(), paintable.height());
-
-            let old_paintable = self_.paintable.replace(Some(paintable));
-            let old_size = old_paintable.map(|p| (p.width(), p.height()));
-
-            if Some(new_size) != old_size {
-                gst_debug!(
-                    imp::CAT,
-                    obj: self,
-                    "Size changed from {:?} to {:?}",
-                    old_size,
-                    new_size,
-                );
-                self.invalidate_size();
-            }
-
-            self.invalidate_contents();
-        }
+        self_.handle_frame_changed(self, frame);
     }
 }