Merge branch 'gtk-dmabuf-import' into 'main'

gtk4: Implement support for directly importing dmabufs

Closes #441

See merge request gstreamer/gst-plugins-rs!1547

Cargo.lock

@@ -2454,6 +2454,7 @@ dependencies = [
  "gdk4-x11",
  "gst-plugin-version-helper",
  "gstreamer",
+ "gstreamer-allocators",
  "gstreamer-base",
  "gstreamer-gl",
  "gstreamer-gl-egl",
@@ -3059,6 +3060,30 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "gstreamer-allocators"
+version = "0.23.0"
+source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae"
+dependencies = [
+ "glib",
+ "gstreamer",
+ "gstreamer-allocators-sys",
+ "libc",
+ "once_cell",
+]
+
+[[package]]
+name = "gstreamer-allocators-sys"
+version = "0.23.0"
+source = "git+https://gitlab.freedesktop.org/gstreamer/gstreamer-rs?branch=main#e117010bc001f87551713c528bf3abff5c9848ae"
+dependencies = [
+ "glib-sys",
+ "gobject-sys",
+ "gstreamer-sys",
+ "libc",
+ "system-deps",
+]
+
 [[package]]
 name = "gstreamer-app"
 version = "0.23.0"

Cargo.toml

@@ -134,6 +134,7 @@ gdk-wayland = { package = "gdk4-wayland", git = "https://github.com/gtk-rs/gtk4-
 gdk-x11 = { package = "gdk4-x11", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"}
 gdk-win32 = { package = "gdk4-win32", git = "https://github.com/gtk-rs/gtk4-rs", branch = "master"}
 gst = { package = "gstreamer", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
+gst-allocators = { package = "gstreamer-allocators", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
 gst-app = { package = "gstreamer-app", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
 gst-audio = { package = "gstreamer-audio", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }
 gst-base = { package = "gstreamer-base", git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs", branch = "main" }

docs/plugins/gst_plugins_cache.json

@@ -2472,11 +2472,14 @@
                     "GInitiallyUnowned",
                     "GObject"
                 ],
+                "interfaces": [
+                    "GstChildProxy"
+                ],
                 "klass": "Sink/Video",
                 "long-name": "GTK 4 Paintable Sink",
                 "pad-templates": {
                     "sink": {
-                        "caps": "video/x-raw:\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n         format: { RGBA, RGB }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n         format: { RGBA, RGB }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n",
+                        "caps": "video/x-raw(memory:GLMemory, meta:GstVideoOverlayComposition):\n         format: { RGBA, RGB }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n         format: { RGBA, RGB }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:SystemMemory, meta:GstVideoOverlayComposition):\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n\nvideo/x-raw(meta:GstVideoOverlayComposition):\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\nvideo/x-raw:\n         format: { BGRA, ARGB, RGBA, ABGR, RGB, BGR }\n          width: [ 1, 2147483647 ]\n         height: [ 1, 2147483647 ]\n      framerate: [ 0/1, 2147483647/1 ]\n",
                         "direction": "sink",
                         "presence": "always"
                     }

meson.build

@@ -304,6 +304,23 @@ if get_option('gtk4').allowed()
       gtk4_features += 'winegl'
     endif
   endif
+
+  gst_allocators_dep = dependency('gstreamer-allocators-1.0', version: '>=1.24', required: false)
+  gtk_dep = dependency('gtk4', version: '>=4.6', required: get_option('gtk4'))
+  if gtk_dep.found()
+    if host_system == 'linux' and gtk_dep.version().version_compare('>=4.14') and gst_allocators_dep.found()
+      gtk4_features += 'dmabuf'
+    endif
+
+    if gtk_dep.version().version_compare('>=4.14')
+      gtk4_features += 'gtk_v4_14'
+    elif gtk_dep.version().version_compare('>=4.12')
+      gtk4_features += 'gtk_v4_12'
+    elif gtk_dep.version().version_compare('>=4.10')
+      gtk4_features += 'gtk_v4_10'
+    endif
+  endif
+
   plugins += {
     'gtk4': {
       'library': 'libgstgtk4',

video/gtk4/Cargo.toml

@@ -17,6 +17,7 @@ gst = { workspace = true, features = ["v1_16"] }
 gst-base.workspace = true
 gst-video.workspace = true
 gst-gl = { workspace = true, features = ["v1_16"], optional = true }
+gst-allocators = { workspace = true, features = ["v1_24"], optional = true }
 
 gst-gl-wayland = { workspace = true, features = ["v1_16"], optional = true }
 gst-gl-x11 = { workspace = true, features = ["v1_16"], optional = true }
@@ -50,6 +51,7 @@ wayland = ["gtk/v4_6", "gdk-wayland", "gst-gl", "gst-gl-wayland"]
 x11glx = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-x11"]
 x11egl = ["gtk/v4_6", "gdk-x11", "gst-gl", "gst-gl-egl"]
 winegl = ["gdk-win32/egl", "gst-gl-egl"]
+dmabuf = ["gst-allocators", "wayland", "gtk_v4_14", "gst-video/v1_24"]
 capi = []
 doc = ["gst/v1_18"]
 gtk_v4_10 = ["gtk/v4_10"]

video/gtk4/README.md

@@ -1,10 +1,20 @@
-# Gtk 4 Sink & Paintable
+# GTK 4 Sink & Paintable
 
 GTK 4 provides `gtk::Video` & `gtk::Picture` for rendering media such as videos. As the default `gtk::Video` widget doesn't
 offer the possibility to use a custom `gst::Pipeline`. The plugin provides a `gst_video::VideoSink` along with a `gdk::Paintable` that's capable of rendering the sink's frames.
 
-The Sink can generate GL Textures if the system is capable of it, but it needs to be compiled
+The sink can generate GL Textures if the system is capable of it, but it needs
-with either `wayland`, `x11glx` or `x11egl` cargo features.
+to be compiled with either `wayland`, `x11glx` or `x11egl` cargo features. On
+Windows and macOS this is enabled by default.
+
+Additionally, the sink can render DMABufs directly on Linux if GTK 4.14 or
+newer is used. For this the `dmabuf` feature needs to be enabled.
+
+Depending on the GTK version that is used and should be supported as minimum,
+new features or more efficient processing can be opted in with the `gtk_v4_10`,
+`gtk_v4_12` and `gtk_v4_14` features. The minimum GTK version required by the
+sink is GTK 4.4 on Linux without GL support, and 4.6 on Windows and macOS, and
+on Linux with GL support.
 
 # Flatpak Integration
 
@@ -44,7 +54,7 @@ To build and include the plugin in a Flatpak manifest, you can add the following
                 {
                     "type": "git",
                     "url": "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs",
-                    "branch": "0.10"
+                    "branch": "0.12"
                 }
             ],
             "build-options": {

video/gtk4/examples/gtksink.rs

@@ -6,13 +6,6 @@ use gtk::{gdk, gio, glib};
 use std::cell::RefCell;
 
 fn create_ui(app: &gtk::Application) {
-    let window = gtk::ApplicationWindow::new(app);
-    window.set_default_size(640, 480);
-
-    let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0);
-    let picture = gtk::Picture::new();
-    let label = gtk::Label::new(Some("Position: 00:00:00"));
-
     let pipeline = gst::Pipeline::new();
 
     let overlay = gst::ElementFactory::make("clockoverlay")
@@ -64,8 +57,26 @@ fn create_ui(app: &gtk::Application) {
     src.link_filtered(&overlay, &caps).unwrap();
     overlay.link(&sink).unwrap();
 
+    let window = gtk::ApplicationWindow::new(app);
+    window.set_default_size(640, 480);
+
+    let vbox = gtk::Box::new(gtk::Orientation::Vertical, 0);
+
+    let picture = gtk::Picture::new();
     picture.set_paintable(Some(&paintable));
-    vbox.append(&picture);
+
+    #[cfg(feature = "gtk_v4_14")]
+    {
+        let offload = gtk::GraphicsOffload::new(Some(&picture));
+        offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled);
+        vbox.append(&offload);
+    }
+    #[cfg(not(feature = "gtk_v4_14"))]
+    {
+        vbox.append(&picture);
+    }
+
+    let label = gtk::Label::new(Some("Position: 00:00:00"));
     vbox.append(&label);
 
     window.set_child(Some(&vbox));

video/gtk4/src/sink/frame.rs

@@ -14,7 +14,61 @@ use gst_video::prelude::*;
 #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
 use gst_gl::prelude::*;
 use gtk::{gdk, glib};
-use std::collections::{HashMap, HashSet};
+use std::{
+    collections::{HashMap, HashSet},
+    ops,
+};
+
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum VideoInfo {
+    VideoInfo(gst_video::VideoInfo),
+    #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+    DmaDrm(gst_video::VideoInfoDmaDrm),
+}
+
+impl From<gst_video::VideoInfo> for VideoInfo {
+    fn from(v: gst_video::VideoInfo) -> Self {
+        VideoInfo::VideoInfo(v)
+    }
+}
+
+#[cfg(all(target_os = "linux", feature = "dmabuf"))]
+impl From<gst_video::VideoInfoDmaDrm> for VideoInfo {
+    fn from(v: gst_video::VideoInfoDmaDrm) -> Self {
+        VideoInfo::DmaDrm(v)
+    }
+}
+
+impl ops::Deref for VideoInfo {
+    type Target = gst_video::VideoInfo;
+
+    fn deref(&self) -> &Self::Target {
+        match self {
+            VideoInfo::VideoInfo(info) => info,
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            VideoInfo::DmaDrm(info) => info,
+        }
+    }
+}
+
+impl VideoInfo {
+    #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+    fn dma_drm(&self) -> Option<&gst_video::VideoInfoDmaDrm> {
+        match self {
+            VideoInfo::VideoInfo(..) => None,
+            VideoInfo::DmaDrm(info) => Some(info),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub enum TextureCacheId {
+    Memory(usize),
+    #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
+    GL(usize),
+    #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+    DmaBuf([i32; 4]),
+}
 
 #[derive(Debug)]
 enum MappedFrame {
@@ -24,6 +78,17 @@ enum MappedFrame {
         frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
         wrapped_context: gst_gl::GLContext,
     },
+    #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+    DmaBuf {
+        buffer: gst::Buffer,
+        info: gst_video::VideoInfoDmaDrm,
+        n_planes: u32,
+        fds: [i32; 4],
+        offsets: [usize; 4],
+        strides: [usize; 4],
+        width: u32,
+        height: u32,
+    },
 }
 
 impl MappedFrame {
@@ -32,6 +97,8 @@ impl MappedFrame {
             MappedFrame::SysMem(frame) => frame.buffer(),
             #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
             MappedFrame::GL { frame, .. } => frame.buffer(),
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            MappedFrame::DmaBuf { buffer, .. } => buffer,
         }
     }
 
@@ -40,6 +107,8 @@ impl MappedFrame {
             MappedFrame::SysMem(frame) => frame.width(),
             #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
             MappedFrame::GL { frame, .. } => frame.width(),
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            MappedFrame::DmaBuf { info, .. } => info.width(),
         }
     }
 
@@ -48,6 +117,8 @@ impl MappedFrame {
             MappedFrame::SysMem(frame) => frame.height(),
             #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
             MappedFrame::GL { frame, .. } => frame.height(),
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            MappedFrame::DmaBuf { info, .. } => info.height(),
         }
     }
 
@@ -56,6 +127,8 @@ impl MappedFrame {
             MappedFrame::SysMem(frame) => frame.format_info(),
             #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
             MappedFrame::GL { frame, .. } => frame.format_info(),
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            MappedFrame::DmaBuf { info, .. } => info.format_info(),
         }
     }
 }
@@ -108,16 +181,16 @@ fn video_format_to_memory_format(f: gst_video::VideoFormat) -> gdk::MemoryFormat
 
 fn video_frame_to_memory_texture(
     frame: gst_video::VideoFrame<gst_video::video_frame::Readable>,
-    cached_textures: &mut HashMap<usize, gdk::Texture>,
+    cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
-    used_textures: &mut HashSet<usize>,
+    used_textures: &mut HashSet<TextureCacheId>,
 ) -> (gdk::Texture, f64) {
-    let texture_id = frame.plane_data(0).unwrap().as_ptr() as usize;
+    let ptr = frame.plane_data(0).unwrap().as_ptr() as usize;
 
     let pixel_aspect_ratio =
         (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64);
 
-    if let Some(texture) = cached_textures.get(&texture_id) {
+    if let Some(texture) = cached_textures.get(&TextureCacheId::Memory(ptr)) {
-        used_textures.insert(texture_id);
+        used_textures.insert(TextureCacheId::Memory(ptr));
         return (texture.clone(), pixel_aspect_ratio);
     }
 
@@ -135,8 +208,8 @@ fn video_frame_to_memory_texture(
     )
     .upcast::<gdk::Texture>();
 
-    cached_textures.insert(texture_id, texture.clone());
+    cached_textures.insert(TextureCacheId::Memory(ptr), texture.clone());
-    used_textures.insert(texture_id);
+    used_textures.insert(TextureCacheId::Memory(ptr));
 
     (texture, pixel_aspect_ratio)
 }
@@ -144,8 +217,8 @@ fn video_frame_to_memory_texture(
 #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
 fn video_frame_to_gl_texture(
     frame: gst_gl::GLVideoFrame<gst_gl::gl_video_frame::Readable>,
-    cached_textures: &mut HashMap<usize, gdk::Texture>,
+    cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
-    used_textures: &mut HashSet<usize>,
+    used_textures: &mut HashSet<TextureCacheId>,
     gdk_context: &gdk::GLContext,
     #[allow(unused)] wrapped_context: &gst_gl::GLContext,
 ) -> (gdk::Texture, f64) {
@@ -154,8 +227,8 @@ fn video_frame_to_gl_texture(
     let pixel_aspect_ratio =
         (frame.info().par().numer() as f64) / (frame.info().par().denom() as f64);
 
-    if let Some(texture) = cached_textures.get(&(texture_id)) {
+    if let Some(texture) = cached_textures.get(&TextureCacheId::GL(texture_id)) {
-        used_textures.insert(texture_id);
+        used_textures.insert(TextureCacheId::GL(texture_id));
         return (texture.clone(), pixel_aspect_ratio);
     }
 
@@ -237,18 +310,64 @@ fn video_frame_to_gl_texture(
         .upcast::<gdk::Texture>()
     };
 
-    cached_textures.insert(texture_id, texture.clone());
+    cached_textures.insert(TextureCacheId::GL(texture_id), texture.clone());
-    used_textures.insert(texture_id);
+    used_textures.insert(TextureCacheId::GL(texture_id));
 
     (texture, pixel_aspect_ratio)
 }
 
+#[cfg(all(target_os = "linux", feature = "dmabuf"))]
+#[allow(clippy::too_many_arguments)]
+fn video_frame_to_dmabuf_texture(
+    buffer: gst::Buffer,
+    cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
+    used_textures: &mut HashSet<TextureCacheId>,
+    info: &gst_video::VideoInfoDmaDrm,
+    n_planes: u32,
+    fds: &[i32; 4],
+    offsets: &[usize; 4],
+    strides: &[usize; 4],
+    width: u32,
+    height: u32,
+) -> Result<(gdk::Texture, f64), glib::Error> {
+    let pixel_aspect_ratio = (info.par().numer() as f64) / (info.par().denom() as f64);
+
+    if let Some(texture) = cached_textures.get(&TextureCacheId::DmaBuf(*fds)) {
+        used_textures.insert(TextureCacheId::DmaBuf(*fds));
+        return Ok((texture.clone(), pixel_aspect_ratio));
+    }
+
+    let builder = gdk::DmabufTextureBuilder::new();
+    builder.set_display(&gdk::Display::default().unwrap());
+    builder.set_fourcc(info.fourcc());
+    builder.set_modifier(info.modifier());
+    builder.set_width(width);
+    builder.set_height(height);
+    builder.set_n_planes(n_planes);
+    for plane in 0..(n_planes as usize) {
+        builder.set_fd(plane as u32, fds[plane]);
+        builder.set_offset(plane as u32, offsets[plane] as u32);
+        builder.set_stride(plane as u32, strides[plane] as u32);
+    }
+
+    let texture = unsafe {
+        builder.build_with_release_func(move || {
+            drop(buffer);
+        })?
+    };
+
+    cached_textures.insert(TextureCacheId::DmaBuf(*fds), texture.clone());
+    used_textures.insert(TextureCacheId::DmaBuf(*fds));
+
+    Ok((texture, pixel_aspect_ratio))
+}
+
 impl Frame {
     pub(crate) fn into_textures(
         self,
         #[allow(unused_variables)] gdk_context: Option<&gdk::GLContext>,
-        cached_textures: &mut HashMap<usize, gdk::Texture>,
+        cached_textures: &mut HashMap<TextureCacheId, gdk::Texture>,
-    ) -> Vec<Texture> {
+    ) -> Result<Vec<Texture>, glib::Error> {
         let mut textures = Vec::with_capacity(1 + self.overlays.len());
         let mut used_textures = HashSet::with_capacity(1 + self.overlays.len());
 
@@ -278,6 +397,28 @@ impl Frame {
                     &wrapped_context,
                 )
             }
+            #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+            MappedFrame::DmaBuf {
+                buffer,
+                info,
+                n_planes,
+                fds,
+                offsets,
+                strides,
+                width,
+                height,
+            } => video_frame_to_dmabuf_texture(
+                buffer,
+                cached_textures,
+                &mut used_textures,
+                &info,
+                n_planes,
+                &fds,
+                &offsets,
+                &strides,
+                width,
+                height,
+            )?,
         };
 
         textures.push(Texture {
@@ -309,14 +450,14 @@ impl Frame {
         // Remove textures that were not used this time
         cached_textures.retain(|id, _| used_textures.contains(id));
 
-        textures
+        Ok(textures)
     }
 }
 
 impl Frame {
     pub(crate) fn new(
         buffer: &gst::Buffer,
-        info: &gst_video::VideoInfo,
+        info: &VideoInfo,
         #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))] wrapped_context: Option<
             &gst_gl::GLContext,
         >,
@@ -327,77 +468,125 @@ impl Frame {
         // Empty buffers get filtered out in show_frame
         debug_assert!(buffer.n_memory() > 0);
 
-        let mut frame;
+        #[allow(unused_mut)]
+        let mut frame = None;
 
-        #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst-gl")))]
+        #[cfg(all(target_os = "linux", feature = "dmabuf"))]
         {
-            frame = Self {
+            // Check we received a buffer with dmabuf memory and if so do some checks before
-                frame: MappedFrame::SysMem(
+            // passing it onwards
+            if frame.is_none()
+                && buffer
+                    .peek_memory(0)
+                    .is_memory_type::<gst_allocators::DmaBufMemory>()
+            {
+                if let Some((vmeta, info)) =
+                    Option::zip(buffer.meta::<gst_video::VideoMeta>(), info.dma_drm())
+                {
+                    let mut fds = [-1i32; 4];
+                    let mut offsets = [0; 4];
+                    let mut strides = [0; 4];
+                    let n_planes = vmeta.n_planes() as usize;
+
+                    let vmeta_offsets = vmeta.offset();
+                    let vmeta_strides = vmeta.stride();
+
+                    for plane in 0..n_planes {
+                        let Some((range, skip)) =
+                            buffer.find_memory(vmeta_offsets[plane]..(vmeta_offsets[plane] + 1))
+                        else {
+                            break;
+                        };
+
+                        let mem = buffer.peek_memory(range.start);
+                        let Some(mem) = mem.downcast_memory_ref::<gst_allocators::DmaBufMemory>()
+                        else {
+                            break;
+                        };
+
+                        let fd = mem.fd();
+                        fds[plane] = fd;
+                        offsets[plane] = mem.offset() + skip;
+                        strides[plane] = vmeta_strides[plane] as usize;
+                    }
+
+                    // All fds valid?
+                    if fds[0..n_planes].iter().all(|fd| *fd != -1) {
+                        frame = Some(MappedFrame::DmaBuf {
+                            buffer: buffer.clone(),
+                            info: info.clone(),
+                            n_planes: n_planes as u32,
+                            fds,
+                            offsets,
+                            strides,
+                            width: vmeta.width(),
+                            height: vmeta.height(),
+                        });
+                    }
+                }
+            }
+            #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
+            {
+                if frame.is_none() {
+                    // Check we received a buffer with GL memory and if the context of that memory
+                    // can share with the wrapped context around the GDK GL context.
+                    //
+                    // If not it has to be uploaded to the GPU.
+                    let memory_ctx = buffer
+                        .peek_memory(0)
+                        .downcast_memory_ref::<gst_gl::GLBaseMemory>()
+                        .and_then(|m| {
+                            let ctx = m.context();
+                            if wrapped_context
+                                .map_or(false, |wrapped_context| wrapped_context.can_share(ctx))
+                            {
+                                Some(ctx)
+                            } else {
+                                None
+                            }
+                        });
+
+                    if let Some(memory_ctx) = memory_ctx {
+                        // If there is no GLSyncMeta yet then we need to add one here now, which requires
+                        // obtaining a writable buffer.
+                        let mapped_frame = if buffer.meta::<gst_gl::GLSyncMeta>().is_some() {
+                            gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info)
+                                .map_err(|_| gst::FlowError::Error)?
+                        } else {
+                            let mut buffer = buffer.clone();
+                            {
+                                let buffer = buffer.make_mut();
+                                gst_gl::GLSyncMeta::add(buffer, memory_ctx);
+                            }
+                            gst_gl::GLVideoFrame::from_buffer_readable(buffer, info)
+                                .map_err(|_| gst::FlowError::Error)?
+                        };
+
+                        // Now that it's guaranteed that there is a sync meta and the frame is mapped, set
+                        // a sync point so we can ensure that the texture is ready later when making use of
+                        // it as gdk::GLTexture.
+                        let meta = mapped_frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
+                        meta.set_sync_point(memory_ctx);
+
+                        frame = Some(MappedFrame::GL {
+                            frame: mapped_frame,
+                            wrapped_context: wrapped_context.unwrap().clone(),
+                        });
+                    }
+                }
+            }
+        }
+
+        let mut frame = Self {
+            frame: match frame {
+                Some(frame) => frame,
+                None => MappedFrame::SysMem(
                     gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
                         .map_err(|_| gst::FlowError::Error)?,
                 ),
-                overlays: vec![],
+            },
-            };
+            overlays: vec![],
-        }
+        };
-        #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
-        {
-            // Check we received a buffer with GL memory and if the context of that memory
-            // can share with the wrapped context around the GDK GL context.
-            //
-            // If not it has to be uploaded to the GPU.
-            let memory_ctx = buffer
-                .peek_memory(0)
-                .downcast_memory_ref::<gst_gl::GLBaseMemory>()
-                .and_then(|m| {
-                    let ctx = m.context();
-                    if wrapped_context
-                        .map_or(false, |wrapped_context| wrapped_context.can_share(ctx))
-                    {
-                        Some(ctx)
-                    } else {
-                        None
-                    }
-                });
-
-            if let Some(memory_ctx) = memory_ctx {
-                // If there is no GLSyncMeta yet then we need to add one here now, which requires
-                // obtaining a writable buffer.
-                let mapped_frame = if buffer.meta::<gst_gl::GLSyncMeta>().is_some() {
-                    gst_gl::GLVideoFrame::from_buffer_readable(buffer.clone(), info)
-                        .map_err(|_| gst::FlowError::Error)?
-                } else {
-                    let mut buffer = buffer.clone();
-                    {
-                        let buffer = buffer.make_mut();
-                        gst_gl::GLSyncMeta::add(buffer, memory_ctx);
-                    }
-                    gst_gl::GLVideoFrame::from_buffer_readable(buffer, info)
-                        .map_err(|_| gst::FlowError::Error)?
-                };
-
-                // Now that it's guaranteed that there is a sync meta and the frame is mapped, set
-                // a sync point so we can ensure that the texture is ready later when making use of
-                // it as gdk::GLTexture.
-                let meta = mapped_frame.buffer().meta::<gst_gl::GLSyncMeta>().unwrap();
-                meta.set_sync_point(memory_ctx);
-
-                frame = Self {
-                    frame: MappedFrame::GL {
-                        frame: mapped_frame,
-                        wrapped_context: wrapped_context.unwrap().clone(),
-                    },
-                    overlays: vec![],
-                };
-            } else {
-                frame = Self {
-                    frame: MappedFrame::SysMem(
-                        gst_video::VideoFrame::from_buffer_readable(buffer.clone(), info)
-                            .map_err(|_| gst::FlowError::Error)?,
-                    ),
-                    overlays: vec![],
-                };
-            }
-        }
 
         frame.overlays = frame
             .frame

video/gtk4/src/sink/imp.rs

@@ -1,7 +1,7 @@
 //
 // Copyright (C) 2021 Bilal Elmoussaoui <bil.elmoussaoui@gmail.com>
 // Copyright (C) 2021 Jordan Petridis <jordan@centricular.com>
-// Copyright (C) 2021 Sebastian Dröge <sebastian@centricular.com>
+// Copyright (C) 2021-2024 Sebastian Dröge <sebastian@centricular.com>
 //
 // This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
 // If a copy of the MPL was not distributed with this file, You can obtain one at
@@ -62,7 +62,7 @@ pub(crate) static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
 pub struct PaintableSink {
     paintable: Mutex<Option<ThreadGuard<Paintable>>>,
     window: Mutex<Option<ThreadGuard<gtk::Window>>>,
-    info: Mutex<Option<gst_video::VideoInfo>>,
+    info: Mutex<Option<super::frame::VideoInfo>>,
     sender: Mutex<Option<async_channel::Sender<SinkEvent>>>,
     pending_frame: Mutex<Option<Frame>>,
     cached_caps: Mutex<Option<gst::Caps>>,
@@ -82,6 +82,7 @@ impl ObjectSubclass for PaintableSink {
     const NAME: &'static str = "GstGtk4PaintableSink";
     type Type = super::PaintableSink;
     type ParentType = gst_video::VideoSink;
+    type Interfaces = (gst::ChildProxy,);
 }
 
 impl ObjectImpl for PaintableSink {
@@ -110,12 +111,14 @@ impl ObjectImpl for PaintableSink {
                     return None::<&gdk::Paintable>.to_value();
                 }
 
-                let mut paintable = self.paintable.lock().unwrap();
+                let mut paintable_guard = self.paintable.lock().unwrap();
-                if paintable.is_none() {
+                let mut created = false;
-                    self.create_paintable(&mut paintable);
+                if paintable_guard.is_none() {
+                    created = true;
+                    self.create_paintable(&mut paintable_guard);
                 }
 
-                let paintable = match &*paintable {
+                let paintable = match &*paintable_guard {
                     Some(ref paintable) => paintable,
                     None => {
                         gst::error!(CAT, imp: self, "Failed to create paintable");
@@ -124,16 +127,31 @@ impl ObjectImpl for PaintableSink {
                 };
 
                 // Getter must be called from the main thread
-                if paintable.is_owner() {
+                if !paintable.is_owner() {
-                    paintable.get_ref().to_value()
-                } else {
                     gst::error!(
                         CAT,
                         imp: self,
                         "Can't retrieve Paintable from non-main thread"
                     );
-                    None::<&gdk::Paintable>.to_value()
+                    return None::<&gdk::Paintable>.to_value();
                 }
+
+                let paintable = paintable.get_ref().clone();
+                drop(paintable_guard);
+
+                if created {
+                    let self_ = self.to_owned();
+                    glib::MainContext::default().invoke(move || {
+                        let paintable_guard = self_.paintable.lock().unwrap();
+                        if let Some(paintable) = &*paintable_guard {
+                            let paintable_clone = paintable.get_ref().clone();
+                            drop(paintable_guard);
+                            self_.obj().child_added(&paintable_clone, "paintable");
+                        }
+                    });
+                }
+
+                paintable.to_value()
             }
             _ => unimplemented!(),
         }
@@ -163,53 +181,99 @@ impl ElementImpl for PaintableSink {
             {
                 let caps = caps.get_mut().unwrap();
 
+                #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+                {
+                    for features in [
+                        [
+                            gst_allocators::CAPS_FEATURE_MEMORY_DMABUF,
+                            gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
+                        ]
+                        .as_slice(),
+                        [gst_allocators::CAPS_FEATURE_MEMORY_DMABUF].as_slice(),
+                    ] {
+                        let c = gst_video::VideoCapsBuilder::new()
+                            .format(gst_video::VideoFormat::DmaDrm)
+                            .features(features.iter().copied())
+                            .build();
+                        caps.append(c);
+                    }
+                }
+
                 for features in [
-                    None,
                     #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
                     Some(gst::CapsFeatures::new([
-                        "memory:GLMemory",
+                        gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY,
-                        "meta:GstVideoOverlayComposition",
+                        gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
                     ])),
                     #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
-                    Some(gst::CapsFeatures::new(["memory:GLMemory"])),
+                    Some(gst::CapsFeatures::new([
+                        gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY,
+                    ])),
                     Some(gst::CapsFeatures::new([
                         "memory:SystemMemory",
-                        "meta:GstVideoOverlayComposition",
+                        gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
                     ])),
-                    Some(gst::CapsFeatures::new(["meta:GstVideoOverlayComposition"])),
+                    Some(gst::CapsFeatures::new([
+                        gst_video::CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION,
+                    ])),
+                    None,
                 ] {
-                    const GL_FORMATS: &[gst_video::VideoFormat] =
+                    #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
-                        &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb];
-                    const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[
-                        gst_video::VideoFormat::Bgra,
-                        gst_video::VideoFormat::Argb,
-                        gst_video::VideoFormat::Rgba,
-                        gst_video::VideoFormat::Abgr,
-                        gst_video::VideoFormat::Rgb,
-                        gst_video::VideoFormat::Bgr,
-                    ];
-
-                    let formats = if features
-                        .as_ref()
-                        .is_some_and(|features| features.contains("memory:GLMemory"))
                     {
-                        GL_FORMATS
+                        const GL_FORMATS: &[gst_video::VideoFormat] =
-                    } else {
+                            &[gst_video::VideoFormat::Rgba, gst_video::VideoFormat::Rgb];
-                        NON_GL_FORMATS
+                        const NON_GL_FORMATS: &[gst_video::VideoFormat] = &[
-                    };
+                            gst_video::VideoFormat::Bgra,
+                            gst_video::VideoFormat::Argb,
+                            gst_video::VideoFormat::Rgba,
+                            gst_video::VideoFormat::Abgr,
+                            gst_video::VideoFormat::Rgb,
+                            gst_video::VideoFormat::Bgr,
+                        ];
 
-                    let mut c = gst_video::video_make_raw_caps(formats).build();
+                        let formats = if features.as_ref().is_some_and(|features| {
+                            features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY)
+                        }) {
+                            GL_FORMATS
+                        } else {
+                            NON_GL_FORMATS
+                        };
 
-                    if let Some(features) = features {
+                        let mut c = gst_video::video_make_raw_caps(formats).build();
-                        let c = c.get_mut().unwrap();
 
-                        if features.contains("memory:GLMemory") {
+                        if let Some(features) = features {
-                            c.set("texture-target", "2D")
+                            let c = c.get_mut().unwrap();
+
+                            if features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY) {
+                                c.set("texture-target", "2D")
+                            }
+                            c.set_features_simple(Some(features));
                         }
-                        c.set_features_simple(Some(features));
+                        caps.append(c);
                     }
+                    #[cfg(not(any(
+                        target_os = "macos",
+                        target_os = "windows",
+                        feature = "gst-gl"
+                    )))]
+                    {
+                        const FORMATS: &[gst_video::VideoFormat] = &[
+                            gst_video::VideoFormat::Bgra,
+                            gst_video::VideoFormat::Argb,
+                            gst_video::VideoFormat::Rgba,
+                            gst_video::VideoFormat::Abgr,
+                            gst_video::VideoFormat::Rgb,
+                            gst_video::VideoFormat::Bgr,
+                        ];
 
-                    caps.append(c);
+                        let mut c = gst_video::video_make_raw_caps(FORMATS).build();
+
+                        if let Some(features) = features {
+                            let c = c.get_mut().unwrap();
+                            c.set_features_simple(Some(features));
+                        }
+                        caps.append(c);
+                    }
                 }
             }
 
@@ -244,18 +308,31 @@ impl ElementImpl for PaintableSink {
                     }
                 }
 
-                let mut paintable = self.paintable.lock().unwrap();
+                let mut paintable_guard = self.paintable.lock().unwrap();
-
+                let mut created = false;
-                if paintable.is_none() {
+                if paintable_guard.is_none() {
-                    self.create_paintable(&mut paintable);
+                    created = true;
+                    self.create_paintable(&mut paintable_guard);
                 }
 
-                if paintable.is_none() {
+                if paintable_guard.is_none() {
                     gst::error!(CAT, imp: self, "Failed to create paintable");
                     return Err(gst::StateChangeError);
                 }
 
-                drop(paintable);
+                drop(paintable_guard);
+
+                if created {
+                    let self_ = self.to_owned();
+                    glib::MainContext::default().invoke(move || {
+                        let paintable_guard = self_.paintable.lock().unwrap();
+                        if let Some(paintable) = &*paintable_guard {
+                            let paintable_clone = paintable.get_ref().clone();
+                            drop(paintable_guard);
+                            self_.obj().child_added(&paintable_clone, "paintable");
+                        }
+                    });
+                }
 
                 // Notify the pipeline about the GL display and wrapped context so that any other
                 // elements in the pipeline ideally use the same / create GL contexts that are
@@ -361,8 +438,21 @@ impl BaseSinkImpl for PaintableSink {
     fn set_caps(&self, caps: &gst::Caps) -> Result<(), gst::LoggableError> {
         gst::debug!(CAT, imp: self, "Setting caps {caps:?}");
 
-        let video_info = gst_video::VideoInfo::from_caps(caps)
+        #[allow(unused_mut)]
-            .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?;
+        let mut video_info = None;
+        #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+        {
+            if let Ok(info) = gst_video::VideoInfoDmaDrm::from_caps(caps) {
+                video_info = Some(info.into());
+            }
+        }
+
+        let video_info = match video_info {
+            Some(info) => info,
+            None => gst_video::VideoInfo::from_caps(caps)
+                .map_err(|_| gst::loggable_error!(CAT, "Invalid caps"))?
+                .into(),
+        };
 
         self.info.lock().unwrap().replace(video_info);
 
@@ -516,10 +606,11 @@ impl PaintableSink {
 
         match action {
             SinkEvent::FrameChanged => {
+                let Some(frame) = self.pending_frame() else {
+                    return glib::ControlFlow::Continue;
+                };
                 gst::trace!(CAT, imp: self, "Frame changed");
-                paintable
+                paintable.get_ref().handle_frame_changed(&self.obj(), frame);
-                    .get_ref()
-                    .handle_frame_changed(self.pending_frame())
             }
         }
 
@@ -530,13 +621,59 @@ impl PaintableSink {
         #[allow(unused_mut)]
         let mut tmp_caps = Self::pad_templates()[0].caps().clone();
 
+        #[cfg(all(target_os = "linux", feature = "dmabuf"))]
+        {
+            let formats = utils::invoke_on_main_thread(move || {
+                let Some(display) = gdk::Display::default() else {
+                    return vec![];
+                };
+                let dmabuf_formats = display.dmabuf_formats();
+
+                let mut formats = vec![];
+                let n_formats = dmabuf_formats.n_formats();
+                for i in 0..n_formats {
+                    let (fourcc, modifier) = dmabuf_formats.format(i);
+
+                    if fourcc == 0 || modifier == (u64::MAX >> 8) {
+                        continue;
+                    }
+
+                    formats.push(gst_video::dma_drm_fourcc_to_string(fourcc, modifier));
+                }
+
+                formats
+            });
+
+            if formats.is_empty() {
+                // Filter out dmabufs caps from the template pads if we have no supported formats
+                if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) {
+                    tmp_caps = tmp_caps
+                        .iter_with_features()
+                        .filter(|(_, features)| {
+                            !features.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF)
+                        })
+                        .map(|(s, c)| (s.to_owned(), c.to_owned()))
+                        .collect::<gst::Caps>();
+                }
+            } else {
+                let tmp_caps = tmp_caps.make_mut();
+                for (s, f) in tmp_caps.iter_with_features_mut() {
+                    if f.contains(gst_allocators::CAPS_FEATURE_MEMORY_DMABUF) {
+                        s.set("drm-format", gst::List::new(&formats));
+                    }
+                }
+            }
+        }
+
         #[cfg(any(target_os = "macos", target_os = "windows", feature = "gst-gl"))]
         {
             // Filter out GL caps from the template pads if we have no context
             if !matches!(&*GL_CONTEXT.lock().unwrap(), GLContext::Initialized { .. }) {
                 tmp_caps = tmp_caps
                     .iter_with_features()
-                    .filter(|(_, features)| !features.contains("memory:GLMemory"))
+                    .filter(|(_, features)| {
+                        !features.contains(gst_gl::CAPS_FEATURE_MEMORY_GL_MEMORY)
+                    })
                     .map(|(s, c)| (s.to_owned(), c.to_owned()))
                     .collect::<gst::Caps>();
             }
@@ -564,7 +701,17 @@ impl PaintableSink {
             let window = gtk::Window::new();
             let picture = gtk::Picture::new();
             picture.set_paintable(Some(&paintable));
-            window.set_child(Some(&picture));
+
+            #[cfg(feature = "gtk_v4_14")]
+            {
+                let offload = gtk::GraphicsOffload::new(Some(&picture));
+                offload.set_enabled(gtk::GraphicsOffloadEnabled::Enabled);
+                window.set_child(Some(&offload));
+            }
+            #[cfg(not(feature = "gtk_v4_14"))]
+            {
+                window.set_child(Some(&picture));
+            }
             window.set_default_size(640, 480);
 
             window.connect_close_request({
@@ -1073,3 +1220,33 @@ impl PaintableSink {
         }
     }
 }
+
+impl ChildProxyImpl for PaintableSink {
+    fn child_by_index(&self, index: u32) -> Option<glib::Object> {
+        if index != 0 {
+            return None;
+        }
+
+        let paintable = self.paintable.lock().unwrap();
+        paintable
+            .as_ref()
+            .filter(|p| p.is_owner())
+            .map(|p| p.get_ref().upcast_ref::<glib::Object>().clone())
+    }
+
+    fn child_by_name(&self, name: &str) -> Option<glib::Object> {
+        if name == "paintable" {
+            return self.child_by_index(0);
+        }
+        None
+    }
+
+    fn children_count(&self) -> u32 {
+        let paintable = self.paintable.lock().unwrap();
+        if paintable.is_some() {
+            1
+        } else {
+            0
+        }
+    }
+}

video/gtk4/src/sink/mod.rs

@@ -22,7 +22,8 @@ enum SinkEvent {
 
 glib::wrapper! {
     pub struct PaintableSink(ObjectSubclass<imp::PaintableSink>)
-        @extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object;
+        @extends gst_video::VideoSink, gst_base::BaseSink, gst::Element, gst::Object,
+        @implements gst::ChildProxy;
 }
 
 impl PaintableSink {

video/gtk4/src/sink/paintable/imp.rs

@@ -31,12 +31,13 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
 #[derive(Debug)]
 pub struct Paintable {
     paintables: RefCell<Vec<Texture>>,
-    cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
+    cached_textures: RefCell<HashMap<super::super::frame::TextureCacheId, gdk::Texture>>,
     gl_context: RefCell<Option<gdk::GLContext>>,
     background_color: Cell<gdk::RGBA>,
     #[cfg(feature = "gtk_v4_10")]
     scaling_filter: Cell<gsk::ScalingFilter>,
     use_scaling_filter: Cell<bool>,
+    force_aspect_ratio: Cell<bool>,
     #[cfg(not(feature = "gtk_v4_10"))]
     premult_shader: gsk::GLShader,
 }
@@ -51,6 +52,7 @@ impl Default for Paintable {
             #[cfg(feature = "gtk_v4_10")]
             scaling_filter: Cell::new(gsk::ScalingFilter::Linear),
             use_scaling_filter: Cell::new(false),
+            force_aspect_ratio: Cell::new(false),
             #[cfg(not(feature = "gtk_v4_10"))]
             premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
                 "premult.glsl"
@@ -94,6 +96,11 @@ impl ObjectImpl for Paintable {
                     .blurb("Use selected scaling filter or GTK default for rendering")
                     .default_value(false)
                     .build(),
+                glib::ParamSpecBoolean::builder("force-aspect-ratio")
+                    .nick("Force Aspect Ratio")
+                    .blurb("When enabled, scaling will respect original aspect ratio")
+                    .default_value(true)
+                    .build(),
             ]
         });
 
@@ -117,6 +124,7 @@ impl ObjectImpl for Paintable {
             "scaling-filter" => self.scaling_filter.get().to_value(),
             #[cfg(feature = "gtk_v4_10")]
             "use-scaling-filter" => self.use_scaling_filter.get().to_value(),
+            "force-aspect-ratio" => self.force_aspect_ratio.get().to_value(),
             _ => unimplemented!(),
         }
     }
@@ -139,6 +147,7 @@ impl ObjectImpl for Paintable {
             "scaling-filter" => self.scaling_filter.set(value.get().unwrap()),
             #[cfg(feature = "gtk_v4_10")]
             "use-scaling-filter" => self.use_scaling_filter.set(value.get().unwrap()),
+            "force-aspect-ratio" => self.force_aspect_ratio.set(value.get().unwrap()),
             _ => unimplemented!(),
         }
     }
@@ -173,40 +182,66 @@ impl PaintableImpl for Paintable {
         let snapshot = snapshot.downcast_ref::<gtk::Snapshot>().unwrap();
 
         let background_color = self.background_color.get();
+        let force_aspect_ratio = self.force_aspect_ratio.get();
         let paintables = self.paintables.borrow();
 
-        if !paintables.is_empty() {
+        let Some(first_paintable) = paintables.first() else {
-            gst::trace!(CAT, imp: self, "Snapshotting frame");
+            gst::trace!(CAT, imp: self, "Snapshotting black frame");
-
-            let (frame_width, frame_height) =
-                paintables.first().map(|p| (p.width, p.height)).unwrap();
-
-            let mut scale_x = width / frame_width as f64;
-            let mut scale_y = height / frame_height as f64;
-            let mut trans_x = 0.0;
-            let mut trans_y = 0.0;
-
-            // TODO: Property for keeping aspect ratio or not
-            if (scale_x - scale_y).abs() > f64::EPSILON {
-                if scale_x > scale_y {
-                    trans_x =
-                        ((frame_width as f64 * scale_x) - (frame_width as f64 * scale_y)) / 2.0;
-                    scale_x = scale_y;
-                } else {
-                    trans_y =
-                        ((frame_height as f64 * scale_y) - (frame_height as f64 * scale_x)) / 2.0;
-                    scale_y = scale_x;
-                }
-            }
-
             snapshot.append_color(
                 &background_color,
                 &graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
             );
 
-            snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
+            return;
+        };
 
-            for Texture {
+        gst::trace!(CAT, imp: self, "Snapshotting frame");
+
+        // The first paintable is the actual video frame and defines the overall size.
+        //
+        // Based on its size relative to the snapshot width/height, all other paintables are
+        // scaled accordingly.
+        let (frame_width, frame_height) = (first_paintable.width, first_paintable.height);
+
+        let mut scale_x = width / frame_width as f64;
+        let mut scale_y = height / frame_height as f64;
+
+        // Usually the caller makes sure that the aspect ratio is preserved. To enforce this here
+        // optionally, we scale the frame equally in both directions and center it. In addition the
+        // background color is drawn behind the frame to fill the gaps.
+        //
+        // This is not done by default for performance reasons and usually would draw a <1px
+        // background.
+        if force_aspect_ratio {
+            let mut trans_x = 0.0;
+            let mut trans_y = 0.0;
+
+            if (scale_x - scale_y).abs() > f64::EPSILON {
+                if scale_x > scale_y {
+                    trans_x = (width - (frame_width as f64 * scale_y)) / 2.0;
+                    scale_x = scale_y;
+                } else {
+                    trans_y = (height - (frame_height as f64 * scale_x)) / 2.0;
+                    scale_y = scale_x;
+                }
+            }
+
+            if !background_color.is_clear() && (trans_x > f64::EPSILON || trans_y > f64::EPSILON) {
+                snapshot.append_color(
+                    &background_color,
+                    &graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
+                );
+            }
+            snapshot.translate(&graphene::Point::new(trans_x as f32, trans_y as f32));
+        }
+
+        // Make immutable
+        let scale_x = scale_x;
+        let scale_y = scale_y;
+
+        for (
+            idx,
+            Texture {
                 texture,
                 x,
                 y,
@@ -214,151 +249,159 @@ impl PaintableImpl for Paintable {
                 height: paintable_height,
                 global_alpha,
                 has_alpha,
-            } in &*paintables
+            },
+        ) in paintables.iter().enumerate()
+        {
+            snapshot.push_opacity(*global_alpha as f64);
+
+            let bounds = if !force_aspect_ratio && idx == 0 {
+                // While this should end up with width again, be explicit in this case to avoid
+                // rounding errors and fill the whole area with the video frame.
+                graphene::Rect::new(0.0, 0.0, width as f32, height as f32)
+            } else {
+                // Scale texture position and size with the same scale factor as the main video
+                // frame, and make sure to not render outside (0, 0, width, height).
+                let x = f32::clamp(*x * scale_x as f32, 0.0, width as f32);
+                let y = f32::clamp(*y * scale_y as f32, 0.0, height as f32);
+                let texture_width = f32::min(*paintable_width * scale_x as f32, width as f32);
+                let texture_height = f32::min(*paintable_height * scale_y as f32, height as f32);
+                graphene::Rect::new(x, y, texture_width, texture_height)
+            };
+
+            // Only premultiply GL textures that expect to be in premultiplied RGBA format.
+            //
+            // For GTK 4.14 or newer we use the correct format directly when building the
+            // texture, but only if a GLES3+ context is used. In that case the NGL renderer is
+            // used by GTK, which supports non-premultiplied formats correctly and fast.
+            //
+            // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
+            // self-mask to pre-multiply the alpha.
+            //
+            // For GTK before 4.10, we use a GL shader and hope that it works.
+            #[cfg(feature = "gtk_v4_10")]
             {
-                snapshot.push_opacity(*global_alpha as f64);
+                let context_requires_premult = {
-
+                    #[cfg(feature = "gtk_v4_14")]
-                let texture_width = *paintable_width * scale_x as f32;
+                    {
-                let texture_height = *paintable_height * scale_y as f32;
+                        self.gl_context.borrow().as_ref().map_or(false, |context| {
-                let x = *x * scale_x as f32;
+                            context.api() != gdk::GLAPI::GLES || context.version().0 < 3
-                let y = *y * scale_y as f32;
+                        })
-                let bounds = graphene::Rect::new(x, y, texture_width, texture_height);
-
-                // Only premultiply GL textures that expect to be in premultiplied RGBA format.
-                //
-                // For GTK 4.14 or newer we use the correct format directly when building the
-                // texture, but only if a GLES3+ context is used. In that case the NGL renderer is
-                // used by GTK, which supports non-premultiplied formats correctly and fast.
-                //
-                // For GTK 4.10-4.12, or 4.14 and newer if a GLES2 context is used, we use a
-                // self-mask to pre-multiply the alpha.
-                //
-                // For GTK before 4.10, we use a GL shader and hope that it works.
-                #[cfg(feature = "gtk_v4_10")]
-                {
-                    let context_requires_premult = {
-                        #[cfg(feature = "gtk_v4_14")]
-                        {
-                            self.gl_context.borrow().as_ref().map_or(false, |context| {
-                                context.api() != gdk::GLAPI::GLES || context.version().0 < 3
-                            })
-                        }
-
-                        #[cfg(not(feature = "gtk_v4_14"))]
-                        {
-                            true
-                        }
-                    };
-
-                    let do_premult =
-                        context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
-                    if do_premult {
-                        snapshot.push_mask(gsk::MaskMode::Alpha);
-                        if self.use_scaling_filter.get() {
-                            #[cfg(feature = "gtk_v4_10")]
-                            snapshot.append_scaled_texture(
-                                texture,
-                                self.scaling_filter.get(),
-                                &bounds,
-                            );
-                        } else {
-                            snapshot.append_texture(texture, &bounds);
-                        }
-                        snapshot.pop(); // pop mask
-
-                        // color matrix to set alpha of the source to 1.0 as it was
-                        // already applied via the mask just above.
-                        snapshot.push_color_matrix(
-                            &graphene::Matrix::from_float({
-                                [
-                                    1.0, 0.0, 0.0, 0.0, //
-                                    0.0, 1.0, 0.0, 0.0, //
-                                    0.0, 0.0, 1.0, 0.0, //
-                                    0.0, 0.0, 0.0, 0.0,
-                                ]
-                            }),
-                            &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
-                        );
                     }
 
+                    #[cfg(not(feature = "gtk_v4_14"))]
+                    {
+                        true
+                    }
+                };
+
+                let do_premult =
+                    context_requires_premult && texture.is::<gdk::GLTexture>() && *has_alpha;
+                if do_premult {
+                    snapshot.push_mask(gsk::MaskMode::Alpha);
                     if self.use_scaling_filter.get() {
                         #[cfg(feature = "gtk_v4_10")]
                         snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
                     } else {
                         snapshot.append_texture(texture, &bounds);
                     }
+                    snapshot.pop(); // pop mask
 
-                    if do_premult {
+                    // color matrix to set alpha of the source to 1.0 as it was
-                        snapshot.pop(); // pop color matrix
+                    // already applied via the mask just above.
-                        snapshot.pop(); // pop mask 2
+                    snapshot.push_color_matrix(
-                    }
+                        &graphene::Matrix::from_float({
-                }
+                            [
-                #[cfg(not(feature = "gtk_v4_10"))]
+                                1.0, 0.0, 0.0, 0.0, //
-                {
+                                0.0, 1.0, 0.0, 0.0, //
-                    let do_premult =
+                                0.0, 0.0, 1.0, 0.0, //
-                        texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
+                                0.0, 0.0, 0.0, 0.0,
-                    if do_premult {
+                            ]
-                        snapshot.push_gl_shader(
+                        }),
-                            &self.premult_shader,
+                        &graphene::Vec4::new(0.0, 0.0, 0.0, 1.0),
-                            &bounds,
+                    );
-                            gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
-                        );
-                    }
-
-                    if self.use_scaling_filter.get() {
-                        #[cfg(feature = "gtk_v4_10")]
-                        snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
-                    } else {
-                        snapshot.append_texture(texture, &bounds);
-                    }
-
-                    if do_premult {
-                        snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
-                        snapshot.pop(); // pop shader
-                    }
                 }
 
-                snapshot.pop(); // pop opacity
+                if self.use_scaling_filter.get() {
+                    #[cfg(feature = "gtk_v4_10")]
+                    snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
+                } else {
+                    snapshot.append_texture(texture, &bounds);
+                }
+
+                if do_premult {
+                    snapshot.pop(); // pop color matrix
+                    snapshot.pop(); // pop mask 2
+                }
             }
-        } else {
+            #[cfg(not(feature = "gtk_v4_10"))]
-            gst::trace!(CAT, imp: self, "Snapshotting black frame");
+            {
-            snapshot.append_color(
+                let do_premult =
-                &background_color,
+                    texture.is::<gdk::GLTexture>() && *has_alpha && gtk::micro_version() < 13;
-                &graphene::Rect::new(0f32, 0f32, width as f32, height as f32),
+                if do_premult {
-            );
+                    snapshot.push_gl_shader(
+                        &self.premult_shader,
+                        &bounds,
+                        gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
+                    );
+                }
+
+                if self.use_scaling_filter.get() {
+                    #[cfg(feature = "gtk_v4_10")]
+                    snapshot.append_scaled_texture(texture, self.scaling_filter.get(), &bounds);
+                } else {
+                    snapshot.append_texture(texture, &bounds);
+                }
+
+                if do_premult {
+                    snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
+                    snapshot.pop(); // pop shader
+                }
+            }
+
+            snapshot.pop(); // pop opacity
         }
     }
 }
 
 impl Paintable {
-    pub(super) fn handle_frame_changed(&self, frame: Option<Frame>) {
+    pub(super) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) {
         let context = self.gl_context.borrow();
-        if let Some(frame) = frame {
-            gst::trace!(CAT, imp: self, "Received new frame");
 
-            let new_paintables =
+        gst::trace!(CAT, imp: self, "Received new frame");
-                frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut());
-            let new_size = new_paintables
-                .first()
-                .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))
-                .unwrap();
 
-            let old_paintables = self.paintables.replace(new_paintables);
+        let new_paintables =
-            let old_size = old_paintables
+            match frame.into_textures(context.as_ref(), &mut self.cached_textures.borrow_mut()) {
-                .first()
+                Ok(textures) => textures,
-                .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32));
+                Err(err) => {
+                    gst::element_error!(
+                        sink,
+                        gst::ResourceError::Failed,
+                        ["Failed to transform frame into textures: {err}"]
+                    );
+                    return;
+                }
+            };
 
-            if Some(new_size) != old_size {
+        let new_size = new_paintables
-                gst::debug!(
+            .first()
-                    CAT,
+            .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32))
-                    imp: self,
+            .unwrap();
-                    "Size changed from {old_size:?} to {new_size:?}",
-                );
-                self.obj().invalidate_size();
-            }
 
-            self.obj().invalidate_contents();
+        let old_paintables = self.paintables.replace(new_paintables);
+        let old_size = old_paintables
+            .first()
+            .map(|p| (f32::round(p.width) as u32, f32::round(p.height) as u32));
+
+        if Some(new_size) != old_size {
+            gst::debug!(
+                CAT,
+                imp: self,
+                "Size changed from {old_size:?} to {new_size:?}",
+            );
+            self.obj().invalidate_size();
         }
+
+        self.obj().invalidate_contents();
     }
 
     pub(super) fn handle_flush_frames(&self) {

video/gtk4/src/sink/paintable/mod.rs

@@ -30,8 +30,8 @@ impl Paintable {
 }
 
 impl Paintable {
-    pub(crate) fn handle_frame_changed(&self, frame: Option<Frame>) {
+    pub(crate) fn handle_frame_changed(&self, sink: &crate::PaintableSink, frame: Frame) {
-        self.imp().handle_frame_changed(frame);
+        self.imp().handle_frame_changed(sink, frame);
     }
 
     pub(crate) fn handle_flush_frames(&self) {