From 54ae12c8c2e289cf847176e8d7f63ff32def80d4 Mon Sep 17 00:00:00 2001
From: Ivan Molodetskikh <yalterz@gmail.com>
Date: Thu, 24 Aug 2023 14:05:15 +0400
Subject: [PATCH] gtk4: Premultiply alpha in GL textures

GTK expects GL textures to have premultiplied alpha. The ones we get
from GStreamer don't, leading to incorrect rendering of semitransparent
frames.

GTK 4.12 gained an API to set a different GL texture format, but it
won't help for older GTK versions. Plus, at the time of writing, it
causes a very slow download/upload path in GTK.

So, use a GTK GL shader node to premultiply the alpha without leaving
the GPU.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs/-/merge_requests/1330>
---
 video/gtk4/src/sink/frame.rs               |  5 +++
 video/gtk4/src/sink/paintable/imp.rs       | 45 ++++++++++++++++++----
 video/gtk4/src/sink/paintable/premult.glsl | 11 ++++++
 3 files changed, 54 insertions(+), 7 deletions(-)
 create mode 100644 video/gtk4/src/sink/paintable/premult.glsl

diff --git a/video/gtk4/src/sink/frame.rs b/video/gtk4/src/sink/frame.rs
index 4a53e1a0..d5223a61 100644
--- a/video/gtk4/src/sink/frame.rs
+++ b/video/gtk4/src/sink/frame.rs
@@ -42,6 +42,7 @@ pub(crate) struct Texture {
     pub width: f32,
     pub height: f32,
     pub global_alpha: f32,
+    pub has_alpha: bool,
 }
 
 struct FrameWrapper(gst_video::VideoFrame<gst_video::video_frame::Readable>);
@@ -149,6 +150,7 @@ impl Frame {
 
         let width = self.frame.width();
         let height = self.frame.height();
+        let has_alpha = self.frame.format_info().has_alpha();
         let (texture, pixel_aspect_ratio) = {
             #[cfg(not(any(target_os = "macos", target_os = "windows", feature = "gst_gl")))]
             {
@@ -183,9 +185,11 @@ impl Frame {
             width: width as f32 * pixel_aspect_ratio as f32,
             height: height as f32,
             global_alpha: 1.0,
+            has_alpha,
         });
 
         for overlay in self.overlays {
+            let has_alpha = overlay.frame.format_info().has_alpha();
             let (texture, _pixel_aspect_ratio) =
                 video_frame_to_memory_texture(overlay.frame, cached_textures, &mut used_textures);
 
@@ -196,6 +200,7 @@ impl Frame {
                 width: overlay.width as f32,
                 height: overlay.height as f32,
                 global_alpha: overlay.global_alpha,
+                has_alpha,
             });
         }
 
diff --git a/video/gtk4/src/sink/paintable/imp.rs b/video/gtk4/src/sink/paintable/imp.rs
index eddfb88e..5fc9eb9c 100644
--- a/video/gtk4/src/sink/paintable/imp.rs
+++ b/video/gtk4/src/sink/paintable/imp.rs
@@ -11,7 +11,7 @@
 
 use gtk::prelude::*;
 use gtk::subclass::prelude::*;
-use gtk::{gdk, glib, graphene};
+use gtk::{gdk, glib, graphene, gsk};
 
 use crate::sink::frame::{Frame, Texture};
 
@@ -28,11 +28,25 @@ static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
     )
 });
 
-#[derive(Default, Debug)]
+#[derive(Debug)]
 pub struct Paintable {
     paintables: RefCell<Vec<Texture>>,
     cached_textures: RefCell<HashMap<usize, gdk::Texture>>,
     gl_context: RefCell<Option<gdk::GLContext>>,
+    premult_shader: gsk::GLShader,
+}
+
+impl Default for Paintable {
+    fn default() -> Self {
+        Self {
+            paintables: Default::default(),
+            cached_textures: Default::default(),
+            gl_context: Default::default(),
+            premult_shader: gsk::GLShader::from_bytes(&glib::Bytes::from_static(include_bytes!(
+                "premult.glsl"
+            ))),
+        }
+    }
 }
 
 #[glib::object_subclass]
@@ -145,14 +159,31 @@ impl PaintableImpl for Paintable {
                 width: paintable_width,
                 height: paintable_height,
                 global_alpha,
+                has_alpha,
             } in &*paintables
             {
                 snapshot.push_opacity(*global_alpha as f64);
-                snapshot.append_texture(
-                    texture,
-                    &graphene::Rect::new(*x, *y, *paintable_width, *paintable_height),
-                );
-                snapshot.pop();
+
+                let bounds = graphene::Rect::new(*x, *y, *paintable_width, *paintable_height);
+
+                // Only premultiply GL textures that expect to be in premultiplied RGBA format.
+                let do_premult = texture.is::<gdk::GLTexture>() && *has_alpha;
+                if do_premult {
+                    snapshot.push_gl_shader(
+                        &self.premult_shader,
+                        &bounds,
+                        gsk::ShaderArgsBuilder::new(&self.premult_shader, None).to_args(),
+                    );
+                }
+
+                snapshot.append_texture(texture, &bounds);
+
+                if do_premult {
+                    snapshot.gl_shader_pop_texture(); // pop texture appended above from the shader
+                    snapshot.pop(); // pop shader
+                }
+
+                snapshot.pop(); // pop opacity
             }
         } else {
             gst::trace!(CAT, imp: self, "Snapshotting black frame");
diff --git a/video/gtk4/src/sink/paintable/premult.glsl b/video/gtk4/src/sink/paintable/premult.glsl
new file mode 100644
index 00000000..f2ab7ec5
--- /dev/null
+++ b/video/gtk4/src/sink/paintable/premult.glsl
@@ -0,0 +1,11 @@
+uniform sampler2D u_texture1;
+
+void mainImage(
+    out vec4 fragColor,
+    in vec2 fragCoord,
+    in vec2 resolution,
+    in vec2 uv
+) {
+  fragColor = GskTexture(u_texture1, uv);
+  fragColor.rgb = fragColor.rgb * fragColor.a;
+}