overlay: add support for global-alpha.

Handle global-alpha from GstVideoOverlayComposition API. Likewise,
the same code path could also work for premultiplied-alpha but this
was not tested.

Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne@intel.com>

gst-libs/gst/vaapi/gstvaapisubpicture.c

@@ -300,24 +300,29 @@ gst_vaapi_subpicture_new_from_overlay_rectangle(
     GstVaapiImage *image;
     GstVaapiImageRaw raw_image;
     GstBuffer *buffer;
+    gfloat global_alpha;
     guint width, height, stride;
+    guint hw_flags, flags;
 
     g_return_val_if_fail(GST_IS_VIDEO_OVERLAY_RECTANGLE(rect), NULL);
 
-    buffer = gst_video_overlay_rectangle_get_pixels_unscaled_argb(
-        rect,
-        &width, &height, &stride,
-        GST_VIDEO_OVERLAY_FORMAT_FLAG_NONE
-    );
-    if (!buffer)
-        return NULL;
-
     /* XXX: use gst_vaapi_image_format_from_video() */
 #if G_BYTE_ORDER == G_LITTLE_ENDIAN
     format = GST_VAAPI_IMAGE_BGRA;
 #else
     format = GST_VAAPI_IMAGE_ARGB;
 #endif
+    if (!gst_vaapi_display_has_subpicture_format(display, format, &hw_flags))
+        return NULL;
+    
+    flags = hw_flags & from_GstVideoOverlayFormatFlags(
+        gst_video_overlay_rectangle_get_flags(rect));
+
+    buffer = gst_video_overlay_rectangle_get_pixels_unscaled_argb(rect,
+        &width, &height, &stride, to_GstVideoOverlayFormatFlags(flags));
+    if (!buffer)
+        return NULL;
+
     image = gst_vaapi_image_new(display, format, width, height);
     if (!image)
         return NULL;
@@ -334,8 +339,16 @@ gst_vaapi_subpicture_new_from_overlay_rectangle(
         return NULL;
     }
 
-    subpicture = gst_vaapi_subpicture_new(image, 0);
+    subpicture = gst_vaapi_subpicture_new(image, flags);
     g_object_unref(image);
+    if (!subpicture)
+        return NULL;
+
+    if (flags & GST_VAAPI_SUBPICTURE_FLAG_GLOBAL_ALPHA) {
+        global_alpha = gst_video_overlay_rectangle_get_global_alpha(rect);
+        if (!gst_vaapi_subpicture_set_global_alpha(subpicture, global_alpha))
+            return NULL;
+    }
     return subpicture;
 }
 

gst-libs/gst/vaapi/gstvaapiutils.c

@@ -247,6 +247,46 @@ to_GstVaapiSubpictureFlags(guint va_flags)
     return flags;
 }
 
+/**
+ * from_GstVideoOverlayFormatFlags:
+ * @flags: the #GstVideoOverlayFormatFlags flags to translate
+ *
+ * Converts #GstVaapiSubpictureFlags to #GstVaapiSubpictureFlags.
+ *
+ * Return value: the #GstVaapiSubpictureFlags flags
+ */
+guint
+from_GstVideoOverlayFormatFlags(guint ovl_flags)
+{
+    guint flags = 0;
+
+    if (ovl_flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA)
+        flags |= GST_VAAPI_SUBPICTURE_FLAG_PREMULTIPLIED_ALPHA;
+    if (ovl_flags & GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA)
+        flags |= GST_VAAPI_SUBPICTURE_FLAG_GLOBAL_ALPHA;
+    return flags;
+}
+
+/**
+ * to_GstVideoOverlayFormatFlags:
+ * @flags: the #GstVaapiSubpictureFlags flags to translate
+ *
+ * Converts #GstVaapiSubpictureFlags to #GstVideoOverlayFormatFlags.
+ *
+ * Return value: the #GstVideoOverlayFormatFlags flags
+ */
+guint
+to_GstVideoOverlayFormatFlags(guint flags)
+{
+    guint ovl_flags = 0;
+
+    if (flags & GST_VAAPI_SUBPICTURE_FLAG_PREMULTIPLIED_ALPHA)
+        ovl_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA;
+    if (flags & GST_VAAPI_SUBPICTURE_FLAG_GLOBAL_ALPHA)
+        ovl_flags |= GST_VIDEO_OVERLAY_FORMAT_FLAG_GLOBAL_ALPHA;
+    return ovl_flags;
+}
+
 /**
  * from_GstVaapiSurfaceRenderFlags:
  * @flags: the #GstVaapiSurfaceRenderFlags

gst-libs/gst/vaapi/gstvaapiutils.h

@@ -81,6 +81,14 @@ G_GNUC_INTERNAL
 guint
 to_GstVaapiSubpictureFlags(guint va_flags);
 
+G_GNUC_INTERNAL
+guint
+from_GstVideoOverlayFormatFlags(guint ovl_flags);
+
+G_GNUC_INTERNAL
+guint
+to_GstVideoOverlayFormatFlags(guint flags);
+
 G_GNUC_INTERNAL
 guint
 from_GstVaapiSurfaceRenderFlags(guint flags);