v4l2codecs: add wrappers for alpha decode

codecalpha is a new plugin introduced to support VP8/VP9 alpha as
defined in the WebM and Matroska specifications. It splits the stream
into two streams, one for the alpha and one for the actual content,
then it decodes them separately with vpxdec and finally combine the
results as A420 or AV12 (i.e. YUV + an extra alpha plane).

The workflow above is setup by means of a bin, gstcodecalphabin.

This patch simulates the same workflow into the v4l2codecs namespace,
thus using the new v4l2 stateless decoders for hardware acceleration.

This is so we can register the new alpha decode elements only if the
hardware produces formats we support, i.e. I420 or NV12 for now.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2272>

sys/v4l2codecs/gstv4l2codecalphadecodebin.c

@@ -0,0 +1,226 @@
+/* GStreamer
+ * Copyright (C) <2021> Collabora Ltd.
+ *   Author: Daniel Almeida <daniel.almeida@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <gst/pbutils/pbutils.h>
+
+#include "gstv4l2codecalphadecodebin.h"
+#include "gstv4l2decoder.h"
+
+GST_DEBUG_CATEGORY_STATIC (v4l2_codecalphadecodebin_debug);
+#define GST_CAT_DEFAULT (v4l2_codecalphadecodebin_debug)
+
+typedef struct
+{
+  GstBin parent;
+
+  gboolean constructed;
+  const gchar *missing_element;
+} GstV4l2CodecAlphaDecodeBinPrivate;
+
+#define gst_v4l2_codec_alpha_decode_bin_parent_class parent_class
+G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstV4l2CodecAlphaDecodeBin,
+    gst_v4l2_codec_alpha_decode_bin, GST_TYPE_BIN,
+    G_ADD_PRIVATE (GstV4l2CodecAlphaDecodeBin);
+    GST_DEBUG_CATEGORY_INIT (v4l2_codecalphadecodebin_debug,
+        "v4l2codecs-alphadecodebin", 0, "V4L2 stateless alpha decode bin"));
+
+
+static GstStaticPadTemplate gst_alpha_decode_bin_src_template =
+GST_STATIC_PAD_TEMPLATE ("src",
+    GST_PAD_SRC,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS ("ANY")
+    );
+
+static gboolean
+gst_v4l2_codec_alpha_decode_bin_open (GstV4l2CodecAlphaDecodeBin * self)
+{
+  GstV4l2CodecAlphaDecodeBinPrivate *priv =
+      gst_v4l2_codec_alpha_decode_bin_get_instance_private (self);
+
+  if (priv->missing_element) {
+    gst_element_post_message (GST_ELEMENT (self),
+        gst_missing_element_message_new (GST_ELEMENT (self),
+            priv->missing_element));
+  } else if (!priv->constructed) {
+    GST_ELEMENT_ERROR (self, CORE, FAILED,
+        ("Failed to construct alpha decoder pipeline."), (NULL));
+  }
+
+  return priv->constructed;
+}
+
+static GstStateChangeReturn
+gst_v4l2_codec_alpha_decode_bin_change_state (GstElement * element,
+    GstStateChange transition)
+{
+  GstV4l2CodecAlphaDecodeBin *self = GST_V4L2_CODEC_ALPHA_DECODE_BIN (element);
+
+  switch (transition) {
+    case GST_STATE_CHANGE_NULL_TO_READY:
+      if (!gst_v4l2_codec_alpha_decode_bin_open (self))
+        return GST_STATE_CHANGE_FAILURE;
+      break;
+    default:
+      break;
+  }
+
+  return GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
+}
+
+static void
+gst_v4l2_codec_alpha_decode_bin_constructed (GObject * obj)
+{
+  GstV4l2CodecAlphaDecodeBin *self = GST_V4L2_CODEC_ALPHA_DECODE_BIN (obj);
+  GstV4l2CodecAlphaDecodeBinPrivate *priv =
+      gst_v4l2_codec_alpha_decode_bin_get_instance_private (self);
+  GstV4l2CodecAlphaDecodeBinClass *klass =
+      GST_V4L2_CODEC_ALPHA_DECODE_BIN_GET_CLASS (self);
+  GstPad *src_gpad, *sink_gpad;
+  GstPad *src_pad = NULL, *sink_pad = NULL;
+  GstElement *alphademux = NULL;
+  GstElement *queue = NULL;
+  GstElement *alpha_queue = NULL;
+  GstElement *decoder = NULL;
+  GstElement *alpha_decoder = NULL;
+  GstElement *alphacombine = NULL;
+
+  /* setup ghost pads */
+  sink_gpad = gst_ghost_pad_new_no_target_from_template ("sink",
+      gst_element_class_get_pad_template (GST_ELEMENT_CLASS (klass), "sink"));
+  gst_element_add_pad (GST_ELEMENT (self), sink_gpad);
+
+  src_gpad = gst_ghost_pad_new_no_target_from_template ("src",
+      gst_element_class_get_pad_template (GST_ELEMENT_CLASS (klass), "src"));
+  gst_element_add_pad (GST_ELEMENT (self), src_gpad);
+
+  /* create elements */
+  alphademux = gst_element_factory_make ("codecalphademux", NULL);
+  if (!alphademux) {
+    priv->missing_element = "codecalphademux";
+    goto cleanup;
+  }
+
+  queue = gst_element_factory_make ("queue", NULL);
+  alpha_queue = gst_element_factory_make ("queue", NULL);
+  if (!queue || !alpha_queue) {
+    priv->missing_element = "queue";
+    goto cleanup;
+  }
+
+  decoder = gst_element_factory_make (klass->decoder_name, "maindec");
+  if (!decoder) {
+    priv->missing_element = klass->decoder_name;
+    goto cleanup;
+  }
+
+  alpha_decoder = gst_element_factory_make (klass->decoder_name, "alphadec");
+  if (!alpha_decoder) {
+    priv->missing_element = klass->decoder_name;
+    goto cleanup;
+  }
+
+  alphacombine = gst_element_factory_make ("alphacombine", NULL);
+  if (!alphacombine) {
+    priv->missing_element = "alphacombine";
+    goto cleanup;
+  }
+
+  gst_bin_add_many (GST_BIN (self), alphademux, queue, alpha_queue, decoder,
+      alpha_decoder, alphacombine, NULL);
+
+  /* link elements */
+  sink_pad = gst_element_get_static_pad (alphademux, "sink");
+  gst_ghost_pad_set_target (GST_GHOST_PAD (sink_gpad), sink_pad);
+  gst_clear_object (&sink_pad);
+
+  gst_element_link_pads (alphademux, "src", queue, "sink");
+  gst_element_link_pads (queue, "src", decoder, "sink");
+  gst_element_link_pads (decoder, "src", alphacombine, "sink");
+
+  gst_element_link_pads (alphademux, "alpha", alpha_queue, "sink");
+  gst_element_link_pads (alpha_queue, "src", alpha_decoder, "sink");
+  gst_element_link_pads (alpha_decoder, "src", alphacombine, "alpha");
+
+  src_pad = gst_element_get_static_pad (alphacombine, "src");
+  gst_ghost_pad_set_target (GST_GHOST_PAD (src_gpad), src_pad);
+  gst_object_unref (src_pad);
+
+  g_object_set (queue, "max-size-bytes", 0, "max-size-time", 0,
+      "max-size-buffers", 1, NULL);
+  g_object_set (alpha_queue, "max-size-bytes", 0, "max-size-time", 0,
+      "max-size-buffers", 1, NULL);
+
+  /* signal success, we will handle this in NULL->READY transition */
+  priv->constructed = TRUE;
+  return;
+
+cleanup:
+  gst_clear_object (&alphademux);
+  gst_clear_object (&queue);
+  gst_clear_object (&alpha_queue);
+  gst_clear_object (&decoder);
+  gst_clear_object (&alpha_decoder);
+  gst_clear_object (&alphacombine);
+
+  G_OBJECT_CLASS (parent_class)->constructed (obj);
+}
+
+static void
+gst_v4l2_codec_alpha_decode_bin_class_init (GstV4l2CodecAlphaDecodeBinClass *
+    klass)
+{
+  GstElementClass *element_class = (GstElementClass *) klass;
+  GObjectClass *obj_class = (GObjectClass *) klass;
+
+  /* This is needed to access the subclass class instance, otherwise we cannot
+   * read the class parameters */
+  obj_class->constructed = gst_v4l2_codec_alpha_decode_bin_constructed;
+
+  gst_element_class_add_static_pad_template (element_class,
+      &gst_alpha_decode_bin_src_template);
+  element_class->change_state =
+      GST_DEBUG_FUNCPTR (gst_v4l2_codec_alpha_decode_bin_change_state);
+
+  /* let's make the doc generator happy */
+  gst_type_mark_as_plugin_api (GST_TYPE_V4L2_CODEC_ALPHA_DECODE_BIN, 0);
+}
+
+static void
+gst_v4l2_codec_alpha_decode_bin_init (GstV4l2CodecAlphaDecodeBin * self)
+{
+}
+
+void
+gst_v4l2_codec_alpha_decode_bin_register (GstPlugin * plugin,
+    GClassInitFunc class_init, gconstpointer class_data,
+    const gchar * element_name_tmpl, GstV4l2CodecDevice * device, guint rank)
+{
+  /* TODO check that we have compatible src format */
+
+  gst_v4l2_decoder_register (plugin,
+      GST_TYPE_V4L2_CODEC_ALPHA_DECODE_BIN, class_init, class_data, NULL,
+      element_name_tmpl, device,
+      rank + GST_V4L2_CODEC_ALPHA_DECODE_BIN_RANK_OFFSET, NULL);
+}

sys/v4l2codecs/gstv4l2codecalphadecodebin.h

@@ -0,0 +1,57 @@
+/* GStreamer
+ * Copyright (C) <2021> Collabora Ltd.
+ *   Author: Daniel Almeida <daniel.almeida@collabora.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_V4L2_CODEC_ALPHA_DECODE_BIN_H__
+#define __GST_V4L2_CODEC_ALPHA_DECODE_BIN_H__
+
+#include <gst/gst.h>
+#include <gstv4l2decoder.h>
+
+/* When wrapping, use the original rank plus this offset. The ad-hoc rules is
+ * that hardware implementation will use PRIMARY+1 or +2 to override the
+ * software decoder, so the offset must be large enough to jump over those.
+ * This should also be small enough so that a marginal (64) or secondary
+ * wrapper does not cross the PRIMARY line.
+ */
+#define GST_V4L2_CODEC_ALPHA_DECODE_BIN_RANK_OFFSET 10
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_V4L2_CODEC_ALPHA_DECODE_BIN (gst_v4l2_codec_alpha_decode_bin_get_type())
+G_DECLARE_DERIVABLE_TYPE (GstV4l2CodecAlphaDecodeBin,
+    gst_v4l2_codec_alpha_decode_bin, GST, V4L2_CODEC_ALPHA_DECODE_BIN, GstBin);
+
+struct _GstV4l2CodecAlphaDecodeBinClass
+{
+  GstBinClass parent_class;
+  gchar *decoder_name;
+};
+
+void gst_v4l2_codec_alpha_decode_bin_register (GstPlugin * plugin,
+                                               GClassInitFunc class_init,
+                                               gconstpointer class_data,
+                                               const gchar * element_name_tmpl, 
+                                               GstV4l2CodecDevice * device,
+                                               guint rank);
+
+
+
+G_END_DECLS
+#endif /* __GST_V4L2_CODEC_ALPHA_DECODE_BIN_H__ */

sys/v4l2codecs/gstv4l2codech264dec.c

@@ -1404,8 +1404,10 @@ void
 gst_v4l2_codec_h264_dec_register (GstPlugin * plugin,
     GstV4l2CodecDevice * device, guint rank)
 {
-  gst_v4l2_decoder_register (plugin, GST_TYPE_V4L2_CODEC_H264_DEC,
+  gst_v4l2_decoder_register (plugin,
+      GST_TYPE_V4L2_CODEC_H264_DEC,
       (GClassInitFunc) gst_v4l2_codec_h264_dec_subclass_init,
+      gst_mini_object_ref (GST_MINI_OBJECT (device)),
       (GInstanceInitFunc) gst_v4l2_codec_h264_dec_subinit,
-      "v4l2sl%sh264dec", device, rank);
+      "v4l2sl%sh264dec", device, rank, NULL);
 }

sys/v4l2codecs/gstv4l2codecvp8dec.c

@@ -22,6 +22,7 @@
 #endif
 
 #include "gstv4l2codecallocator.h"
+#include "gstv4l2codecalphadecodebin.h"
 #include "gstv4l2codecpool.h"
 #include "gstv4l2codecvp8dec.h"
 #include "linux/vp8-ctrls.h"
@@ -41,6 +42,12 @@ GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SINK_NAME,
     GST_STATIC_CAPS ("video/x-vp8")
     );
 
+static GstStaticPadTemplate alpha_template =
+GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SINK_NAME,
+    GST_PAD_SINK, GST_PAD_ALWAYS,
+    GST_STATIC_CAPS ("video/x-vp8, codec-alpha = (boolean) true")
+    );
+
 static GstStaticPadTemplate src_template =
 GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SRC_NAME,
     GST_PAD_SRC, GST_PAD_ALWAYS,
@@ -848,12 +855,37 @@ gst_v4l2_codec_vp8_dec_subclass_init (GstV4l2CodecVp8DecClass * klass,
   gst_v4l2_decoder_install_properties (gobject_class, PROP_LAST, device);
 }
 
+static void gst_v4l2_codec_vp8_alpha_decode_bin_subclass_init
+    (GstV4l2CodecAlphaDecodeBinClass * klass, gchar * decoder_name)
+{
+  GstV4l2CodecAlphaDecodeBinClass *adbin_class =
+      (GstV4l2CodecAlphaDecodeBinClass *) klass;
+  GstElementClass *element_class = (GstElementClass *) klass;
+
+  adbin_class->decoder_name = decoder_name;
+  gst_element_class_add_static_pad_template (element_class, &alpha_template);
+
+  gst_element_class_set_static_metadata (element_class,
+      "VP8 Alpha Decoder", "Codec/Decoder/Video",
+      "Wrapper bin to decode VP8 with alpha stream.",
+      "Daniel Almeida <daniel.almeida@collabora.com>");
+}
+
 void
 gst_v4l2_codec_vp8_dec_register (GstPlugin * plugin,
     GstV4l2CodecDevice * device, guint rank)
 {
+  gchar *element_name;
+
   gst_v4l2_decoder_register (plugin, GST_TYPE_V4L2_CODEC_VP8_DEC,
       (GClassInitFunc) gst_v4l2_codec_vp8_dec_subclass_init,
+      gst_mini_object_ref (GST_MINI_OBJECT (device)),
       (GInstanceInitFunc) gst_v4l2_codec_vp8_dec_subinit,
-      "v4l2sl%svp8dec", device, rank);
+      "v4l2sl%svp8dec", device, rank, &element_name);
+
+  if (element_name) {
+    gst_v4l2_codec_alpha_decode_bin_register (plugin,
+        (GClassInitFunc) gst_v4l2_codec_vp8_alpha_decode_bin_subclass_init,
+        element_name, "v4l2slvp8%salphadecodebin", device, rank);
+  }
 }

sys/v4l2codecs/gstv4l2decoder.c

@@ -813,10 +813,27 @@ gst_v4l2_decoder_get_property (GObject * object, guint prop_id,
   }
 }
 
+/**
+ * gst_v4l2_decoder_register:
+ * @plugin: a #GstPlugin
+ * @dec_type: A #GType for the codec
+ * @class_init: The #GClassInitFunc for #dec_type
+ * @instance_init: The #GInstanceInitFunc for #dec_type
+ * @element_name_tmpl: A string to use for the first codec found and as a template for the next ones.
+ * @device: (transfer full) A #GstV4l2CodecDevice
+ * @rank: The rank to use for the element
+ * @class_data: (nullable) (transfer full) A #gpointer to pass as class_data, set to @device if null
+ * @element_name (nullable) (out) Sets the pointer to the new element name
+ *
+ * Registers a decoder element as a subtype of @dec_type for @plugin.
+ * Will create a different sub_types for each subsequent @decoder of the
+ * same type.
+ */
 void
 gst_v4l2_decoder_register (GstPlugin * plugin,
-    GType dec_type, GClassInitFunc class_init, GInstanceInitFunc instance_init,
-    const gchar * element_name_tmpl, GstV4l2CodecDevice * device, guint rank)
+    GType dec_type, GClassInitFunc class_init, gconstpointer class_data,
+    GInstanceInitFunc instance_init, const gchar * element_name_tmpl,
+    GstV4l2CodecDevice * device, guint rank, gchar ** element_name)
 {
   GTypeQuery type_query;
   GTypeInfo type_info = { 0, };
@@ -828,9 +845,11 @@ gst_v4l2_decoder_register (GstPlugin * plugin,
   type_info.class_size = type_query.class_size;
   type_info.instance_size = type_query.instance_size;
   type_info.class_init = class_init;
-  type_info.class_data = gst_mini_object_ref (GST_MINI_OBJECT (device));
+  type_info.class_data = class_data;
   type_info.instance_init = instance_init;
-  GST_MINI_OBJECT_FLAG_SET (device, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
+
+  if (class_data == device)
+    GST_MINI_OBJECT_FLAG_SET (device, GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED);
 
   /* The first decoder to be registered should use a constant name, like
    * v4l2slvp8dec, for any additional decoders, we create unique names. Decoder
@@ -848,10 +867,16 @@ gst_v4l2_decoder_register (GstPlugin * plugin,
 
   subtype = g_type_register_static (dec_type, type_name, &type_info, 0);
 
-  if (!gst_element_register (plugin, type_name, rank, subtype))
+  if (!gst_element_register (plugin, type_name, rank, subtype)) {
     GST_WARNING ("Failed to register plugin '%s'", type_name);
+    g_free (type_name);
+    type_name = NULL;
+  }
 
-  g_free (type_name);
+  if (element_name)
+    *element_name = type_name;
+  else
+    g_free (type_name);
 }
 
 /*

sys/v4l2codecs/gstv4l2decoder.h

@@ -100,10 +100,12 @@ void              gst_v4l2_decoder_get_property (GObject * object, guint prop_id
 void              gst_v4l2_decoder_register (GstPlugin * plugin,
                                              GType dec_type,
                                              GClassInitFunc class_init,
+                                             gconstpointer class_data,
                                              GInstanceInitFunc instance_init,
                                              const gchar *element_name_tmpl,
                                              GstV4l2CodecDevice * device,
-                                             guint rank);
+                                             guint rank,
+                                             gchar ** element_name);
 
 GstV4l2Request   *gst_v4l2_decoder_alloc_request (GstV4l2Decoder * self,
                                                   guint32 frame_num,

sys/v4l2codecs/meson.build

@@ -7,6 +7,7 @@ v4l2codecs_sources = [
   'gstv4l2codecvp8dec.c',
   'gstv4l2decoder.c',
   'gstv4l2format.c',
+  'gstv4l2codecalphadecodebin.c',
 ]
 
 libgudev_dep = dependency('gudev-1.0', required: get_option('v4l2codecs'))
@@ -37,7 +38,8 @@ if have_v4l2 and libgudev_dep.found()
     c_args : gst_plugins_bad_args,
     cpp_args: gst_plugins_bad_args,
     include_directories : [configinc],
-    dependencies : [gstbase_dep, gstcodecs_dep, gstallocators_dep, libgudev_dep],
+    dependencies : [gstbase_dep, gstcodecs_dep, gstallocators_dep, libgudev_dep,
+                   gstpbutils_dep,],
     install : true,
     install_dir : plugins_install_dir,
   )