qml6: add a mixer element

Can take multiple input streams and a qml scene and layout the input
videos inside the qml scene.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4609>

subprojects/gst-plugins-good/ext/qt6/gstplugin.cc

@@ -31,6 +31,7 @@ plugin_init (GstPlugin * plugin)
 
   ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin);
   ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin);
+  ret |= GST_ELEMENT_REGISTER (qml6glmixer, plugin);
   ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin);
 
   return ret;

subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.cc

@@ -0,0 +1,628 @@
+/*
+ * GStreamer
+ * Copyright (C) 2023 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * SECTION:gstqml6gmixer
+ *
+ * `qml6glmixer` provides a way to render an almost-arbitrary QML scene within
+ * GStreamer pipeline using the same OpenGL context that GStreamer uses
+ * internally.  This avoids attempting to share multiple OpenGL contexts
+ * avoiding increased synchronisation points and attempting to share an OpenGL
+ * context at runtime which some drivers do not like.  The Intel driver on
+ * Windows is a notable example of the last point.
+ *
+ * `qml6glmixer` will attempt to retrieve the windowing system display connection
+ * that Qt is using (#GstGLDisplay).  This may be different to any already
+ * existing window system display connection already in use in the pipeline for
+ * a number of reasons.  A couple of examples of this are:
+ *
+ * 1. Adding `qml6glmixer` to an already running pipeline
+ * 2. Not having any `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element
+ *    start up before any other OpenGL-based element in the pipeline.
+ *
+ * If one of these scenarios occurs, then there will be multiple OpenGL contexts
+ * in use in the pipeline.  This means that either the pipeline will fail to
+ * start up correctly, a downstream element may reject buffers, or a complete
+ * GPU->System memory->GPU transfer is performed for every buffer.
+ *
+ * The requirement to avoid this is that all elements share the same
+ * #GstGLDisplay object and as Qt cannot currently share an existing window
+ * system display connection, GStreamer must use the window system display
+ * connection provided by Qt.  This window system display connection can be
+ * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element or a
+ * `qmlglmixer element. The recommended usage is to have either elements
+ * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer) be the first to propagate
+ * the #GstGLDisplay for the entire pipeline to use by setting either element
+ * to the READY element state before any other OpenGL element in the pipeline.
+ *
+ * In the dynamically adding `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`)
+ * to a pipeline case, there are some considerations for ensuring that the
+ * window system display and OpenGL contexts are compatible with Qt.  When the
+ * `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element is added and
+ * brought up to READY, it will propagate it's own #GstGLDisplay using the
+ * #GstContext mechanism regardless of any existing #GstGLDisplay used by the
+ * pipeline previously.  In order for the new #GstGLDisplay to be used, the
+ * application must then set the provided #GstGLDisplay containing #GstContext
+ * on the pipeline.  This may effectively cause each OpenGL element to replace
+ * the window system display and also the OpenGL context it is using.  As such
+ * this process may take a significant amount of time and resources as objects
+ * are recreated in the new OpenGL context.
+ *
+ * All instances of `qml6glmixer`, `qml6glsink`, and `qml6gloverlay` will return
+ * the exact same #GstGLDisplay object while the pipeline is running regardless
+ * of whether any `qml6glmixer`, `qml6glsink`, or `qml6gloverlay` elements are
+ * added or removed from the pipeline.
+ *
+ * The Qml scene will run at configured output framerate.  The timestamps on the
+ * output buffers are used to drive the animation time.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstqt6elements.h"
+#include "gstqml6glmixer.h"
+#include "qt6glrenderer.h"
+#include "gstqt6glutility.h"
+
+#include <QtGui/QGuiApplication>
+
+#include <gst/gl/gl.h>
+#include <gst/gl/gstglfuncs.h>
+
+#define GST_CAT_DEFAULT gst_debug_qml6_gl_mixer
+GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
+
+enum
+{
+  PROP_PAD_0,
+  PROP_PAD_WIDGET,
+};
+
+struct _GstQml6GLMixerPad
+{
+  GstGLMixerPad parent;
+
+  QSharedPointer<Qt6GLVideoItemInterface> widget;
+};
+
+G_DEFINE_FINAL_TYPE (GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST_TYPE_GL_MIXER_PAD);
+
+static gboolean
+gst_qml6_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad *vagg_pad, GstVideoAggregator * vagg,
+    GstBuffer *buffer, GstVideoFrame * prepared_frame)
+{
+  GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (vagg_pad);
+
+  if (!GST_VIDEO_AGGREGATOR_PAD_CLASS (gst_qml6_gl_mixer_pad_parent_class)->prepare_frame (vagg_pad, vagg, buffer, prepared_frame))
+    return FALSE;
+
+  if (pad->widget) {
+    GstMemory *mem;
+    GstGLMemory *gl_mem;
+    GstCaps *in_caps;
+    GstGLContext *context;
+
+    in_caps = gst_video_info_to_caps (&vagg->info);
+    gst_caps_set_features_simple (in_caps, gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
+    pad->widget->setCaps (in_caps);
+    gst_clear_caps (&in_caps);
+
+    mem = gst_buffer_peek_memory (buffer, 0);
+    if (!gst_is_gl_memory (mem)) {
+      GST_ELEMENT_ERROR (vagg_pad, RESOURCE, NOT_FOUND,
+          (NULL), ("Input memory must be a GstGLMemory"));
+      return GST_FLOW_ERROR;
+    }
+    gl_mem = (GstGLMemory *) mem;
+    context = gst_gl_base_mixer_get_gl_context (GST_GL_BASE_MIXER (vagg));
+    if (!gst_gl_context_can_share (gl_mem->mem.context, context)) {
+      GST_WARNING_OBJECT (vagg_pad, "Cannot use the current input texture "
+          "(input buffer GL context %" GST_PTR_FORMAT " cannot share "
+          "resources with the configured OpenGL context %" GST_PTR_FORMAT ")",
+          gl_mem->mem.context, context);
+    } else {
+      pad->widget->setBuffer (buffer);
+    }
+  }
+
+  return TRUE;
+}
+
+static void
+gst_qml6_gl_mixer_pad_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object);
+
+  switch (prop_id) {
+    case PROP_PAD_WIDGET: {
+      Qt6GLVideoItem *qt_item = static_cast<Qt6GLVideoItem *> (g_value_get_pointer (value));
+      if (qt_item)
+        qml6_gl_mixer_pad->widget = qt_item->getInterface();
+      else
+        qml6_gl_mixer_pad->widget.clear();
+      break;
+    }
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_qml6_gl_mixer_pad_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * pspec)
+{
+  GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object);
+
+  switch (prop_id) {
+    case PROP_PAD_WIDGET:
+      /* This is not really safe - the app needs to be
+       * sure the widget is going to be kept alive or
+       * this can crash */
+      if (qml6_gl_mixer_pad->widget)
+        g_value_set_pointer (value, qml6_gl_mixer_pad->widget->videoItem());
+      else
+        g_value_set_pointer (value, NULL);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_qml6_gl_mixer_pad_finalize (GObject * object)
+{
+  GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (object);
+
+  pad->widget.clear();
+
+  G_OBJECT_CLASS (gst_qml6_gl_mixer_pad_parent_class)->finalize (object);
+}
+
+static void
+gst_qml6_gl_mixer_pad_class_init (GstQml6GLMixerPadClass * klass)
+{
+  GObjectClass *gobject_class = (GObjectClass *) klass;
+  GstVideoAggregatorPadClass *vagg_pad_class = (GstVideoAggregatorPadClass *) klass;
+
+  gobject_class->set_property = gst_qml6_gl_mixer_pad_set_property;
+  gobject_class->get_property = gst_qml6_gl_mixer_pad_get_property;
+  gobject_class->finalize = gst_qml6_gl_mixer_pad_finalize;
+
+  g_object_class_install_property (gobject_class, PROP_PAD_WIDGET,
+      g_param_spec_pointer ("widget", "QQuickItem",
+          "The QQuickItem to place the input video in the object hierarchy",
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+
+  vagg_pad_class->prepare_frame = gst_qml6_gl_mixer_pad_prepare_frame;
+}
+
+static void
+gst_qml6_gl_mixer_pad_init (GstQml6GLMixerPad * pad)
+{
+  pad->widget = QSharedPointer<Qt6GLVideoItemInterface>();
+}
+
+static void gst_qml6_gl_mixer_finalize (GObject * object);
+static void gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * param_spec);
+static void gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * param_spec);
+
+static gboolean gst_qml6_gl_mixer_process_buffers (GstGLMixer * btrans,
+    GstBuffer * outbuf);
+
+static gboolean gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer);
+static void gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer);
+
+static GstFlowReturn gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf);
+
+static gboolean gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps);
+
+static GstStateChangeReturn gst_qml6_gl_mixer_change_state (GstElement * element,
+    GstStateChange transition);
+
+enum
+{
+  PROP_0,
+  PROP_QML_SCENE,
+  PROP_ROOT_ITEM,
+};
+
+enum
+{
+  SIGNAL_0,
+  SIGNAL_QML_SCENE_INITIALIZED,
+  SIGNAL_QML_SCENE_DESTROYED,
+  LAST_SIGNAL
+};
+
+static guint gst_qml6_gl_mixer_signals[LAST_SIGNAL] = { 0 };
+
+static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
+    GST_PAD_SRC,
+    GST_PAD_ALWAYS,
+    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
+            "RGBA"))
+    );
+
+static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
+    GST_PAD_SINK,
+    GST_PAD_REQUEST,
+    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+        (GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
+            "RGBA"))
+    );
+
+struct _GstQml6GLMixer {
+  GstGLMixer parent;
+
+  gchar *qml_scene;
+
+  GstQt6QuickRenderer *renderer;
+  GstBuffer *outbuf;
+};
+
+#define gst_qml6_gl_mixer_parent_class parent_class
+G_DEFINE_FINAL_TYPE_WITH_CODE (GstQml6GLMixer, gst_qml6_gl_mixer,
+    GST_TYPE_GL_MIXER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
+        "qml6glmixer", 0, "Qt6 Video Mixer"));
+GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glmixer, "qml6glmixer",
+    GST_RANK_NONE, GST_TYPE_QML6_GL_MIXER, qt6_element_init (plugin));
+
+static void
+gst_qml6_gl_mixer_class_init (GstQml6GLMixerClass * klass)
+{
+  GObjectClass *gobject_class;
+  GstElementClass *gstelement_class;
+  GstAggregatorClass *agg_class;
+  GstVideoAggregatorClass *vagg_class;
+  GstGLBaseMixerClass *glbasemixer_class;
+  GstGLMixerClass *glmixer_class;
+  GstElementClass *element_class;
+
+  gobject_class = (GObjectClass *) klass;
+  gstelement_class = (GstElementClass *) klass;
+  glbasemixer_class = (GstGLBaseMixerClass *) klass;
+  glmixer_class = (GstGLMixerClass *) klass;
+  vagg_class = (GstVideoAggregatorClass *) klass;
+  agg_class = (GstAggregatorClass *) klass;
+  element_class = (GstElementClass *) klass;
+
+  gobject_class->set_property = gst_qml6_gl_mixer_set_property;
+  gobject_class->get_property = gst_qml6_gl_mixer_get_property;
+  gobject_class->finalize = gst_qml6_gl_mixer_finalize;
+
+  gst_element_class_set_metadata (gstelement_class, "Qt6 Video Mixer",
+      "Video/QML/Mixer", "A mixer that renders a QML scene",
+      "Matthew Waters <matthew@centricular.com>");
+
+  g_object_class_install_property (gobject_class, PROP_QML_SCENE,
+      g_param_spec_string ("qml-scene", "QML Scene",
+          "The contents of the QML scene", NULL,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
+  g_object_class_install_property (gobject_class, PROP_ROOT_ITEM,
+      g_param_spec_pointer ("root-item", "QQuickItem",
+          "The root QQuickItem from the qml-scene used to render",
+          (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
+
+  /**
+   * GstQmlGLMixer::qml-scene-initialized
+   * @element: the #GstQmlGLMixer
+   * @user_data: user provided data
+   */
+  gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED] =
+      g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass),
+      G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
+
+  /**
+   * GstQmlGLMixer::qml-scene-destroyed
+   * @element: the #GstQmlGLMixer
+   * @user_data: user provided data
+   */
+  gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED] =
+      g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass),
+      G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
+
+  glbasemixer_class->gl_start = gst_qml6_gl_mixer_gl_start;
+  glbasemixer_class->gl_stop = gst_qml6_gl_mixer_gl_stop;
+
+  glmixer_class->process_buffers = gst_qml6_gl_mixer_process_buffers;
+
+  vagg_class->create_output_buffer = gst_qml6_gl_mixer_create_output_buffer;
+
+  agg_class->negotiated_src_caps = gst_qml6_gl_mixer_negotiated_src_caps;
+
+  element_class->change_state = gst_qml6_gl_mixer_change_state;
+
+  gst_element_class_add_static_pad_template_with_gtype (element_class,
+      &src_factory, GST_TYPE_AGGREGATOR_PAD);
+  gst_element_class_add_static_pad_template_with_gtype (element_class,
+      &sink_factory, GST_TYPE_QML6_GL_MIXER_PAD);
+}
+
+static void
+gst_qml6_gl_mixer_init (GstQml6GLMixer * qml6_gl_mixer)
+{
+  qml6_gl_mixer->qml_scene = NULL;
+}
+
+static void
+gst_qml6_gl_mixer_finalize (GObject * object)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
+
+  g_free (qml6_gl_mixer->qml_scene);
+  qml6_gl_mixer->qml_scene = NULL;
+
+  G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
+static void
+gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
+
+  switch (prop_id) {
+    case PROP_QML_SCENE:
+      g_free (qml6_gl_mixer->qml_scene);
+      qml6_gl_mixer->qml_scene = g_value_dup_string (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * pspec)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
+
+  switch (prop_id) {
+    case PROP_QML_SCENE:
+      g_value_set_string (value, qml6_gl_mixer->qml_scene);
+      break;
+    case PROP_ROOT_ITEM:
+      GST_OBJECT_LOCK (qml6_gl_mixer);
+      if (qml6_gl_mixer->renderer) {
+        QQuickItem *root = qml6_gl_mixer->renderer->rootItem();
+        if (root)
+          g_value_set_pointer (value, root);
+        else
+          g_value_set_pointer (value, NULL);
+      } else {
+        g_value_set_pointer (value, NULL);
+      }
+      GST_OBJECT_UNLOCK (qml6_gl_mixer);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static gboolean
+gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (aggregator);
+  GstVideoInfo out_info;
+
+  if (!gst_video_info_from_caps (&out_info, out_caps))
+    return FALSE;
+
+  qml6_gl_mixer->renderer->setSize (GST_VIDEO_INFO_WIDTH (&out_info),
+      GST_VIDEO_INFO_HEIGHT (&out_info));
+  
+  return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (aggregator, out_caps);
+}
+
+static gboolean
+gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer);
+
+  QQuickItem *root;
+  GError *error = NULL;
+
+  GST_TRACE_OBJECT (bmixer, "using scene:\n%s", qml6_gl_mixer->qml_scene);
+
+  if (!qml6_gl_mixer->qml_scene || g_strcmp0 (qml6_gl_mixer->qml_scene, "") == 0) {
+    GST_ELEMENT_ERROR (bmixer, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL));
+    return FALSE;
+  }
+
+  if (!GST_GL_BASE_MIXER_CLASS (parent_class)->gl_start (bmixer))
+    return FALSE;
+
+  GST_OBJECT_LOCK (bmixer);
+  qml6_gl_mixer->renderer = new GstQt6QuickRenderer;
+  if (!qml6_gl_mixer->renderer->init (bmixer->context, &error)) {
+    GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
+        ("%s", error->message), (NULL));
+    delete qml6_gl_mixer->renderer;
+    qml6_gl_mixer->renderer = NULL;
+    GST_OBJECT_UNLOCK (bmixer);
+    return FALSE;
+  }
+
+  /* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */
+  if (!qml6_gl_mixer->renderer->setQmlScene (qml6_gl_mixer->qml_scene, &error)) {
+    GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
+        ("%s", error->message), (NULL));
+    goto fail_renderer;
+  }
+
+  root = qml6_gl_mixer->renderer->rootItem();
+  if (!root) {
+    GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
+        ("Qml scene does not have a root item"), (NULL));
+    goto fail_renderer;
+  }
+  GST_OBJECT_UNLOCK (bmixer);
+
+  g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item");
+  g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED], 0);
+
+  return TRUE;
+
+fail_renderer:
+  {
+    qml6_gl_mixer->renderer->cleanup();
+    delete qml6_gl_mixer->renderer;
+    qml6_gl_mixer->renderer = NULL;
+    GST_OBJECT_UNLOCK (bmixer);
+    return FALSE;
+  }
+}
+
+static void
+gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer);
+  GstQt6QuickRenderer *renderer = NULL;
+
+  /* notify before actually destroying anything */
+  GST_OBJECT_LOCK (qml6_gl_mixer);
+  if (qml6_gl_mixer->renderer)
+    renderer = qml6_gl_mixer->renderer;
+  qml6_gl_mixer->renderer = NULL;
+  GST_OBJECT_UNLOCK (qml6_gl_mixer);
+
+  g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED], 0);
+  g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item");
+
+  /* TODO: clear all pad buffers in the items?
+  if (qml6_gl_mixer->widget)
+    qml6_gl_mixer->widget->setBuffer (NULL);
+*/
+  if (renderer) {
+    renderer->cleanup();
+    delete renderer;
+  }
+
+  GST_GL_BASE_MIXER_CLASS (parent_class)->gl_stop (bmixer);
+}
+
+static GstFlowReturn
+gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf)
+{
+  *outbuf = gst_buffer_new();
+
+  return GST_FLOW_OK;
+}
+
+static gboolean
+qml6_gl_mixer_gl_callback (GstGLContext *context, GstQml6GLMixer * qml6_gl_mixer)
+{
+  GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (qml6_gl_mixer);
+  GstGLMemory *out_mem;
+
+  /* XXX: is this the correct ts to drive the animation */
+  out_mem = qml6_gl_mixer->renderer->generateOutput (GST_BUFFER_PTS (qml6_gl_mixer->outbuf));
+  if (!out_mem) {
+    GST_ERROR_OBJECT (qml6_gl_mixer, "Failed to generate output");
+    return FALSE;
+  }
+
+  gst_buffer_append_memory (qml6_gl_mixer->outbuf, (GstMemory *) out_mem);
+  gst_buffer_add_video_meta (qml6_gl_mixer->outbuf, (GstVideoFrameFlags) 0,
+      GST_VIDEO_INFO_FORMAT (&vagg->info),
+      GST_VIDEO_INFO_WIDTH (&vagg->info),
+      GST_VIDEO_INFO_HEIGHT (&vagg->info));
+
+  return TRUE;
+}
+
+static gboolean
+gst_qml6_gl_mixer_process_buffers (GstGLMixer * mix,
+    GstBuffer * outbuf)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (mix);
+  GstGLBaseMixer *bmix = GST_GL_BASE_MIXER (mix);
+  GstGLContext *context = gst_gl_base_mixer_get_gl_context (bmix);
+
+  qml6_gl_mixer->outbuf = outbuf;
+  gst_gl_context_thread_add (context,
+    (GstGLContextThreadFunc) qml6_gl_mixer_gl_callback, qml6_gl_mixer);
+  qml6_gl_mixer->outbuf = NULL;
+
+  gst_clear_object (&context);
+
+  return TRUE;
+}
+
+static GstStateChangeReturn
+gst_qml6_gl_mixer_change_state (GstElement * element,
+    GstStateChange transition)
+{
+  GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (element);
+  GstGLBaseMixer *bmixer = GST_GL_BASE_MIXER (element);
+  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
+
+  GST_DEBUG_OBJECT (element, "changing state: %s => %s",
+      gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
+      gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
+
+  switch (transition) {
+    case GST_STATE_CHANGE_NULL_TO_READY: {
+      QGuiApplication *app;
+      GstGLDisplay *display = NULL;
+
+      app = static_cast<QGuiApplication *> (QCoreApplication::instance ());
+      if (!app) {
+        GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
+            ("%s", "Failed to connect to Qt"),
+            ("%s", "Could not retrieve QGuiApplication instance"));
+        return GST_STATE_CHANGE_FAILURE;
+      }
+
+      display = gst_qml6_get_gl_display (FALSE);
+
+      if (display != bmixer->display)
+        /* always propagate. The application may need to choose between window
+         * system display connections */
+        gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_mixer), display);
+      gst_object_unref (display);
+      break;
+    }
+    default:
+      break;
+  }
+
+  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
+  if (ret == GST_STATE_CHANGE_FAILURE)
+    return ret;
+
+  switch (transition) {
+    default:
+      break;
+  }
+
+  return ret;
+
+}

subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.h

@@ -0,0 +1,40 @@
+/*
+ * GStreamer
+ * Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_QML6_GL_MIXER_H__
+#define __GST_QML6_GL_MIXER_H__
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/gl/gl.h>
+#include "qt6glrenderer.h"
+#include "qt6glitem.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_QML6_GL_MIXER_PAD (gst_qml6_gl_mixer_pad_get_type())
+G_DECLARE_FINAL_TYPE(GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST, QML6_GL_MIXER_PAD, GstGLMixerPad);
+
+#define GST_TYPE_QML6_GL_MIXER (gst_qml6_gl_mixer_get_type())
+G_DECLARE_FINAL_TYPE(GstQml6GLMixer, gst_qml6_gl_mixer, GST, QML6_GL_MIXER, GstGLMixer);
+
+G_END_DECLS
+
+#endif /* __GST_QML6_GL_MIXER_H__ */

subprojects/gst-plugins-good/ext/qt6/gstqml6gloverlay.cc

@@ -34,8 +34,8 @@
  * a number of reasons.  A couple of examples of this are:
  *
  * 1. Adding `qml6gloverlay` to an already running pipeline
- * 2. Not having any `qml6gloverlay` (or `qml6glsink`) element start up before any
+ * 2. Not having any `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element
- *    other OpenGL-based element in the pipeline.
+ *    start up before any other OpenGL-based element in the pipeline.
  *
  * If one of these scenarios occurs, then there will be multiple OpenGL contexts
  * in use in the pipeline.  This means that either the pipeline will fail to
@@ -46,29 +46,29 @@
  * #GstGLDisplay object and as Qt cannot currently share an existing window
  * system display connection, GStreamer must use the window system display
  * connection provided by Qt.  This window system display connection can be
- * retrieved by either a `qml6glsink` element or a `qml6gloverlay` element. The
+ * retrieved by either a `qml6glsink` element, a `qml6gloverlay`, or a
- * recommended usage is to have either element (`qml6glsink` or `qml6gloverlay`)
+ * `qml6glmixer` element. The recommended usage is to have either element
- * be the first to propagate the #GstGLDisplay for the entire pipeline to use by
+ * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer`) be the first to propagate
- * setting either element to the READY element state before any other OpenGL
+ * the #GstGLDisplay for the entire pipeline to use by setting either element
- * element in the pipeline.
+ * to the READY element state before any other OpenGL element in the pipeline.
  *
- * In a dynamically adding `qml6gloverlay` (or `qml6glsink`) to a pipeline case,
+ * In the dynamically adding `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`)
- * there are some considerations for ensuring that the window system display
+ * to a pipeline case, there are some considerations for ensuring that the
- * and OpenGL contexts are compatible with Qt.  When the `qml6gloverlay` (or
+ * window system display and OpenGL contexts are compatible with Qt.  When the
- * `qml6glsink`) element is added and brought up to READY, it will propagate it's
+ * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and
- * own #GstGLDisplay using the #GstContext mechanism regardless of any existing
+ * brought up to READY, it will propagate it's own #GstGLDisplay using the
- * #GstGLDisplay used by the pipeline previously.  In order for the new
+ * #GstContext mechanism regardless of any existing #GstGLDisplay used by the
- * #GstGLDisplay to be used, the application must then set the provided
+ * pipeline previously.  In order for the new #GstGLDisplay to be used, the
- * #GstGLDisplay containing #GstContext on the pipeline.  This may effectively
+ * application must then set the provided #GstGLDisplay containing #GstContext
- * cause each OpenGL element to replace the window system display and also the
+ * on the pipeline.  This may effectively cause each OpenGL element to replace
- * OpenGL context it is using.  As such this process may take a significant
+ * the window system display and also the OpenGL context it is using.  As such
- * amount of time and resources as objects are recreated in the new OpenGL
+ * this process may take a significant amount of time and resources as objects
- * context.
+ * are recreated in the new OpenGL context.
  *
- * All instances of `qml6gloverlay` and `qml6glsink` will return the exact same
+ * All instances of `qml6gloverlay`, `qml6glsink`, and `qml6glmixer` will
- * #GstGLDisplay object while the pipeline is running regardless of whether
+ * return the exact same #GstGLDisplay object while the pipeline is running
- * any `qml6gloverlay` or `qml6glsink` elements are added or removed from the
+ * regardless of whether any `qml6gloverlay` or `qml6glsink` elements are
- * pipeline.
+ * added or removed from the pipeline.
  *
  * The Qml scene will run at the pace of incoming buffers.  One input buffer
  * will cause a render of one output buffer.  The timestamps on the input

subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc

@@ -43,29 +43,30 @@
  * #GstGLDisplay object and as Qt cannot currently share an existing window
  * system display connection, GStreamer must use the window system display
  * connection provided by Qt.  This window system display connection can be
- * retrieved by either a qmlglsink element or a qmlgloverlay element. The
+ * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element, or a
- * recommended usage is to have either element (qmlglsink or qmlgloverlay)
+ * `qml6glmixer` element. The recommended usage is to have either element
- * be the first to propagate the #GstGLDisplay for the entire pipeline to use by
+ * (`qml6glsink`, or `qml6gloverlay`, or `qml6glmixer`) be the first to
- * setting either element to the READY element state before any other OpenGL
+ * propagate the #GstGLDisplay for the entire pipeline to use by setting either
- * element in the pipeline.
+ * element to the READY element state before any other OpenGL element in the
- *
- * In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case,
- * there are some considerations for ensuring that the window system display
- * and OpenGL contexts are compatible with Qt.  When the qmlgloverlay (or
- * qmlglsink) element is added and brought up to READY, it will propagate it's
- * own #GstGLDisplay using the #GstContext mechanism regardless of any existing
- * #GstGLDisplay used by the pipeline previously.  In order for the new
- * #GstGLDisplay to be used, the application must then set the provided
- * #GstGLDisplay containing #GstContext on the pipeline.  This may effectively
- * cause each OpenGL element to replace the window system display and also the
- * OpenGL context it is using.  As such this process may take a significant
- * amount of time and resources as objects are recreated in the new OpenGL
- * context.
- *
- * All instances of qmlglsink and qmlgloverlay will return the exact same
- * #GstGLDisplay object while the pipeline is running regardless of whether
- * any qmlglsink or qmlgloverlay elements are added or removed from the
  * pipeline.
+ *
+ * In the dynamically adding `qml6glsink` (or `qml6gloverlay`, or `qml6glmixer`)
+ * to a pipeline case, there are some considerations for ensuring that the
+ * window system display and OpenGL contexts are compatible with Qt.  When the
+ * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and
+ * brought up to READY, it will propagate it's own #GstGLDisplay using the
+ * #GstContext mechanism regardless of any existing #GstGLDisplay used by the
+ * pipeline previously.  In order for the new #GstGLDisplay to be used, the
+ * application must then set the provided #GstGLDisplay containing #GstContext
+ * on the pipeline.  This may effectively cause each OpenGL element to replace
+ * the window system display and also the OpenGL context it is using.  As such
+ * this process may take a significant amount of time and resources as objects
+ * are recreated in the new OpenGL context.
+ *
+ * All instances of `qml6glsink`, `qml6gloverlay`, and `qml6glmixer` will
+ * return the exact same #GstGLDisplay object while the pipeline is running
+ * regardless of whether any `qml6glsink` or `qml6gloverlay` elements are
+ * added or removed from the pipeline.
  */
 
 #ifdef HAVE_CONFIG_H

subprojects/gst-plugins-good/ext/qt6/gstqt6elements.h

@@ -28,6 +28,7 @@ void qt6_element_init (GstPlugin * plugin);
 
 GST_ELEMENT_REGISTER_DECLARE (qml6glsink);
 GST_ELEMENT_REGISTER_DECLARE (qml6glsrc);
+GST_ELEMENT_REGISTER_DECLARE (qml6glmixer);
 GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay);
 
 G_END_DECLS

subprojects/gst-plugins-good/ext/qt6/meson.build

@@ -5,6 +5,7 @@ sources = [
   'gstqt6glutility.cc',
   'gstqml6glsink.cc',
   'gstqml6glsrc.cc',
+  'gstqml6glmixer.cc',
   'gstqml6gloverlay.cc',
   'qt6glitem.cc',
   'qt6glwindow.cc',

subprojects/gst-plugins-good/tests/examples/qt6/meson.build

@@ -17,3 +17,4 @@ endif
 subdir('qmlsink')
 subdir('qmlsrc')
 subdir('qmloverlay')
+subdir('qmlmixer')

subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.cpp

@@ -0,0 +1,127 @@
+#include <QApplication>
+#include <QQmlApplicationEngine>
+#include <QQuickWindow>
+#include <QQuickItem>
+#include <QRunnable>
+#include <QDirIterator>
+#include <gst/gst.h>
+
+class SetPlaying : public QRunnable
+{
+public:
+  SetPlaying(GstElement *);
+  ~SetPlaying();
+
+  void run ();
+
+private:
+  GstElement * pipeline_;
+};
+
+SetPlaying::SetPlaying (GstElement * pipeline)
+{
+  this->pipeline_ = pipeline ? static_cast<GstElement *> (gst_object_ref (pipeline)) : NULL;
+}
+
+SetPlaying::~SetPlaying ()
+{
+  if (this->pipeline_)
+    gst_object_unref (this->pipeline_);
+}
+
+void
+SetPlaying::run ()
+{
+  if (this->pipeline_)
+    gst_element_set_state (this->pipeline_, GST_STATE_PLAYING);
+}
+
+static void
+on_mixer_scene_initialized (GstElement * mixer, gpointer unused)
+{
+  QQuickItem *rootObject;
+  GST_INFO ("scene initialized");
+  g_object_get (mixer, "root-item", &rootObject, NULL);
+
+  QQuickItem *videoItem0 = rootObject->findChild<QQuickItem *> ("inputVideoItem0");
+  GstPad *sink0 = gst_element_get_static_pad (mixer, "sink_0");
+  g_object_set (sink0, "widget", videoItem0, NULL);
+  gst_clear_object (&sink0);
+
+  QQuickItem *videoItem1 = rootObject->findChild<QQuickItem *> ("inputVideoItem1");
+  GstPad *sink1 = gst_element_get_static_pad (mixer, "sink_1");
+  g_object_set (sink1, "widget", videoItem1, NULL);
+  gst_clear_object (&sink1);
+}
+
+int main(int argc, char *argv[])
+{
+  int ret;
+
+  gst_init (&argc, &argv);
+
+  {
+    QGuiApplication app(argc, argv);
+
+    QQuickWindow::setGraphicsApi(QSGRendererInterface::OpenGL);
+
+    GstElement *pipeline = gst_pipeline_new (NULL);
+    GstElement *src0 = gst_element_factory_make ("videotestsrc", NULL);
+    GstElement *glupload0 = gst_element_factory_make ("glupload", NULL);
+    GstElement *src1 = gst_element_factory_make ("videotestsrc", NULL);
+    gst_util_set_object_arg ((GObject *) src1, "pattern", "ball");
+    GstElement *glupload1 = gst_element_factory_make ("glupload", NULL);
+    /* the plugin must be loaded before loading the qml file to register the
+     * GstGLVideoItem qml item */
+    GstElement *mixer = gst_element_factory_make ("qml6glmixer", NULL);
+    GstElement *sink = gst_element_factory_make ("qml6glsink", NULL);
+
+    g_assert (src0 && glupload0 && mixer && sink);
+
+    gst_bin_add_many (GST_BIN (pipeline), src0, glupload0, src1, glupload1, mixer, sink, NULL);
+    gst_element_link_many (src0, glupload0, mixer, sink, NULL);
+    gst_element_link_many (src1, glupload1, mixer, NULL);
+
+    /* load qmlglsink output */
+    QQmlApplicationEngine engine;
+    engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
+
+    QQuickItem *videoItem;
+    QQuickWindow *rootObject;
+
+    /* find and set the videoItem on the sink */
+    rootObject = static_cast<QQuickWindow *> (engine.rootObjects().first());
+    videoItem = rootObject->findChild<QQuickItem *> ("videoItem");
+    g_assert (videoItem);
+    g_object_set(sink, "widget", videoItem, NULL);
+
+    QDirIterator it(":", QDirIterator::Subdirectories);
+    while (it.hasNext()) {
+        qDebug() << it.next();
+    }
+
+    QFile f(":/mixer.qml");
+    if(!f.open(QIODevice::ReadOnly)) {
+        qWarning() << "error: " << f.errorString();
+        return 1;
+    }
+    QByteArray overlay_scene = f.readAll();
+    qDebug() << overlay_scene;
+
+    /* load qmlgloverlay contents */
+    g_signal_connect (mixer, "qml-scene-initialized", G_CALLBACK (on_mixer_scene_initialized), NULL);
+    g_object_set (mixer, "qml-scene", overlay_scene.data(), NULL);
+
+    rootObject->scheduleRenderJob (new SetPlaying (pipeline),
+        QQuickWindow::BeforeSynchronizingStage);
+
+    ret = app.exec();
+
+    gst_element_set_state (pipeline, GST_STATE_NULL);
+    gst_object_unref (pipeline);
+  }
+
+  gst_deinit ();
+
+  return ret;
+}

subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.qml

@@ -0,0 +1,38 @@
+import QtQuick 6.0
+import QtQuick.Controls 6.0
+import QtQuick.Dialogs 6.0
+import QtQuick.Window 6.0
+
+import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
+
+ApplicationWindow {
+    id: window
+    visible: true
+    width: 640
+    height: 480
+    x: 30
+    y: 30
+    color: "black"
+
+    Item {
+        anchors.fill: parent
+
+        GstGLQt6VideoItem {
+            id: video
+            objectName: "videoItem"
+            anchors.centerIn: parent
+            width: parent.width
+            height: parent.height
+        }
+
+        Text {
+            anchors.bottom: parent.bottom
+            anchors.horizontalCenter: parent.horizontalCenter
+            text: "qmlglsink text"
+            font.pointSize: 20
+            color: "yellow"
+            style: Text.Outline
+            styleColor: "blue"
+        }
+    }
+}

subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/meson.build

@@ -0,0 +1,11 @@
+sources = [
+  'main.cpp',
+]
+
+qt_preprocessed = qt6_mod.preprocess(qresources : 'qmlmixer.qrc')
+executable('qml6glmixer', sources, qt_preprocessed,
+    dependencies : [gst_dep, qt6qml_example_deps],
+    override_options : ['cpp_std=c++17'],
+    c_args : gst_plugins_good_args,
+    include_directories : [configinc],
+    install: false)

subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/mixer.qml

@@ -0,0 +1,67 @@
+import QtQuick 6.0
+
+import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
+
+Item {
+    /* render upside down for GStreamer */
+    transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 }
+
+    GstGLQt6VideoItem {
+        id: video0
+        objectName: "inputVideoItem0"
+        anchors.left: parent.left
+        anchors.right: parent.horizontalCenter
+        width: parent.width / 2
+        height: parent.height
+    }
+
+    GstGLQt6VideoItem {
+        id: video1
+        objectName: "inputVideoItem1"
+        anchors.left: parent.horizontalCenter
+        anchors.right: parent.right
+        width: parent.width / 2
+        height: parent.height
+    }
+
+    Text {
+        id: rotatingText
+        anchors.centerIn: parent
+        text: "Qt Quick\nrendered to\na texture"
+        font.pointSize: 20
+        color: "black"
+        style: Text.Outline
+        styleColor: "white"
+
+        RotationAnimator {
+            target: rotatingText;
+            from: 0;
+            to: 360;
+            duration: 5000
+            running: true
+            loops: Animation.Infinite
+        }
+    }
+
+    Text {
+        property int elapsedTime: 0
+
+        id: time
+        anchors.top: rotatingText.bottom
+        anchors.horizontalCenter: rotatingText.horizontalCenter
+        font.pointSize: 12
+        style: Text.Outline
+        styleColor: "black"
+        color: "white"
+
+        Timer {
+            interval: 1000
+            running: true
+            repeat: true
+            onTriggered: {
+                parent.elapsedTime += interval / 1000
+                parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds"
+            }
+        }
+    }
+}

subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/qmlmixer.qrc

@@ -0,0 +1,6 @@
+<RCC>
+    <qresource prefix="/">
+        <file>main.qml</file>
+        <file>mixer.qml</file>
+    </qresource>
+</RCC>