From 3f4bfa097a6d8e381f824e350984aed2f89175f7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 12 May 2023 13:49:20 +1000 Subject: [PATCH] qml6: add a mixer element Can take multiple input streams and a qml scene and layout the input videos inside the qml scene. Part-of: --- .../gst-plugins-good/ext/qt6/gstplugin.cc | 1 + .../ext/qt6/gstqml6glmixer.cc | 628 ++++++++++++++++++ .../gst-plugins-good/ext/qt6/gstqml6glmixer.h | 40 ++ .../ext/qt6/gstqml6gloverlay.cc | 46 +- .../gst-plugins-good/ext/qt6/gstqml6glsink.cc | 45 +- .../gst-plugins-good/ext/qt6/gstqt6elements.h | 1 + .../gst-plugins-good/ext/qt6/meson.build | 1 + .../tests/examples/qt6/meson.build | 1 + .../tests/examples/qt6/qmlmixer/main.cpp | 127 ++++ .../tests/examples/qt6/qmlmixer/main.qml | 38 ++ .../tests/examples/qt6/qmlmixer/meson.build | 11 + .../tests/examples/qt6/qmlmixer/mixer.qml | 67 ++ .../tests/examples/qt6/qmlmixer/qmlmixer.qrc | 6 + 13 files changed, 967 insertions(+), 45 deletions(-) create mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.cc create mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.h create mode 100644 subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.cpp create mode 100644 subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.qml create mode 100644 subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/meson.build create mode 100644 subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/mixer.qml create mode 100644 subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/qmlmixer.qrc diff --git a/subprojects/gst-plugins-good/ext/qt6/gstplugin.cc b/subprojects/gst-plugins-good/ext/qt6/gstplugin.cc index 587cc96bee..d9e6546148 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstplugin.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstplugin.cc @@ -31,6 +31,7 @@ plugin_init (GstPlugin * plugin) ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin); + ret |= GST_ELEMENT_REGISTER (qml6glmixer, plugin); ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin); return ret; diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.cc b/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.cc new file mode 100644 index 0000000000..6c9372986f --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.cc @@ -0,0 +1,628 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstqml6gmixer + * + * `qml6glmixer` provides a way to render an almost-arbitrary QML scene within + * GStreamer pipeline using the same OpenGL context that GStreamer uses + * internally. This avoids attempting to share multiple OpenGL contexts + * avoiding increased synchronisation points and attempting to share an OpenGL + * context at runtime which some drivers do not like. The Intel driver on + * Windows is a notable example of the last point. + * + * `qml6glmixer` will attempt to retrieve the windowing system display connection + * that Qt is using (#GstGLDisplay). This may be different to any already + * existing window system display connection already in use in the pipeline for + * a number of reasons. A couple of examples of this are: + * + * 1. Adding `qml6glmixer` to an already running pipeline + * 2. Not having any `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element + * start up before any other OpenGL-based element in the pipeline. + * + * If one of these scenarios occurs, then there will be multiple OpenGL contexts + * in use in the pipeline. This means that either the pipeline will fail to + * start up correctly, a downstream element may reject buffers, or a complete + * GPU->System memory->GPU transfer is performed for every buffer. + * + * The requirement to avoid this is that all elements share the same + * #GstGLDisplay object and as Qt cannot currently share an existing window + * system display connection, GStreamer must use the window system display + * connection provided by Qt. This window system display connection can be + * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element or a + * `qmlglmixer element. The recommended usage is to have either elements + * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer) be the first to propagate + * the #GstGLDisplay for the entire pipeline to use by setting either element + * to the READY element state before any other OpenGL element in the pipeline. + * + * In the dynamically adding `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. + * + * All instances of `qml6glmixer`, `qml6glsink`, and `qml6gloverlay` will return + * the exact same #GstGLDisplay object while the pipeline is running regardless + * of whether any `qml6glmixer`, `qml6glsink`, or `qml6gloverlay` elements are + * added or removed from the pipeline. + * + * The Qml scene will run at configured output framerate. The timestamps on the + * output buffers are used to drive the animation time. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstqt6elements.h" +#include "gstqml6glmixer.h" +#include "qt6glrenderer.h" +#include "gstqt6glutility.h" + +#include + +#include +#include + +#define GST_CAT_DEFAULT gst_debug_qml6_gl_mixer +GST_DEBUG_CATEGORY (GST_CAT_DEFAULT); + +enum +{ + PROP_PAD_0, + PROP_PAD_WIDGET, +}; + +struct _GstQml6GLMixerPad +{ + GstGLMixerPad parent; + + QSharedPointer widget; +}; + +G_DEFINE_FINAL_TYPE (GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST_TYPE_GL_MIXER_PAD); + +static gboolean +gst_qml6_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad *vagg_pad, GstVideoAggregator * vagg, + GstBuffer *buffer, GstVideoFrame * prepared_frame) +{ + GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (vagg_pad); + + if (!GST_VIDEO_AGGREGATOR_PAD_CLASS (gst_qml6_gl_mixer_pad_parent_class)->prepare_frame (vagg_pad, vagg, buffer, prepared_frame)) + return FALSE; + + if (pad->widget) { + GstMemory *mem; + GstGLMemory *gl_mem; + GstCaps *in_caps; + GstGLContext *context; + + in_caps = gst_video_info_to_caps (&vagg->info); + gst_caps_set_features_simple (in_caps, gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + pad->widget->setCaps (in_caps); + gst_clear_caps (&in_caps); + + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_gl_memory (mem)) { + GST_ELEMENT_ERROR (vagg_pad, RESOURCE, NOT_FOUND, + (NULL), ("Input memory must be a GstGLMemory")); + return GST_FLOW_ERROR; + } + gl_mem = (GstGLMemory *) mem; + context = gst_gl_base_mixer_get_gl_context (GST_GL_BASE_MIXER (vagg)); + if (!gst_gl_context_can_share (gl_mem->mem.context, context)) { + GST_WARNING_OBJECT (vagg_pad, "Cannot use the current input texture " + "(input buffer GL context %" GST_PTR_FORMAT " cannot share " + "resources with the configured OpenGL context %" GST_PTR_FORMAT ")", + gl_mem->mem.context, context); + } else { + pad->widget->setBuffer (buffer); + } + } + + return TRUE; +} + +static void +gst_qml6_gl_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object); + + switch (prop_id) { + case PROP_PAD_WIDGET: { + Qt6GLVideoItem *qt_item = static_cast (g_value_get_pointer (value)); + if (qt_item) + qml6_gl_mixer_pad->widget = qt_item->getInterface(); + else + qml6_gl_mixer_pad->widget.clear(); + break; + } + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object); + + switch (prop_id) { + case PROP_PAD_WIDGET: + /* This is not really safe - the app needs to be + * sure the widget is going to be kept alive or + * this can crash */ + if (qml6_gl_mixer_pad->widget) + g_value_set_pointer (value, qml6_gl_mixer_pad->widget->videoItem()); + else + g_value_set_pointer (value, NULL); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_pad_finalize (GObject * object) +{ + GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (object); + + pad->widget.clear(); + + G_OBJECT_CLASS (gst_qml6_gl_mixer_pad_parent_class)->finalize (object); +} + +static void +gst_qml6_gl_mixer_pad_class_init (GstQml6GLMixerPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vagg_pad_class = (GstVideoAggregatorPadClass *) klass; + + gobject_class->set_property = gst_qml6_gl_mixer_pad_set_property; + gobject_class->get_property = gst_qml6_gl_mixer_pad_get_property; + gobject_class->finalize = gst_qml6_gl_mixer_pad_finalize; + + g_object_class_install_property (gobject_class, PROP_PAD_WIDGET, + g_param_spec_pointer ("widget", "QQuickItem", + "The QQuickItem to place the input video in the object hierarchy", + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + vagg_pad_class->prepare_frame = gst_qml6_gl_mixer_pad_prepare_frame; +} + +static void +gst_qml6_gl_mixer_pad_init (GstQml6GLMixerPad * pad) +{ + pad->widget = QSharedPointer(); +} + +static void gst_qml6_gl_mixer_finalize (GObject * object); +static void gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * param_spec); +static void gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * param_spec); + +static gboolean gst_qml6_gl_mixer_process_buffers (GstGLMixer * btrans, + GstBuffer * outbuf); + +static gboolean gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer); +static void gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer); + +static GstFlowReturn gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf); + +static gboolean gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps); + +static GstStateChangeReturn gst_qml6_gl_mixer_change_state (GstElement * element, + GstStateChange transition); + +enum +{ + PROP_0, + PROP_QML_SCENE, + PROP_ROOT_ITEM, +}; + +enum +{ + SIGNAL_0, + SIGNAL_QML_SCENE_INITIALIZED, + SIGNAL_QML_SCENE_DESTROYED, + LAST_SIGNAL +}; + +static guint gst_qml6_gl_mixer_signals[LAST_SIGNAL] = { 0 }; + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA")) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA")) + ); + +struct _GstQml6GLMixer { + GstGLMixer parent; + + gchar *qml_scene; + + GstQt6QuickRenderer *renderer; + GstBuffer *outbuf; +}; + +#define gst_qml6_gl_mixer_parent_class parent_class +G_DEFINE_FINAL_TYPE_WITH_CODE (GstQml6GLMixer, gst_qml6_gl_mixer, + GST_TYPE_GL_MIXER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, + "qml6glmixer", 0, "Qt6 Video Mixer")); +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glmixer, "qml6glmixer", + GST_RANK_NONE, GST_TYPE_QML6_GL_MIXER, qt6_element_init (plugin)); + +static void +gst_qml6_gl_mixer_class_init (GstQml6GLMixerClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstAggregatorClass *agg_class; + GstVideoAggregatorClass *vagg_class; + GstGLBaseMixerClass *glbasemixer_class; + GstGLMixerClass *glmixer_class; + GstElementClass *element_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + glbasemixer_class = (GstGLBaseMixerClass *) klass; + glmixer_class = (GstGLMixerClass *) klass; + vagg_class = (GstVideoAggregatorClass *) klass; + agg_class = (GstAggregatorClass *) klass; + element_class = (GstElementClass *) klass; + + gobject_class->set_property = gst_qml6_gl_mixer_set_property; + gobject_class->get_property = gst_qml6_gl_mixer_get_property; + gobject_class->finalize = gst_qml6_gl_mixer_finalize; + + gst_element_class_set_metadata (gstelement_class, "Qt6 Video Mixer", + "Video/QML/Mixer", "A mixer that renders a QML scene", + "Matthew Waters "); + + g_object_class_install_property (gobject_class, PROP_QML_SCENE, + g_param_spec_string ("qml-scene", "QML Scene", + "The contents of the QML scene", NULL, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property (gobject_class, PROP_ROOT_ITEM, + g_param_spec_pointer ("root-item", "QQuickItem", + "The root QQuickItem from the qml-scene used to render", + (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + + /** + * GstQmlGLMixer::qml-scene-initialized + * @element: the #GstQmlGLMixer + * @user_data: user provided data + */ + gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED] = + g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + /** + * GstQmlGLMixer::qml-scene-destroyed + * @element: the #GstQmlGLMixer + * @user_data: user provided data + */ + gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED] = + g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0); + + glbasemixer_class->gl_start = gst_qml6_gl_mixer_gl_start; + glbasemixer_class->gl_stop = gst_qml6_gl_mixer_gl_stop; + + glmixer_class->process_buffers = gst_qml6_gl_mixer_process_buffers; + + vagg_class->create_output_buffer = gst_qml6_gl_mixer_create_output_buffer; + + agg_class->negotiated_src_caps = gst_qml6_gl_mixer_negotiated_src_caps; + + element_class->change_state = gst_qml6_gl_mixer_change_state; + + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_factory, GST_TYPE_AGGREGATOR_PAD); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_factory, GST_TYPE_QML6_GL_MIXER_PAD); +} + +static void +gst_qml6_gl_mixer_init (GstQml6GLMixer * qml6_gl_mixer) +{ + qml6_gl_mixer->qml_scene = NULL; +} + +static void +gst_qml6_gl_mixer_finalize (GObject * object) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + g_free (qml6_gl_mixer->qml_scene); + qml6_gl_mixer->qml_scene = NULL; + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + switch (prop_id) { + case PROP_QML_SCENE: + g_free (qml6_gl_mixer->qml_scene); + qml6_gl_mixer->qml_scene = g_value_dup_string (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object); + + switch (prop_id) { + case PROP_QML_SCENE: + g_value_set_string (value, qml6_gl_mixer->qml_scene); + break; + case PROP_ROOT_ITEM: + GST_OBJECT_LOCK (qml6_gl_mixer); + if (qml6_gl_mixer->renderer) { + QQuickItem *root = qml6_gl_mixer->renderer->rootItem(); + if (root) + g_value_set_pointer (value, root); + else + g_value_set_pointer (value, NULL); + } else { + g_value_set_pointer (value, NULL); + } + GST_OBJECT_UNLOCK (qml6_gl_mixer); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (aggregator); + GstVideoInfo out_info; + + if (!gst_video_info_from_caps (&out_info, out_caps)) + return FALSE; + + qml6_gl_mixer->renderer->setSize (GST_VIDEO_INFO_WIDTH (&out_info), + GST_VIDEO_INFO_HEIGHT (&out_info)); + + return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (aggregator, out_caps); +} + +static gboolean +gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer); + + QQuickItem *root; + GError *error = NULL; + + GST_TRACE_OBJECT (bmixer, "using scene:\n%s", qml6_gl_mixer->qml_scene); + + if (!qml6_gl_mixer->qml_scene || g_strcmp0 (qml6_gl_mixer->qml_scene, "") == 0) { + GST_ELEMENT_ERROR (bmixer, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL)); + return FALSE; + } + + if (!GST_GL_BASE_MIXER_CLASS (parent_class)->gl_start (bmixer)) + return FALSE; + + GST_OBJECT_LOCK (bmixer); + qml6_gl_mixer->renderer = new GstQt6QuickRenderer; + if (!qml6_gl_mixer->renderer->init (bmixer->context, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + delete qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (bmixer); + return FALSE; + } + + /* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */ + if (!qml6_gl_mixer->renderer->setQmlScene (qml6_gl_mixer->qml_scene, &error)) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("%s", error->message), (NULL)); + goto fail_renderer; + } + + root = qml6_gl_mixer->renderer->rootItem(); + if (!root) { + GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND, + ("Qml scene does not have a root item"), (NULL)); + goto fail_renderer; + } + GST_OBJECT_UNLOCK (bmixer); + + g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item"); + g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED], 0); + + return TRUE; + +fail_renderer: + { + qml6_gl_mixer->renderer->cleanup(); + delete qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (bmixer); + return FALSE; + } +} + +static void +gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer); + GstQt6QuickRenderer *renderer = NULL; + + /* notify before actually destroying anything */ + GST_OBJECT_LOCK (qml6_gl_mixer); + if (qml6_gl_mixer->renderer) + renderer = qml6_gl_mixer->renderer; + qml6_gl_mixer->renderer = NULL; + GST_OBJECT_UNLOCK (qml6_gl_mixer); + + g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED], 0); + g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item"); + + /* TODO: clear all pad buffers in the items? + if (qml6_gl_mixer->widget) + qml6_gl_mixer->widget->setBuffer (NULL); +*/ + if (renderer) { + renderer->cleanup(); + delete renderer; + } + + GST_GL_BASE_MIXER_CLASS (parent_class)->gl_stop (bmixer); +} + +static GstFlowReturn +gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf) +{ + *outbuf = gst_buffer_new(); + + return GST_FLOW_OK; +} + +static gboolean +qml6_gl_mixer_gl_callback (GstGLContext *context, GstQml6GLMixer * qml6_gl_mixer) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (qml6_gl_mixer); + GstGLMemory *out_mem; + + /* XXX: is this the correct ts to drive the animation */ + out_mem = qml6_gl_mixer->renderer->generateOutput (GST_BUFFER_PTS (qml6_gl_mixer->outbuf)); + if (!out_mem) { + GST_ERROR_OBJECT (qml6_gl_mixer, "Failed to generate output"); + return FALSE; + } + + gst_buffer_append_memory (qml6_gl_mixer->outbuf, (GstMemory *) out_mem); + gst_buffer_add_video_meta (qml6_gl_mixer->outbuf, (GstVideoFrameFlags) 0, + GST_VIDEO_INFO_FORMAT (&vagg->info), + GST_VIDEO_INFO_WIDTH (&vagg->info), + GST_VIDEO_INFO_HEIGHT (&vagg->info)); + + return TRUE; +} + +static gboolean +gst_qml6_gl_mixer_process_buffers (GstGLMixer * mix, + GstBuffer * outbuf) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (mix); + GstGLBaseMixer *bmix = GST_GL_BASE_MIXER (mix); + GstGLContext *context = gst_gl_base_mixer_get_gl_context (bmix); + + qml6_gl_mixer->outbuf = outbuf; + gst_gl_context_thread_add (context, + (GstGLContextThreadFunc) qml6_gl_mixer_gl_callback, qml6_gl_mixer); + qml6_gl_mixer->outbuf = NULL; + + gst_clear_object (&context); + + return TRUE; +} + +static GstStateChangeReturn +gst_qml6_gl_mixer_change_state (GstElement * element, + GstStateChange transition) +{ + GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (element); + GstGLBaseMixer *bmixer = GST_GL_BASE_MIXER (element); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + + GST_DEBUG_OBJECT (element, "changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: { + QGuiApplication *app; + GstGLDisplay *display = NULL; + + app = static_cast (QCoreApplication::instance ()); + if (!app) { + GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND, + ("%s", "Failed to connect to Qt"), + ("%s", "Could not retrieve QGuiApplication instance")); + return GST_STATE_CHANGE_FAILURE; + } + + display = gst_qml6_get_gl_display (FALSE); + + if (display != bmixer->display) + /* always propagate. The application may need to choose between window + * system display connections */ + gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_mixer), display); + gst_object_unref (display); + break; + } + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + default: + break; + } + + return ret; + +} diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.h b/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.h new file mode 100644 index 0000000000..e58cb5e16b --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6glmixer.h @@ -0,0 +1,40 @@ +/* + * GStreamer + * Copyright (C) 2020 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QML6_GL_MIXER_H__ +#define __GST_QML6_GL_MIXER_H__ + +#include +#include +#include +#include "qt6glrenderer.h" +#include "qt6glitem.h" + +G_BEGIN_DECLS + +#define GST_TYPE_QML6_GL_MIXER_PAD (gst_qml6_gl_mixer_pad_get_type()) +G_DECLARE_FINAL_TYPE(GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST, QML6_GL_MIXER_PAD, GstGLMixerPad); + +#define GST_TYPE_QML6_GL_MIXER (gst_qml6_gl_mixer_get_type()) +G_DECLARE_FINAL_TYPE(GstQml6GLMixer, gst_qml6_gl_mixer, GST, QML6_GL_MIXER, GstGLMixer); + +G_END_DECLS + +#endif /* __GST_QML6_GL_MIXER_H__ */ diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6gloverlay.cc b/subprojects/gst-plugins-good/ext/qt6/gstqml6gloverlay.cc index 200baaa91e..3d19ad350c 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqml6gloverlay.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6gloverlay.cc @@ -34,8 +34,8 @@ * a number of reasons. A couple of examples of this are: * * 1. Adding `qml6gloverlay` to an already running pipeline - * 2. Not having any `qml6gloverlay` (or `qml6glsink`) element start up before any - * other OpenGL-based element in the pipeline. + * 2. Not having any `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element + * start up before any other OpenGL-based element in the pipeline. * * If one of these scenarios occurs, then there will be multiple OpenGL contexts * in use in the pipeline. This means that either the pipeline will fail to @@ -46,29 +46,29 @@ * #GstGLDisplay object and as Qt cannot currently share an existing window * system display connection, GStreamer must use the window system display * connection provided by Qt. This window system display connection can be - * retrieved by either a `qml6glsink` element or a `qml6gloverlay` element. The - * recommended usage is to have either element (`qml6glsink` or `qml6gloverlay`) - * be the first to propagate the #GstGLDisplay for the entire pipeline to use by - * setting either element to the READY element state before any other OpenGL - * element in the pipeline. + * retrieved by either a `qml6glsink` element, a `qml6gloverlay`, or a + * `qml6glmixer` element. The recommended usage is to have either element + * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer`) be the first to propagate + * the #GstGLDisplay for the entire pipeline to use by setting either element + * to the READY element state before any other OpenGL element in the pipeline. * - * In a dynamically adding `qml6gloverlay` (or `qml6glsink`) to a pipeline case, - * there are some considerations for ensuring that the window system display - * and OpenGL contexts are compatible with Qt. When the `qml6gloverlay` (or - * `qml6glsink`) element is added and brought up to READY, it will propagate it's - * own #GstGLDisplay using the #GstContext mechanism regardless of any existing - * #GstGLDisplay used by the pipeline previously. In order for the new - * #GstGLDisplay to be used, the application must then set the provided - * #GstGLDisplay containing #GstContext on the pipeline. This may effectively - * cause each OpenGL element to replace the window system display and also the - * OpenGL context it is using. As such this process may take a significant - * amount of time and resources as objects are recreated in the new OpenGL - * context. + * In the dynamically adding `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. * - * All instances of `qml6gloverlay` and `qml6glsink` will return the exact same - * #GstGLDisplay object while the pipeline is running regardless of whether - * any `qml6gloverlay` or `qml6glsink` elements are added or removed from the - * pipeline. + * All instances of `qml6gloverlay`, `qml6glsink`, and `qml6glmixer` will + * return the exact same #GstGLDisplay object while the pipeline is running + * regardless of whether any `qml6gloverlay` or `qml6glsink` elements are + * added or removed from the pipeline. * * The Qml scene will run at the pace of incoming buffers. One input buffer * will cause a render of one output buffer. The timestamps on the input diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc index cff6277955..531bcea2ed 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc @@ -43,29 +43,30 @@ * #GstGLDisplay object and as Qt cannot currently share an existing window * system display connection, GStreamer must use the window system display * connection provided by Qt. This window system display connection can be - * retrieved by either a qmlglsink element or a qmlgloverlay element. The - * recommended usage is to have either element (qmlglsink or qmlgloverlay) - * be the first to propagate the #GstGLDisplay for the entire pipeline to use by - * setting either element to the READY element state before any other OpenGL - * element in the pipeline. - * - * In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case, - * there are some considerations for ensuring that the window system display - * and OpenGL contexts are compatible with Qt. When the qmlgloverlay (or - * qmlglsink) element is added and brought up to READY, it will propagate it's - * own #GstGLDisplay using the #GstContext mechanism regardless of any existing - * #GstGLDisplay used by the pipeline previously. In order for the new - * #GstGLDisplay to be used, the application must then set the provided - * #GstGLDisplay containing #GstContext on the pipeline. This may effectively - * cause each OpenGL element to replace the window system display and also the - * OpenGL context it is using. As such this process may take a significant - * amount of time and resources as objects are recreated in the new OpenGL - * context. - * - * All instances of qmlglsink and qmlgloverlay will return the exact same - * #GstGLDisplay object while the pipeline is running regardless of whether - * any qmlglsink or qmlgloverlay elements are added or removed from the + * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element, or a + * `qml6glmixer` element. The recommended usage is to have either element + * (`qml6glsink`, or `qml6gloverlay`, or `qml6glmixer`) be the first to + * propagate the #GstGLDisplay for the entire pipeline to use by setting either + * element to the READY element state before any other OpenGL element in the * pipeline. + * + * In the dynamically adding `qml6glsink` (or `qml6gloverlay`, or `qml6glmixer`) + * to a pipeline case, there are some considerations for ensuring that the + * window system display and OpenGL contexts are compatible with Qt. When the + * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and + * brought up to READY, it will propagate it's own #GstGLDisplay using the + * #GstContext mechanism regardless of any existing #GstGLDisplay used by the + * pipeline previously. In order for the new #GstGLDisplay to be used, the + * application must then set the provided #GstGLDisplay containing #GstContext + * on the pipeline. This may effectively cause each OpenGL element to replace + * the window system display and also the OpenGL context it is using. As such + * this process may take a significant amount of time and resources as objects + * are recreated in the new OpenGL context. + * + * All instances of `qml6glsink`, `qml6gloverlay`, and `qml6glmixer` will + * return the exact same #GstGLDisplay object while the pipeline is running + * regardless of whether any `qml6glsink` or `qml6gloverlay` elements are + * added or removed from the pipeline. */ #ifdef HAVE_CONFIG_H diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqt6elements.h b/subprojects/gst-plugins-good/ext/qt6/gstqt6elements.h index 1aa4ed5e00..c1a7955a1a 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqt6elements.h +++ b/subprojects/gst-plugins-good/ext/qt6/gstqt6elements.h @@ -28,6 +28,7 @@ void qt6_element_init (GstPlugin * plugin); GST_ELEMENT_REGISTER_DECLARE (qml6glsink); GST_ELEMENT_REGISTER_DECLARE (qml6glsrc); +GST_ELEMENT_REGISTER_DECLARE (qml6glmixer); GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay); G_END_DECLS diff --git a/subprojects/gst-plugins-good/ext/qt6/meson.build b/subprojects/gst-plugins-good/ext/qt6/meson.build index 955d2445d4..ee1b103d6d 100644 --- a/subprojects/gst-plugins-good/ext/qt6/meson.build +++ b/subprojects/gst-plugins-good/ext/qt6/meson.build @@ -5,6 +5,7 @@ sources = [ 'gstqt6glutility.cc', 'gstqml6glsink.cc', 'gstqml6glsrc.cc', + 'gstqml6glmixer.cc', 'gstqml6gloverlay.cc', 'qt6glitem.cc', 'qt6glwindow.cc', diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/meson.build b/subprojects/gst-plugins-good/tests/examples/qt6/meson.build index 3b2f9958d2..9e8f83574c 100644 --- a/subprojects/gst-plugins-good/tests/examples/qt6/meson.build +++ b/subprojects/gst-plugins-good/tests/examples/qt6/meson.build @@ -17,3 +17,4 @@ endif subdir('qmlsink') subdir('qmlsrc') subdir('qmloverlay') +subdir('qmlmixer') diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.cpp b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.cpp new file mode 100644 index 0000000000..011e3460c4 --- /dev/null +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.cpp @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include +#include +#include + +class SetPlaying : public QRunnable +{ +public: + SetPlaying(GstElement *); + ~SetPlaying(); + + void run (); + +private: + GstElement * pipeline_; +}; + +SetPlaying::SetPlaying (GstElement * pipeline) +{ + this->pipeline_ = pipeline ? static_cast (gst_object_ref (pipeline)) : NULL; +} + +SetPlaying::~SetPlaying () +{ + if (this->pipeline_) + gst_object_unref (this->pipeline_); +} + +void +SetPlaying::run () +{ + if (this->pipeline_) + gst_element_set_state (this->pipeline_, GST_STATE_PLAYING); +} + +static void +on_mixer_scene_initialized (GstElement * mixer, gpointer unused) +{ + QQuickItem *rootObject; + GST_INFO ("scene initialized"); + g_object_get (mixer, "root-item", &rootObject, NULL); + + QQuickItem *videoItem0 = rootObject->findChild ("inputVideoItem0"); + GstPad *sink0 = gst_element_get_static_pad (mixer, "sink_0"); + g_object_set (sink0, "widget", videoItem0, NULL); + gst_clear_object (&sink0); + + QQuickItem *videoItem1 = rootObject->findChild ("inputVideoItem1"); + GstPad *sink1 = gst_element_get_static_pad (mixer, "sink_1"); + g_object_set (sink1, "widget", videoItem1, NULL); + gst_clear_object (&sink1); +} + +int main(int argc, char *argv[]) +{ + int ret; + + gst_init (&argc, &argv); + + { + QGuiApplication app(argc, argv); + + QQuickWindow::setGraphicsApi(QSGRendererInterface::OpenGL); + + GstElement *pipeline = gst_pipeline_new (NULL); + GstElement *src0 = gst_element_factory_make ("videotestsrc", NULL); + GstElement *glupload0 = gst_element_factory_make ("glupload", NULL); + GstElement *src1 = gst_element_factory_make ("videotestsrc", NULL); + gst_util_set_object_arg ((GObject *) src1, "pattern", "ball"); + GstElement *glupload1 = gst_element_factory_make ("glupload", NULL); + /* the plugin must be loaded before loading the qml file to register the + * GstGLVideoItem qml item */ + GstElement *mixer = gst_element_factory_make ("qml6glmixer", NULL); + GstElement *sink = gst_element_factory_make ("qml6glsink", NULL); + + g_assert (src0 && glupload0 && mixer && sink); + + gst_bin_add_many (GST_BIN (pipeline), src0, glupload0, src1, glupload1, mixer, sink, NULL); + gst_element_link_many (src0, glupload0, mixer, sink, NULL); + gst_element_link_many (src1, glupload1, mixer, NULL); + + /* load qmlglsink output */ + QQmlApplicationEngine engine; + engine.load(QUrl(QStringLiteral("qrc:/main.qml"))); + + QQuickItem *videoItem; + QQuickWindow *rootObject; + + /* find and set the videoItem on the sink */ + rootObject = static_cast (engine.rootObjects().first()); + videoItem = rootObject->findChild ("videoItem"); + g_assert (videoItem); + g_object_set(sink, "widget", videoItem, NULL); + + QDirIterator it(":", QDirIterator::Subdirectories); + while (it.hasNext()) { + qDebug() << it.next(); + } + + QFile f(":/mixer.qml"); + if(!f.open(QIODevice::ReadOnly)) { + qWarning() << "error: " << f.errorString(); + return 1; + } + QByteArray overlay_scene = f.readAll(); + qDebug() << overlay_scene; + + /* load qmlgloverlay contents */ + g_signal_connect (mixer, "qml-scene-initialized", G_CALLBACK (on_mixer_scene_initialized), NULL); + g_object_set (mixer, "qml-scene", overlay_scene.data(), NULL); + + rootObject->scheduleRenderJob (new SetPlaying (pipeline), + QQuickWindow::BeforeSynchronizingStage); + + ret = app.exec(); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); + } + + gst_deinit (); + + return ret; +} diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.qml b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.qml new file mode 100644 index 0000000000..27a6b22807 --- /dev/null +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/main.qml @@ -0,0 +1,38 @@ +import QtQuick 6.0 +import QtQuick.Controls 6.0 +import QtQuick.Dialogs 6.0 +import QtQuick.Window 6.0 + +import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0 + +ApplicationWindow { + id: window + visible: true + width: 640 + height: 480 + x: 30 + y: 30 + color: "black" + + Item { + anchors.fill: parent + + GstGLQt6VideoItem { + id: video + objectName: "videoItem" + anchors.centerIn: parent + width: parent.width + height: parent.height + } + + Text { + anchors.bottom: parent.bottom + anchors.horizontalCenter: parent.horizontalCenter + text: "qmlglsink text" + font.pointSize: 20 + color: "yellow" + style: Text.Outline + styleColor: "blue" + } + } +} diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/meson.build b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/meson.build new file mode 100644 index 0000000000..466b245a6e --- /dev/null +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/meson.build @@ -0,0 +1,11 @@ +sources = [ + 'main.cpp', +] + +qt_preprocessed = qt6_mod.preprocess(qresources : 'qmlmixer.qrc') +executable('qml6glmixer', sources, qt_preprocessed, + dependencies : [gst_dep, qt6qml_example_deps], + override_options : ['cpp_std=c++17'], + c_args : gst_plugins_good_args, + include_directories : [configinc], + install: false) diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/mixer.qml b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/mixer.qml new file mode 100644 index 0000000000..8cb2d6460b --- /dev/null +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/mixer.qml @@ -0,0 +1,67 @@ +import QtQuick 6.0 + +import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0 + +Item { + /* render upside down for GStreamer */ + transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 } + + GstGLQt6VideoItem { + id: video0 + objectName: "inputVideoItem0" + anchors.left: parent.left + anchors.right: parent.horizontalCenter + width: parent.width / 2 + height: parent.height + } + + GstGLQt6VideoItem { + id: video1 + objectName: "inputVideoItem1" + anchors.left: parent.horizontalCenter + anchors.right: parent.right + width: parent.width / 2 + height: parent.height + } + + Text { + id: rotatingText + anchors.centerIn: parent + text: "Qt Quick\nrendered to\na texture" + font.pointSize: 20 + color: "black" + style: Text.Outline + styleColor: "white" + + RotationAnimator { + target: rotatingText; + from: 0; + to: 360; + duration: 5000 + running: true + loops: Animation.Infinite + } + } + + Text { + property int elapsedTime: 0 + + id: time + anchors.top: rotatingText.bottom + anchors.horizontalCenter: rotatingText.horizontalCenter + font.pointSize: 12 + style: Text.Outline + styleColor: "black" + color: "white" + + Timer { + interval: 1000 + running: true + repeat: true + onTriggered: { + parent.elapsedTime += interval / 1000 + parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds" + } + } + } +} diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/qmlmixer.qrc b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/qmlmixer.qrc new file mode 100644 index 0000000000..c749352b18 --- /dev/null +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlmixer/qmlmixer.qrc @@ -0,0 +1,6 @@ + + + main.qml + mixer.qml + +