qml6: add a mixer element

Can take multiple input streams and a qml scene and layout the input
videos inside the qml scene.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4609>
This commit is contained in:
Matthew Waters 2023-05-12 13:49:20 +10:00 committed by GStreamer Marge Bot
parent 3273135a60
commit 3f4bfa097a
13 changed files with 967 additions and 45 deletions

View file

@ -31,6 +31,7 @@ plugin_init (GstPlugin * plugin)
ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin);
ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin); ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin);
ret |= GST_ELEMENT_REGISTER (qml6glmixer, plugin);
ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin); ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin);
return ret; return ret;

View file

@ -0,0 +1,628 @@
/*
* GStreamer
* Copyright (C) 2023 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:gstqml6gmixer
*
* `qml6glmixer` provides a way to render an almost-arbitrary QML scene within
* GStreamer pipeline using the same OpenGL context that GStreamer uses
* internally. This avoids attempting to share multiple OpenGL contexts
* avoiding increased synchronisation points and attempting to share an OpenGL
* context at runtime which some drivers do not like. The Intel driver on
* Windows is a notable example of the last point.
*
* `qml6glmixer` will attempt to retrieve the windowing system display connection
* that Qt is using (#GstGLDisplay). This may be different to any already
* existing window system display connection already in use in the pipeline for
* a number of reasons. A couple of examples of this are:
*
* 1. Adding `qml6glmixer` to an already running pipeline
* 2. Not having any `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element
* start up before any other OpenGL-based element in the pipeline.
*
* If one of these scenarios occurs, then there will be multiple OpenGL contexts
* in use in the pipeline. This means that either the pipeline will fail to
* start up correctly, a downstream element may reject buffers, or a complete
* GPU->System memory->GPU transfer is performed for every buffer.
*
* The requirement to avoid this is that all elements share the same
* #GstGLDisplay object and as Qt cannot currently share an existing window
* system display connection, GStreamer must use the window system display
* connection provided by Qt. This window system display connection can be
* retrieved by either a `qml6glsink` element, a `qml6gloverlay` element or a
* `qmlglmixer element. The recommended usage is to have either elements
* (`qml6glsink` or `qml6gloverlay` or `qml6glmixer) be the first to propagate
* the #GstGLDisplay for the entire pipeline to use by setting either element
* to the READY element state before any other OpenGL element in the pipeline.
*
* In the dynamically adding `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`)
* to a pipeline case, there are some considerations for ensuring that the
* window system display and OpenGL contexts are compatible with Qt. When the
* `qml6glmixer` (or `qml6glsink`, or `qml6gloverlay`) element is added and
* brought up to READY, it will propagate it's own #GstGLDisplay using the
* #GstContext mechanism regardless of any existing #GstGLDisplay used by the
* pipeline previously. In order for the new #GstGLDisplay to be used, the
* application must then set the provided #GstGLDisplay containing #GstContext
* on the pipeline. This may effectively cause each OpenGL element to replace
* the window system display and also the OpenGL context it is using. As such
* this process may take a significant amount of time and resources as objects
* are recreated in the new OpenGL context.
*
* All instances of `qml6glmixer`, `qml6glsink`, and `qml6gloverlay` will return
* the exact same #GstGLDisplay object while the pipeline is running regardless
* of whether any `qml6glmixer`, `qml6glsink`, or `qml6gloverlay` elements are
* added or removed from the pipeline.
*
* The Qml scene will run at configured output framerate. The timestamps on the
* output buffers are used to drive the animation time.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstqt6elements.h"
#include "gstqml6glmixer.h"
#include "qt6glrenderer.h"
#include "gstqt6glutility.h"
#include <QtGui/QGuiApplication>
#include <gst/gl/gl.h>
#include <gst/gl/gstglfuncs.h>
#define GST_CAT_DEFAULT gst_debug_qml6_gl_mixer
GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
enum
{
PROP_PAD_0,
PROP_PAD_WIDGET,
};
struct _GstQml6GLMixerPad
{
GstGLMixerPad parent;
QSharedPointer<Qt6GLVideoItemInterface> widget;
};
G_DEFINE_FINAL_TYPE (GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST_TYPE_GL_MIXER_PAD);
static gboolean
gst_qml6_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad *vagg_pad, GstVideoAggregator * vagg,
GstBuffer *buffer, GstVideoFrame * prepared_frame)
{
GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (vagg_pad);
if (!GST_VIDEO_AGGREGATOR_PAD_CLASS (gst_qml6_gl_mixer_pad_parent_class)->prepare_frame (vagg_pad, vagg, buffer, prepared_frame))
return FALSE;
if (pad->widget) {
GstMemory *mem;
GstGLMemory *gl_mem;
GstCaps *in_caps;
GstGLContext *context;
in_caps = gst_video_info_to_caps (&vagg->info);
gst_caps_set_features_simple (in_caps, gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY));
pad->widget->setCaps (in_caps);
gst_clear_caps (&in_caps);
mem = gst_buffer_peek_memory (buffer, 0);
if (!gst_is_gl_memory (mem)) {
GST_ELEMENT_ERROR (vagg_pad, RESOURCE, NOT_FOUND,
(NULL), ("Input memory must be a GstGLMemory"));
return GST_FLOW_ERROR;
}
gl_mem = (GstGLMemory *) mem;
context = gst_gl_base_mixer_get_gl_context (GST_GL_BASE_MIXER (vagg));
if (!gst_gl_context_can_share (gl_mem->mem.context, context)) {
GST_WARNING_OBJECT (vagg_pad, "Cannot use the current input texture "
"(input buffer GL context %" GST_PTR_FORMAT " cannot share "
"resources with the configured OpenGL context %" GST_PTR_FORMAT ")",
gl_mem->mem.context, context);
} else {
pad->widget->setBuffer (buffer);
}
}
return TRUE;
}
static void
gst_qml6_gl_mixer_pad_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object);
switch (prop_id) {
case PROP_PAD_WIDGET: {
Qt6GLVideoItem *qt_item = static_cast<Qt6GLVideoItem *> (g_value_get_pointer (value));
if (qt_item)
qml6_gl_mixer_pad->widget = qt_item->getInterface();
else
qml6_gl_mixer_pad->widget.clear();
break;
}
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_qml6_gl_mixer_pad_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstQml6GLMixerPad *qml6_gl_mixer_pad = GST_QML6_GL_MIXER_PAD (object);
switch (prop_id) {
case PROP_PAD_WIDGET:
/* This is not really safe - the app needs to be
* sure the widget is going to be kept alive or
* this can crash */
if (qml6_gl_mixer_pad->widget)
g_value_set_pointer (value, qml6_gl_mixer_pad->widget->videoItem());
else
g_value_set_pointer (value, NULL);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_qml6_gl_mixer_pad_finalize (GObject * object)
{
GstQml6GLMixerPad *pad = GST_QML6_GL_MIXER_PAD (object);
pad->widget.clear();
G_OBJECT_CLASS (gst_qml6_gl_mixer_pad_parent_class)->finalize (object);
}
static void
gst_qml6_gl_mixer_pad_class_init (GstQml6GLMixerPadClass * klass)
{
GObjectClass *gobject_class = (GObjectClass *) klass;
GstVideoAggregatorPadClass *vagg_pad_class = (GstVideoAggregatorPadClass *) klass;
gobject_class->set_property = gst_qml6_gl_mixer_pad_set_property;
gobject_class->get_property = gst_qml6_gl_mixer_pad_get_property;
gobject_class->finalize = gst_qml6_gl_mixer_pad_finalize;
g_object_class_install_property (gobject_class, PROP_PAD_WIDGET,
g_param_spec_pointer ("widget", "QQuickItem",
"The QQuickItem to place the input video in the object hierarchy",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
vagg_pad_class->prepare_frame = gst_qml6_gl_mixer_pad_prepare_frame;
}
static void
gst_qml6_gl_mixer_pad_init (GstQml6GLMixerPad * pad)
{
pad->widget = QSharedPointer<Qt6GLVideoItemInterface>();
}
static void gst_qml6_gl_mixer_finalize (GObject * object);
static void gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * param_spec);
static void gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * param_spec);
static gboolean gst_qml6_gl_mixer_process_buffers (GstGLMixer * btrans,
GstBuffer * outbuf);
static gboolean gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer);
static void gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer);
static GstFlowReturn gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf);
static gboolean gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps);
static GstStateChangeReturn gst_qml6_gl_mixer_change_state (GstElement * element,
GstStateChange transition);
enum
{
PROP_0,
PROP_QML_SCENE,
PROP_ROOT_ITEM,
};
enum
{
SIGNAL_0,
SIGNAL_QML_SCENE_INITIALIZED,
SIGNAL_QML_SCENE_DESTROYED,
LAST_SIGNAL
};
static guint gst_qml6_gl_mixer_signals[LAST_SIGNAL] = { 0 };
static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
"RGBA"))
);
static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u",
GST_PAD_SINK,
GST_PAD_REQUEST,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,
"RGBA"))
);
struct _GstQml6GLMixer {
GstGLMixer parent;
gchar *qml_scene;
GstQt6QuickRenderer *renderer;
GstBuffer *outbuf;
};
#define gst_qml6_gl_mixer_parent_class parent_class
G_DEFINE_FINAL_TYPE_WITH_CODE (GstQml6GLMixer, gst_qml6_gl_mixer,
GST_TYPE_GL_MIXER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
"qml6glmixer", 0, "Qt6 Video Mixer"));
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6glmixer, "qml6glmixer",
GST_RANK_NONE, GST_TYPE_QML6_GL_MIXER, qt6_element_init (plugin));
static void
gst_qml6_gl_mixer_class_init (GstQml6GLMixerClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstAggregatorClass *agg_class;
GstVideoAggregatorClass *vagg_class;
GstGLBaseMixerClass *glbasemixer_class;
GstGLMixerClass *glmixer_class;
GstElementClass *element_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
glbasemixer_class = (GstGLBaseMixerClass *) klass;
glmixer_class = (GstGLMixerClass *) klass;
vagg_class = (GstVideoAggregatorClass *) klass;
agg_class = (GstAggregatorClass *) klass;
element_class = (GstElementClass *) klass;
gobject_class->set_property = gst_qml6_gl_mixer_set_property;
gobject_class->get_property = gst_qml6_gl_mixer_get_property;
gobject_class->finalize = gst_qml6_gl_mixer_finalize;
gst_element_class_set_metadata (gstelement_class, "Qt6 Video Mixer",
"Video/QML/Mixer", "A mixer that renders a QML scene",
"Matthew Waters <matthew@centricular.com>");
g_object_class_install_property (gobject_class, PROP_QML_SCENE,
g_param_spec_string ("qml-scene", "QML Scene",
"The contents of the QML scene", NULL,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_ROOT_ITEM,
g_param_spec_pointer ("root-item", "QQuickItem",
"The root QQuickItem from the qml-scene used to render",
(GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
/**
* GstQmlGLMixer::qml-scene-initialized
* @element: the #GstQmlGLMixer
* @user_data: user provided data
*/
gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED] =
g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
/**
* GstQmlGLMixer::qml-scene-destroyed
* @element: the #GstQmlGLMixer
* @user_data: user provided data
*/
gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED] =
g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
glbasemixer_class->gl_start = gst_qml6_gl_mixer_gl_start;
glbasemixer_class->gl_stop = gst_qml6_gl_mixer_gl_stop;
glmixer_class->process_buffers = gst_qml6_gl_mixer_process_buffers;
vagg_class->create_output_buffer = gst_qml6_gl_mixer_create_output_buffer;
agg_class->negotiated_src_caps = gst_qml6_gl_mixer_negotiated_src_caps;
element_class->change_state = gst_qml6_gl_mixer_change_state;
gst_element_class_add_static_pad_template_with_gtype (element_class,
&src_factory, GST_TYPE_AGGREGATOR_PAD);
gst_element_class_add_static_pad_template_with_gtype (element_class,
&sink_factory, GST_TYPE_QML6_GL_MIXER_PAD);
}
static void
gst_qml6_gl_mixer_init (GstQml6GLMixer * qml6_gl_mixer)
{
qml6_gl_mixer->qml_scene = NULL;
}
static void
gst_qml6_gl_mixer_finalize (GObject * object)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
g_free (qml6_gl_mixer->qml_scene);
qml6_gl_mixer->qml_scene = NULL;
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_qml6_gl_mixer_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
switch (prop_id) {
case PROP_QML_SCENE:
g_free (qml6_gl_mixer->qml_scene);
qml6_gl_mixer->qml_scene = g_value_dup_string (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_qml6_gl_mixer_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (object);
switch (prop_id) {
case PROP_QML_SCENE:
g_value_set_string (value, qml6_gl_mixer->qml_scene);
break;
case PROP_ROOT_ITEM:
GST_OBJECT_LOCK (qml6_gl_mixer);
if (qml6_gl_mixer->renderer) {
QQuickItem *root = qml6_gl_mixer->renderer->rootItem();
if (root)
g_value_set_pointer (value, root);
else
g_value_set_pointer (value, NULL);
} else {
g_value_set_pointer (value, NULL);
}
GST_OBJECT_UNLOCK (qml6_gl_mixer);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_qml6_gl_mixer_negotiated_src_caps (GstAggregator * aggregator, GstCaps * out_caps)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (aggregator);
GstVideoInfo out_info;
if (!gst_video_info_from_caps (&out_info, out_caps))
return FALSE;
qml6_gl_mixer->renderer->setSize (GST_VIDEO_INFO_WIDTH (&out_info),
GST_VIDEO_INFO_HEIGHT (&out_info));
return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (aggregator, out_caps);
}
static gboolean
gst_qml6_gl_mixer_gl_start (GstGLBaseMixer * bmixer)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer);
QQuickItem *root;
GError *error = NULL;
GST_TRACE_OBJECT (bmixer, "using scene:\n%s", qml6_gl_mixer->qml_scene);
if (!qml6_gl_mixer->qml_scene || g_strcmp0 (qml6_gl_mixer->qml_scene, "") == 0) {
GST_ELEMENT_ERROR (bmixer, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL));
return FALSE;
}
if (!GST_GL_BASE_MIXER_CLASS (parent_class)->gl_start (bmixer))
return FALSE;
GST_OBJECT_LOCK (bmixer);
qml6_gl_mixer->renderer = new GstQt6QuickRenderer;
if (!qml6_gl_mixer->renderer->init (bmixer->context, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
delete qml6_gl_mixer->renderer;
qml6_gl_mixer->renderer = NULL;
GST_OBJECT_UNLOCK (bmixer);
return FALSE;
}
/* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */
if (!qml6_gl_mixer->renderer->setQmlScene (qml6_gl_mixer->qml_scene, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
goto fail_renderer;
}
root = qml6_gl_mixer->renderer->rootItem();
if (!root) {
GST_ELEMENT_ERROR (GST_ELEMENT (bmixer), RESOURCE, NOT_FOUND,
("Qml scene does not have a root item"), (NULL));
goto fail_renderer;
}
GST_OBJECT_UNLOCK (bmixer);
g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item");
g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_INITIALIZED], 0);
return TRUE;
fail_renderer:
{
qml6_gl_mixer->renderer->cleanup();
delete qml6_gl_mixer->renderer;
qml6_gl_mixer->renderer = NULL;
GST_OBJECT_UNLOCK (bmixer);
return FALSE;
}
}
static void
gst_qml6_gl_mixer_gl_stop (GstGLBaseMixer * bmixer)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (bmixer);
GstQt6QuickRenderer *renderer = NULL;
/* notify before actually destroying anything */
GST_OBJECT_LOCK (qml6_gl_mixer);
if (qml6_gl_mixer->renderer)
renderer = qml6_gl_mixer->renderer;
qml6_gl_mixer->renderer = NULL;
GST_OBJECT_UNLOCK (qml6_gl_mixer);
g_signal_emit (qml6_gl_mixer, gst_qml6_gl_mixer_signals[SIGNAL_QML_SCENE_DESTROYED], 0);
g_object_notify (G_OBJECT (qml6_gl_mixer), "root-item");
/* TODO: clear all pad buffers in the items?
if (qml6_gl_mixer->widget)
qml6_gl_mixer->widget->setBuffer (NULL);
*/
if (renderer) {
renderer->cleanup();
delete renderer;
}
GST_GL_BASE_MIXER_CLASS (parent_class)->gl_stop (bmixer);
}
static GstFlowReturn
gst_qml6_gl_mixer_create_output_buffer (GstVideoAggregator * vagg, GstBuffer ** outbuf)
{
*outbuf = gst_buffer_new();
return GST_FLOW_OK;
}
static gboolean
qml6_gl_mixer_gl_callback (GstGLContext *context, GstQml6GLMixer * qml6_gl_mixer)
{
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (qml6_gl_mixer);
GstGLMemory *out_mem;
/* XXX: is this the correct ts to drive the animation */
out_mem = qml6_gl_mixer->renderer->generateOutput (GST_BUFFER_PTS (qml6_gl_mixer->outbuf));
if (!out_mem) {
GST_ERROR_OBJECT (qml6_gl_mixer, "Failed to generate output");
return FALSE;
}
gst_buffer_append_memory (qml6_gl_mixer->outbuf, (GstMemory *) out_mem);
gst_buffer_add_video_meta (qml6_gl_mixer->outbuf, (GstVideoFrameFlags) 0,
GST_VIDEO_INFO_FORMAT (&vagg->info),
GST_VIDEO_INFO_WIDTH (&vagg->info),
GST_VIDEO_INFO_HEIGHT (&vagg->info));
return TRUE;
}
static gboolean
gst_qml6_gl_mixer_process_buffers (GstGLMixer * mix,
GstBuffer * outbuf)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (mix);
GstGLBaseMixer *bmix = GST_GL_BASE_MIXER (mix);
GstGLContext *context = gst_gl_base_mixer_get_gl_context (bmix);
qml6_gl_mixer->outbuf = outbuf;
gst_gl_context_thread_add (context,
(GstGLContextThreadFunc) qml6_gl_mixer_gl_callback, qml6_gl_mixer);
qml6_gl_mixer->outbuf = NULL;
gst_clear_object (&context);
return TRUE;
}
static GstStateChangeReturn
gst_qml6_gl_mixer_change_state (GstElement * element,
GstStateChange transition)
{
GstQml6GLMixer *qml6_gl_mixer = GST_QML6_GL_MIXER (element);
GstGLBaseMixer *bmixer = GST_GL_BASE_MIXER (element);
GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
GST_DEBUG_OBJECT (element, "changing state: %s => %s",
gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
switch (transition) {
case GST_STATE_CHANGE_NULL_TO_READY: {
QGuiApplication *app;
GstGLDisplay *display = NULL;
app = static_cast<QGuiApplication *> (QCoreApplication::instance ());
if (!app) {
GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
("%s", "Failed to connect to Qt"),
("%s", "Could not retrieve QGuiApplication instance"));
return GST_STATE_CHANGE_FAILURE;
}
display = gst_qml6_get_gl_display (FALSE);
if (display != bmixer->display)
/* always propagate. The application may need to choose between window
* system display connections */
gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_mixer), display);
gst_object_unref (display);
break;
}
default:
break;
}
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
if (ret == GST_STATE_CHANGE_FAILURE)
return ret;
switch (transition) {
default:
break;
}
return ret;
}

View file

@ -0,0 +1,40 @@
/*
* GStreamer
* Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_QML6_GL_MIXER_H__
#define __GST_QML6_GL_MIXER_H__
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include "qt6glrenderer.h"
#include "qt6glitem.h"
G_BEGIN_DECLS
#define GST_TYPE_QML6_GL_MIXER_PAD (gst_qml6_gl_mixer_pad_get_type())
G_DECLARE_FINAL_TYPE(GstQml6GLMixerPad, gst_qml6_gl_mixer_pad, GST, QML6_GL_MIXER_PAD, GstGLMixerPad);
#define GST_TYPE_QML6_GL_MIXER (gst_qml6_gl_mixer_get_type())
G_DECLARE_FINAL_TYPE(GstQml6GLMixer, gst_qml6_gl_mixer, GST, QML6_GL_MIXER, GstGLMixer);
G_END_DECLS
#endif /* __GST_QML6_GL_MIXER_H__ */

View file

@ -34,8 +34,8 @@
* a number of reasons. A couple of examples of this are: * a number of reasons. A couple of examples of this are:
* *
* 1. Adding `qml6gloverlay` to an already running pipeline * 1. Adding `qml6gloverlay` to an already running pipeline
* 2. Not having any `qml6gloverlay` (or `qml6glsink`) element start up before any * 2. Not having any `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element
* other OpenGL-based element in the pipeline. * start up before any other OpenGL-based element in the pipeline.
* *
* If one of these scenarios occurs, then there will be multiple OpenGL contexts * If one of these scenarios occurs, then there will be multiple OpenGL contexts
* in use in the pipeline. This means that either the pipeline will fail to * in use in the pipeline. This means that either the pipeline will fail to
@ -46,29 +46,29 @@
* #GstGLDisplay object and as Qt cannot currently share an existing window * #GstGLDisplay object and as Qt cannot currently share an existing window
* system display connection, GStreamer must use the window system display * system display connection, GStreamer must use the window system display
* connection provided by Qt. This window system display connection can be * connection provided by Qt. This window system display connection can be
* retrieved by either a `qml6glsink` element or a `qml6gloverlay` element. The * retrieved by either a `qml6glsink` element, a `qml6gloverlay`, or a
* recommended usage is to have either element (`qml6glsink` or `qml6gloverlay`) * `qml6glmixer` element. The recommended usage is to have either element
* be the first to propagate the #GstGLDisplay for the entire pipeline to use by * (`qml6glsink` or `qml6gloverlay` or `qml6glmixer`) be the first to propagate
* setting either element to the READY element state before any other OpenGL * the #GstGLDisplay for the entire pipeline to use by setting either element
* element in the pipeline. * to the READY element state before any other OpenGL element in the pipeline.
* *
* In a dynamically adding `qml6gloverlay` (or `qml6glsink`) to a pipeline case, * In the dynamically adding `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`)
* there are some considerations for ensuring that the window system display * to a pipeline case, there are some considerations for ensuring that the
* and OpenGL contexts are compatible with Qt. When the `qml6gloverlay` (or * window system display and OpenGL contexts are compatible with Qt. When the
* `qml6glsink`) element is added and brought up to READY, it will propagate it's * `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and
* own #GstGLDisplay using the #GstContext mechanism regardless of any existing * brought up to READY, it will propagate it's own #GstGLDisplay using the
* #GstGLDisplay used by the pipeline previously. In order for the new * #GstContext mechanism regardless of any existing #GstGLDisplay used by the
* #GstGLDisplay to be used, the application must then set the provided * pipeline previously. In order for the new #GstGLDisplay to be used, the
* #GstGLDisplay containing #GstContext on the pipeline. This may effectively * application must then set the provided #GstGLDisplay containing #GstContext
* cause each OpenGL element to replace the window system display and also the * on the pipeline. This may effectively cause each OpenGL element to replace
* OpenGL context it is using. As such this process may take a significant * the window system display and also the OpenGL context it is using. As such
* amount of time and resources as objects are recreated in the new OpenGL * this process may take a significant amount of time and resources as objects
* context. * are recreated in the new OpenGL context.
* *
* All instances of `qml6gloverlay` and `qml6glsink` will return the exact same * All instances of `qml6gloverlay`, `qml6glsink`, and `qml6glmixer` will
* #GstGLDisplay object while the pipeline is running regardless of whether * return the exact same #GstGLDisplay object while the pipeline is running
* any `qml6gloverlay` or `qml6glsink` elements are added or removed from the * regardless of whether any `qml6gloverlay` or `qml6glsink` elements are
* pipeline. * added or removed from the pipeline.
* *
* The Qml scene will run at the pace of incoming buffers. One input buffer * The Qml scene will run at the pace of incoming buffers. One input buffer
* will cause a render of one output buffer. The timestamps on the input * will cause a render of one output buffer. The timestamps on the input

View file

@ -43,29 +43,30 @@
* #GstGLDisplay object and as Qt cannot currently share an existing window * #GstGLDisplay object and as Qt cannot currently share an existing window
* system display connection, GStreamer must use the window system display * system display connection, GStreamer must use the window system display
* connection provided by Qt. This window system display connection can be * connection provided by Qt. This window system display connection can be
* retrieved by either a qmlglsink element or a qmlgloverlay element. The * retrieved by either a `qml6glsink` element, a `qml6gloverlay` element, or a
* recommended usage is to have either element (qmlglsink or qmlgloverlay) * `qml6glmixer` element. The recommended usage is to have either element
* be the first to propagate the #GstGLDisplay for the entire pipeline to use by * (`qml6glsink`, or `qml6gloverlay`, or `qml6glmixer`) be the first to
* setting either element to the READY element state before any other OpenGL * propagate the #GstGLDisplay for the entire pipeline to use by setting either
* element in the pipeline. * element to the READY element state before any other OpenGL element in the
*
* In a dynamically adding qmlglsink (or qmlgloverlay) to a pipeline case,
* there are some considerations for ensuring that the window system display
* and OpenGL contexts are compatible with Qt. When the qmlgloverlay (or
* qmlglsink) element is added and brought up to READY, it will propagate it's
* own #GstGLDisplay using the #GstContext mechanism regardless of any existing
* #GstGLDisplay used by the pipeline previously. In order for the new
* #GstGLDisplay to be used, the application must then set the provided
* #GstGLDisplay containing #GstContext on the pipeline. This may effectively
* cause each OpenGL element to replace the window system display and also the
* OpenGL context it is using. As such this process may take a significant
* amount of time and resources as objects are recreated in the new OpenGL
* context.
*
* All instances of qmlglsink and qmlgloverlay will return the exact same
* #GstGLDisplay object while the pipeline is running regardless of whether
* any qmlglsink or qmlgloverlay elements are added or removed from the
* pipeline. * pipeline.
*
* In the dynamically adding `qml6glsink` (or `qml6gloverlay`, or `qml6glmixer`)
* to a pipeline case, there are some considerations for ensuring that the
* window system display and OpenGL contexts are compatible with Qt. When the
* `qml6gloverlay` (or `qml6glsink`, or `qml6glmixer`) element is added and
* brought up to READY, it will propagate it's own #GstGLDisplay using the
* #GstContext mechanism regardless of any existing #GstGLDisplay used by the
* pipeline previously. In order for the new #GstGLDisplay to be used, the
* application must then set the provided #GstGLDisplay containing #GstContext
* on the pipeline. This may effectively cause each OpenGL element to replace
* the window system display and also the OpenGL context it is using. As such
* this process may take a significant amount of time and resources as objects
* are recreated in the new OpenGL context.
*
* All instances of `qml6glsink`, `qml6gloverlay`, and `qml6glmixer` will
* return the exact same #GstGLDisplay object while the pipeline is running
* regardless of whether any `qml6glsink` or `qml6gloverlay` elements are
* added or removed from the pipeline.
*/ */
#ifdef HAVE_CONFIG_H #ifdef HAVE_CONFIG_H

View file

@ -28,6 +28,7 @@ void qt6_element_init (GstPlugin * plugin);
GST_ELEMENT_REGISTER_DECLARE (qml6glsink); GST_ELEMENT_REGISTER_DECLARE (qml6glsink);
GST_ELEMENT_REGISTER_DECLARE (qml6glsrc); GST_ELEMENT_REGISTER_DECLARE (qml6glsrc);
GST_ELEMENT_REGISTER_DECLARE (qml6glmixer);
GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay); GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay);
G_END_DECLS G_END_DECLS

View file

@ -5,6 +5,7 @@ sources = [
'gstqt6glutility.cc', 'gstqt6glutility.cc',
'gstqml6glsink.cc', 'gstqml6glsink.cc',
'gstqml6glsrc.cc', 'gstqml6glsrc.cc',
'gstqml6glmixer.cc',
'gstqml6gloverlay.cc', 'gstqml6gloverlay.cc',
'qt6glitem.cc', 'qt6glitem.cc',
'qt6glwindow.cc', 'qt6glwindow.cc',

View file

@ -17,3 +17,4 @@ endif
subdir('qmlsink') subdir('qmlsink')
subdir('qmlsrc') subdir('qmlsrc')
subdir('qmloverlay') subdir('qmloverlay')
subdir('qmlmixer')

View file

@ -0,0 +1,127 @@
#include <QApplication>
#include <QQmlApplicationEngine>
#include <QQuickWindow>
#include <QQuickItem>
#include <QRunnable>
#include <QDirIterator>
#include <gst/gst.h>
class SetPlaying : public QRunnable
{
public:
SetPlaying(GstElement *);
~SetPlaying();
void run ();
private:
GstElement * pipeline_;
};
SetPlaying::SetPlaying (GstElement * pipeline)
{
this->pipeline_ = pipeline ? static_cast<GstElement *> (gst_object_ref (pipeline)) : NULL;
}
SetPlaying::~SetPlaying ()
{
if (this->pipeline_)
gst_object_unref (this->pipeline_);
}
void
SetPlaying::run ()
{
if (this->pipeline_)
gst_element_set_state (this->pipeline_, GST_STATE_PLAYING);
}
static void
on_mixer_scene_initialized (GstElement * mixer, gpointer unused)
{
QQuickItem *rootObject;
GST_INFO ("scene initialized");
g_object_get (mixer, "root-item", &rootObject, NULL);
QQuickItem *videoItem0 = rootObject->findChild<QQuickItem *> ("inputVideoItem0");
GstPad *sink0 = gst_element_get_static_pad (mixer, "sink_0");
g_object_set (sink0, "widget", videoItem0, NULL);
gst_clear_object (&sink0);
QQuickItem *videoItem1 = rootObject->findChild<QQuickItem *> ("inputVideoItem1");
GstPad *sink1 = gst_element_get_static_pad (mixer, "sink_1");
g_object_set (sink1, "widget", videoItem1, NULL);
gst_clear_object (&sink1);
}
int main(int argc, char *argv[])
{
int ret;
gst_init (&argc, &argv);
{
QGuiApplication app(argc, argv);
QQuickWindow::setGraphicsApi(QSGRendererInterface::OpenGL);
GstElement *pipeline = gst_pipeline_new (NULL);
GstElement *src0 = gst_element_factory_make ("videotestsrc", NULL);
GstElement *glupload0 = gst_element_factory_make ("glupload", NULL);
GstElement *src1 = gst_element_factory_make ("videotestsrc", NULL);
gst_util_set_object_arg ((GObject *) src1, "pattern", "ball");
GstElement *glupload1 = gst_element_factory_make ("glupload", NULL);
/* the plugin must be loaded before loading the qml file to register the
* GstGLVideoItem qml item */
GstElement *mixer = gst_element_factory_make ("qml6glmixer", NULL);
GstElement *sink = gst_element_factory_make ("qml6glsink", NULL);
g_assert (src0 && glupload0 && mixer && sink);
gst_bin_add_many (GST_BIN (pipeline), src0, glupload0, src1, glupload1, mixer, sink, NULL);
gst_element_link_many (src0, glupload0, mixer, sink, NULL);
gst_element_link_many (src1, glupload1, mixer, NULL);
/* load qmlglsink output */
QQmlApplicationEngine engine;
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
QQuickItem *videoItem;
QQuickWindow *rootObject;
/* find and set the videoItem on the sink */
rootObject = static_cast<QQuickWindow *> (engine.rootObjects().first());
videoItem = rootObject->findChild<QQuickItem *> ("videoItem");
g_assert (videoItem);
g_object_set(sink, "widget", videoItem, NULL);
QDirIterator it(":", QDirIterator::Subdirectories);
while (it.hasNext()) {
qDebug() << it.next();
}
QFile f(":/mixer.qml");
if(!f.open(QIODevice::ReadOnly)) {
qWarning() << "error: " << f.errorString();
return 1;
}
QByteArray overlay_scene = f.readAll();
qDebug() << overlay_scene;
/* load qmlgloverlay contents */
g_signal_connect (mixer, "qml-scene-initialized", G_CALLBACK (on_mixer_scene_initialized), NULL);
g_object_set (mixer, "qml-scene", overlay_scene.data(), NULL);
rootObject->scheduleRenderJob (new SetPlaying (pipeline),
QQuickWindow::BeforeSynchronizingStage);
ret = app.exec();
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
}
gst_deinit ();
return ret;
}

View file

@ -0,0 +1,38 @@
import QtQuick 6.0
import QtQuick.Controls 6.0
import QtQuick.Dialogs 6.0
import QtQuick.Window 6.0
import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
ApplicationWindow {
id: window
visible: true
width: 640
height: 480
x: 30
y: 30
color: "black"
Item {
anchors.fill: parent
GstGLQt6VideoItem {
id: video
objectName: "videoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
anchors.bottom: parent.bottom
anchors.horizontalCenter: parent.horizontalCenter
text: "qmlglsink text"
font.pointSize: 20
color: "yellow"
style: Text.Outline
styleColor: "blue"
}
}
}

View file

@ -0,0 +1,11 @@
sources = [
'main.cpp',
]
qt_preprocessed = qt6_mod.preprocess(qresources : 'qmlmixer.qrc')
executable('qml6glmixer', sources, qt_preprocessed,
dependencies : [gst_dep, qt6qml_example_deps],
override_options : ['cpp_std=c++17'],
c_args : gst_plugins_good_args,
include_directories : [configinc],
install: false)

View file

@ -0,0 +1,67 @@
import QtQuick 6.0
import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
Item {
/* render upside down for GStreamer */
transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 }
GstGLQt6VideoItem {
id: video0
objectName: "inputVideoItem0"
anchors.left: parent.left
anchors.right: parent.horizontalCenter
width: parent.width / 2
height: parent.height
}
GstGLQt6VideoItem {
id: video1
objectName: "inputVideoItem1"
anchors.left: parent.horizontalCenter
anchors.right: parent.right
width: parent.width / 2
height: parent.height
}
Text {
id: rotatingText
anchors.centerIn: parent
text: "Qt Quick\nrendered to\na texture"
font.pointSize: 20
color: "black"
style: Text.Outline
styleColor: "white"
RotationAnimator {
target: rotatingText;
from: 0;
to: 360;
duration: 5000
running: true
loops: Animation.Infinite
}
}
Text {
property int elapsedTime: 0
id: time
anchors.top: rotatingText.bottom
anchors.horizontalCenter: rotatingText.horizontalCenter
font.pointSize: 12
style: Text.Outline
styleColor: "black"
color: "white"
Timer {
interval: 1000
running: true
repeat: true
onTriggered: {
parent.elapsedTime += interval / 1000
parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds"
}
}
}
}

View file

@ -0,0 +1,6 @@
<RCC>
<qresource prefix="/">
<file>main.qml</file>
<file>mixer.qml</file>
</qresource>
</RCC>