qml6: implement qml6gloverlay

Based on the Qt5 version of qmlgloverlay.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/3845>
This commit is contained in:
Matthew Waters 2023-01-24 15:58:24 +11:00 committed by GStreamer Marge Bot
parent aeb0fa9993
commit 659c45ee7e
16 changed files with 1866 additions and 53 deletions

View file

@ -31,6 +31,7 @@ plugin_init (GstPlugin * plugin)
ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin); ret |= GST_ELEMENT_REGISTER (qml6glsink, plugin);
ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin); ret |= GST_ELEMENT_REGISTER (qml6glsrc, plugin);
ret |= GST_ELEMENT_REGISTER (qml6gloverlay, plugin);
return ret; return ret;
} }

View file

@ -0,0 +1,519 @@
/*
* GStreamer
* Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:gstqml6gloverlay
*
* `qml6gloverlay` provides a way to render an almost-arbitrary QML scene within
* GStreamer pipeline using the same OpenGL context that GStreamer uses
* internally. This avoids attempting to share multiple OpenGL contexts
* avoiding increased synchronisation points and attempting to share an OpenGL
* context at runtime which some drivers do not like. The Intel driver on
* Windows is a notable example of the last point.
*
* `qml6gloverlay` will attempt to retrieve the windowing system display connection
* that Qt is using (#GstGLDisplay). This may be different to any already
* existing window system display connection already in use in the pipeline for
* a number of reasons. A couple of examples of this are:
*
* 1. Adding `qml6gloverlay` to an already running pipeline
* 2. Not having any `qml6gloverlay` (or `qml6glsink`) element start up before any
* other OpenGL-based element in the pipeline.
*
* If one of these scenarios occurs, then there will be multiple OpenGL contexts
* in use in the pipeline. This means that either the pipeline will fail to
* start up correctly, a downstream element may reject buffers, or a complete
* GPU->System memory->GPU transfer is performed for every buffer.
*
* The requirement to avoid this is that all elements share the same
* #GstGLDisplay object and as Qt cannot currently share an existing window
* system display connection, GStreamer must use the window system display
* connection provided by Qt. This window system display connection can be
* retrieved by either a `qml6glsink` element or a `qml6gloverlay` element. The
* recommended usage is to have either element (`qml6glsink` or `qml6gloverlay`)
* be the first to propagate the #GstGLDisplay for the entire pipeline to use by
* setting either element to the READY element state before any other OpenGL
* element in the pipeline.
*
* In a dynamically adding `qml6gloverlay` (or `qml6glsink`) to a pipeline case,
* there are some considerations for ensuring that the window system display
* and OpenGL contexts are compatible with Qt. When the `qml6gloverlay` (or
* `qml6glsink`) element is added and brought up to READY, it will propagate it's
* own #GstGLDisplay using the #GstContext mechanism regardless of any existing
* #GstGLDisplay used by the pipeline previously. In order for the new
* #GstGLDisplay to be used, the application must then set the provided
* #GstGLDisplay containing #GstContext on the pipeline. This may effectively
* cause each OpenGL element to replace the window system display and also the
* OpenGL context it is using. As such this process may take a significant
* amount of time and resources as objects are recreated in the new OpenGL
* context.
*
* All instances of `qml6gloverlay` and `qml6glsink` will return the exact same
* #GstGLDisplay object while the pipeline is running regardless of whether
* any `qml6gloverlay` or `qml6glsink` elements are added or removed from the
* pipeline.
*
* The Qml scene will run at the pace of incoming buffers. One input buffer
* will cause a render of one output buffer. The timestamps on the input
* buffers are used to drive the animation time.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstqt6elements.h"
#include "gstqml6gloverlay.h"
#include "qt6glrenderer.h"
#include "gstqt6glutility.h"
#include <QtGui/QGuiApplication>
#include <gst/gl/gstglfuncs.h>
#define GST_CAT_DEFAULT gst_debug_qml6_gl_overlay
GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
static void gst_qml6_gl_overlay_finalize (GObject * object);
static void gst_qml6_gl_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * param_spec);
static void gst_qml6_gl_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * param_spec);
static gboolean gst_qml6_gl_overlay_gl_start (GstGLBaseFilter * bfilter);
static void gst_qml6_gl_overlay_gl_stop (GstGLBaseFilter * bfilter);
static gboolean gst_qml6_gl_overlay_gl_set_caps (GstGLBaseFilter * bfilter,
GstCaps * in_caps, GstCaps * out_caps);
static GstFlowReturn gst_qml6_gl_overlay_prepare_output_buffer (GstBaseTransform * btrans,
GstBuffer * buffer, GstBuffer ** outbuf);
static GstFlowReturn gst_qml6_gl_overlay_transform (GstBaseTransform * btrans,
GstBuffer * inbuf, GstBuffer * outbuf);
static GstStateChangeReturn gst_qml6_gl_overlay_change_state (GstElement * element,
GstStateChange transition);
enum
{
PROP_0,
PROP_WIDGET,
PROP_QML_SCENE,
PROP_ROOT_ITEM,
};
enum
{
SIGNAL_0,
SIGNAL_QML_SCENE_INITIALIZED,
SIGNAL_QML_SCENE_DESTROYED,
LAST_SIGNAL
};
static guint gst_qml6_gl_overlay_signals[LAST_SIGNAL] = { 0 };
#define gst_qml6_gl_overlay_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstQml6GLOverlay, gst_qml6_gl_overlay,
GST_TYPE_GL_FILTER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
"qml6gloverlay", 0, "Qt6 Video Overlay"));
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (qml6gloverlay, "qml6gloverlay",
GST_RANK_NONE, GST_TYPE_QML6_GL_OVERLAY, qt6_element_init (plugin));
static void
gst_qml6_gl_overlay_class_init (GstQml6GLOverlayClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *btrans_class;
GstGLBaseFilterClass *glbasefilter_class;
GstGLFilterClass *glfilter_class;
GstElementClass *element_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
glbasefilter_class = (GstGLBaseFilterClass *) klass;
glfilter_class = (GstGLFilterClass *) klass;
btrans_class = (GstBaseTransformClass *) klass;
element_class = (GstElementClass *) klass;
gobject_class->set_property = gst_qml6_gl_overlay_set_property;
gobject_class->get_property = gst_qml6_gl_overlay_get_property;
gobject_class->finalize = gst_qml6_gl_overlay_finalize;
gst_element_class_set_metadata (gstelement_class, "Qt Video Overlay",
"Filter/QML/Overlay", "A filter that renders a QML scene onto a video stream",
"Matthew Waters <matthew@centricular.com>");
g_object_class_install_property (gobject_class, PROP_QML_SCENE,
g_param_spec_string ("qml-scene", "QML Scene",
"The contents of the QML scene", NULL,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_WIDGET,
g_param_spec_pointer ("widget", "QQuickItem",
"The QQuickItem to place the input video in the object hierarchy",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_ROOT_ITEM,
g_param_spec_pointer ("root-item", "QQuickItem",
"The root QQuickItem from the qml-scene used to render",
(GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)));
/**
* GstQmlGLOverlay::qml-scene-initialized
* @element: the #GstQmlGLOverlay
* @user_data: user provided data
*/
gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED] =
g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
/**
* GstQmlGLOverlay::qml-scene-destroyed
* @element: the #GstQmlGLOverlay
* @user_data: user provided data
*/
gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_DESTROYED] =
g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
gst_gl_filter_add_rgba_pad_templates (glfilter_class);
btrans_class->prepare_output_buffer = gst_qml6_gl_overlay_prepare_output_buffer;
btrans_class->transform = gst_qml6_gl_overlay_transform;
glbasefilter_class->gl_start = gst_qml6_gl_overlay_gl_start;
glbasefilter_class->gl_stop = gst_qml6_gl_overlay_gl_stop;
glbasefilter_class->gl_set_caps = gst_qml6_gl_overlay_gl_set_caps;
element_class->change_state = gst_qml6_gl_overlay_change_state;
}
static void
gst_qml6_gl_overlay_init (GstQml6GLOverlay * qml6_gl_overlay)
{
qml6_gl_overlay->widget = QSharedPointer<Qt6GLVideoItemInterface>();
qml6_gl_overlay->qml_scene = NULL;
}
static void
gst_qml6_gl_overlay_finalize (GObject * object)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object);
g_free (qml6_gl_overlay->qml_scene);
qml6_gl_overlay->qml_scene = NULL;
qml6_gl_overlay->widget.clear();
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_qml6_gl_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object);
switch (prop_id) {
case PROP_WIDGET: {
Qt6GLVideoItem *qt_item = static_cast<Qt6GLVideoItem *> (g_value_get_pointer (value));
if (qt_item)
qml6_gl_overlay->widget = qt_item->getInterface();
else
qml6_gl_overlay->widget.clear();
break;
}
case PROP_QML_SCENE:
g_free (qml6_gl_overlay->qml_scene);
qml6_gl_overlay->qml_scene = g_value_dup_string (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_qml6_gl_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (object);
switch (prop_id) {
case PROP_WIDGET:
/* This is not really safe - the app needs to be
* sure the widget is going to be kept alive or
* this can crash */
if (qml6_gl_overlay->widget)
g_value_set_pointer (value, qml6_gl_overlay->widget->videoItem());
else
g_value_set_pointer (value, NULL);
break;
case PROP_QML_SCENE:
g_value_set_string (value, qml6_gl_overlay->qml_scene);
break;
case PROP_ROOT_ITEM:
GST_OBJECT_LOCK (qml6_gl_overlay);
if (qml6_gl_overlay->renderer) {
QQuickItem *root = qml6_gl_overlay->renderer->rootItem();
if (root)
g_value_set_pointer (value, root);
else
g_value_set_pointer (value, NULL);
} else {
g_value_set_pointer (value, NULL);
}
GST_OBJECT_UNLOCK (qml6_gl_overlay);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_qml6_gl_overlay_gl_start (GstGLBaseFilter * bfilter)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter);
QQuickItem *root;
GError *error = NULL;
GST_TRACE_OBJECT (bfilter, "using scene:\n%s", qml6_gl_overlay->qml_scene);
if (!qml6_gl_overlay->qml_scene || g_strcmp0 (qml6_gl_overlay->qml_scene, "") == 0) {
GST_ELEMENT_ERROR (bfilter, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL));
return FALSE;
}
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (bfilter))
return FALSE;
GST_OBJECT_LOCK (bfilter);
qml6_gl_overlay->renderer = new GstQt6QuickRenderer;
if (!qml6_gl_overlay->renderer->init (bfilter->context, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
delete qml6_gl_overlay->renderer;
qml6_gl_overlay->renderer = NULL;
GST_OBJECT_UNLOCK (bfilter);
return FALSE;
}
/* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */
if (!qml6_gl_overlay->renderer->setQmlScene (qml6_gl_overlay->qml_scene, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
goto fail_renderer;
return FALSE;
}
root = qml6_gl_overlay->renderer->rootItem();
if (!root) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("Qml scene does not have a root item"), (NULL));
goto fail_renderer;
}
GST_OBJECT_UNLOCK (bfilter);
g_object_notify (G_OBJECT (qml6_gl_overlay), "root-item");
g_signal_emit (qml6_gl_overlay, gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED], 0);
GST_OBJECT_LOCK (bfilter);
if (!qml6_gl_overlay->widget) {
Qt6GLVideoItem *qt_item = static_cast<Qt6GLVideoItem *>(root->findChild<Qt6GLVideoItem *> ());
if (qt_item)
qml6_gl_overlay->widget = qt_item->getInterface();
}
GST_OBJECT_UNLOCK (bfilter);
return TRUE;
fail_renderer:
{
qml6_gl_overlay->renderer->cleanup();
delete qml6_gl_overlay->renderer;
qml6_gl_overlay->renderer = NULL;
GST_OBJECT_UNLOCK (bfilter);
return FALSE;
}
}
static void
gst_qml6_gl_overlay_gl_stop (GstGLBaseFilter * bfilter)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter);
GstQt6QuickRenderer *renderer = NULL;
/* notify before actually destroying anything */
GST_OBJECT_LOCK (qml6_gl_overlay);
if (qml6_gl_overlay->renderer)
renderer = qml6_gl_overlay->renderer;
qml6_gl_overlay->renderer = NULL;
GST_OBJECT_UNLOCK (qml6_gl_overlay);
g_signal_emit (qml6_gl_overlay, gst_qml6_gl_overlay_signals[SIGNAL_QML_SCENE_DESTROYED], 0);
g_object_notify (G_OBJECT (qml6_gl_overlay), "root-item");
if (qml6_gl_overlay->widget)
qml6_gl_overlay->widget->setBuffer (NULL);
if (renderer) {
renderer->cleanup();
delete renderer;
}
GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (bfilter);
}
static gboolean
gst_qml6_gl_overlay_gl_set_caps (GstGLBaseFilter * bfilter, GstCaps * in_caps,
GstCaps * out_caps)
{
GstGLFilter *filter = GST_GL_FILTER (bfilter);
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (bfilter);
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_set_caps (bfilter, in_caps, out_caps))
return FALSE;
qml6_gl_overlay->renderer->setSize (GST_VIDEO_INFO_WIDTH (&filter->out_info),
GST_VIDEO_INFO_HEIGHT (&filter->out_info));
return TRUE;
}
static GstFlowReturn
gst_qml6_gl_overlay_prepare_output_buffer (GstBaseTransform * btrans,
GstBuffer * buffer, GstBuffer ** outbuf)
{
GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (btrans);
GstGLBaseFilter *bfilter = GST_GL_BASE_FILTER (btrans);
GstGLFilter *filter = GST_GL_FILTER (btrans);
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (btrans);
GstGLMemory *out_mem;
GstGLSyncMeta *sync_meta;
if (gst_buffer_n_memory (buffer) <= 0) {
GST_ELEMENT_ERROR (btrans, RESOURCE, NOT_FOUND,
(NULL), ("Buffer must have a memory object"));
return GST_FLOW_ERROR;
}
if (qml6_gl_overlay->widget) {
GstMemory *mem;
GstGLMemory *gl_mem;
qml6_gl_overlay->widget->setCaps (bfilter->in_caps);
mem = gst_buffer_peek_memory (buffer, 0);
if (!gst_is_gl_memory (mem)) {
GST_ELEMENT_ERROR (btrans, RESOURCE, NOT_FOUND,
(NULL), ("Input memory must be a GstGLMemory"));
return GST_FLOW_ERROR;
}
gl_mem = (GstGLMemory *) mem;
if (!gst_gl_context_can_share (gl_mem->mem.context, bfilter->context)) {
GST_WARNING_OBJECT (bfilter, "Cannot use the current input texture "
"(input buffer GL context %" GST_PTR_FORMAT " cannot share "
"resources with the configured OpenGL context %" GST_PTR_FORMAT ")",
gl_mem->mem.context, bfilter->context);
} else {
qml6_gl_overlay->widget->setBuffer (buffer);
}
}
/* XXX: is this the correct ts to drive the animation */
out_mem = qml6_gl_overlay->renderer->generateOutput (GST_BUFFER_PTS (buffer));
if (!out_mem) {
GST_ERROR_OBJECT (qml6_gl_overlay, "Failed to generate output");
return GST_FLOW_ERROR;
}
*outbuf = gst_buffer_new ();
gst_buffer_append_memory (*outbuf, (GstMemory *) out_mem);
gst_buffer_add_video_meta (*outbuf, (GstVideoFrameFlags) 0,
GST_VIDEO_INFO_FORMAT (&filter->out_info),
GST_VIDEO_INFO_WIDTH (&filter->in_info),
GST_VIDEO_INFO_HEIGHT (&filter->out_info));
sync_meta = gst_buffer_add_gl_sync_meta (bfilter->context, *outbuf);
gst_gl_sync_meta_set_sync_point (sync_meta, bfilter->context);
bclass->copy_metadata (btrans, buffer, *outbuf);
return GST_FLOW_OK;
}
static GstFlowReturn
gst_qml6_gl_overlay_transform (GstBaseTransform * btrans, GstBuffer * inbuf,
GstBuffer * outbuf)
{
return GST_FLOW_OK;
}
static GstStateChangeReturn
gst_qml6_gl_overlay_change_state (GstElement * element,
GstStateChange transition)
{
GstQml6GLOverlay *qml6_gl_overlay = GST_QML6_GL_OVERLAY (element);
GstGLBaseFilter *filter = GST_GL_BASE_FILTER (element);
GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;
GST_DEBUG_OBJECT (filter, "changing state: %s => %s",
gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));
switch (transition) {
case GST_STATE_CHANGE_NULL_TO_READY: {
QGuiApplication *app;
GstGLDisplay *display = NULL;
app = static_cast<QGuiApplication *> (QCoreApplication::instance ());
if (!app) {
GST_ELEMENT_ERROR (element, RESOURCE, NOT_FOUND,
("%s", "Failed to connect to Qt"),
("%s", "Could not retrieve QGuiApplication instance"));
return GST_STATE_CHANGE_FAILURE;
}
display = gst_qml6_get_gl_display (FALSE);
if (display != filter->display)
/* always propagate. The application may need to choose between window
* system display connections */
gst_gl_element_propagate_display_context (GST_ELEMENT (qml6_gl_overlay), display);
gst_object_unref (display);
break;
}
default:
break;
}
ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
if (ret == GST_STATE_CHANGE_FAILURE)
return ret;
switch (transition) {
default:
break;
}
return ret;
}

View file

@ -0,0 +1,76 @@
/*
* GStreamer
* Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_QML6_GL_OVERLAY_H__
#define __GST_QML6_GL_OVERLAY_H__
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include "qt6glrenderer.h"
#include "qt6glitem.h"
typedef struct _GstQml6GLOverlay GstQml6GLOverlay;
typedef struct _GstQml6GLOverlayClass GstQml6GLOverlayClass;
typedef struct _GstQml6GLOverlayPrivate GstQml6GLOverlayPrivate;
G_BEGIN_DECLS
GType gst_qml6_gl_overlay_get_type (void);
#define GST_TYPE_QML6_GL_OVERLAY (gst_qml6_gl_overlay_get_type())
#define GST_QML6_GL_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_QML6_GL_OVERLAY,GstQml6GLOverlay))
#define GST_QML6_GL_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_QML6_GL_OVERLAY,GstQml6GLOverlayClass))
#define GST_IS_QML6_GL_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_QML6_GL_OVERLAY))
#define GST_IS_QML6_GL_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_QML6_GL_OVERLAY))
#define GST_QML6_GL_OVERLAY_CAST(obj) ((GstQml6GLOverlay*)(obj))
/**
* GstQml6GLOverlay:
*
* Opaque #GstQml6GLOverlay object
*/
struct _GstQml6GLOverlay
{
/* <private> */
GstGLFilter parent;
gchar *qml_scene;
GstQt6QuickRenderer *renderer;
QSharedPointer<Qt6GLVideoItemInterface> widget;
};
/**
* GstQml6GLOverlayClass:
*
* The #GstQml6GLOverlayClass struct only contains private data
*/
struct _GstQml6GLOverlayClass
{
/* <private> */
GstGLFilterClass parent_class;
};
GstQml6GLOverlay * gst_qml6_gl_overlay_new (void);
G_END_DECLS
#endif /* __GST_QML6_GL_OVERLAY_H__ */

View file

@ -28,6 +28,7 @@ void qt6_element_init (GstPlugin * plugin);
GST_ELEMENT_REGISTER_DECLARE (qml6glsink); GST_ELEMENT_REGISTER_DECLARE (qml6glsink);
GST_ELEMENT_REGISTER_DECLARE (qml6glsrc); GST_ELEMENT_REGISTER_DECLARE (qml6glsrc);
GST_ELEMENT_REGISTER_DECLARE (qml6gloverlay);
G_END_DECLS G_END_DECLS

View file

@ -292,40 +292,53 @@ gst_qml6_get_gl_wrapcontext (GstGLDisplay * display,
#endif #endif
return TRUE; return TRUE;
} }
#if 0
QVariant QOpenGLContext *
qt_opengl_native_context_from_gst_gl_context (GstGLContext * context) qt_opengl_native_context_from_gst_gl_context (GstGLContext * context)
{ {
guintptr handle; guintptr handle;
GstGLPlatform platform; GstGLPlatform platform;
QOpenGLContext *ret = NULL;
handle = gst_gl_context_get_gl_context (context); handle = gst_gl_context_get_gl_context (context);
platform = gst_gl_context_get_gl_platform (context); platform = gst_gl_context_get_gl_platform (context);
/* this is required as Qt doesn't allow retrieving the relevant native
* interface unless the underlying context has been created */
QOpenGLContext *qt_gl_context = new QOpenGLContext();
qt_gl_context->create();
#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11) #if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
if (platform == GST_GL_PLATFORM_GLX) { if (!ret && platform == GST_GL_PLATFORM_GLX) {
auto glx = qt_gl_context->nativeInterface<QNativeInterface::QGLXContext>();
if (!glx) {
GST_WARNING ("Retriving GLX context interface from Qt failed");
} else {
GstGLDisplay *display = gst_gl_context_get_display (context); GstGLDisplay *display = gst_gl_context_get_display (context);
GstGLWindow *window = gst_gl_context_get_window (context); GstGLWindow *window = gst_gl_context_get_window (context);
Display *xdisplay = (Display *) gst_gl_display_get_handle (display);
Window win = gst_gl_window_get_window_handle (window);
gst_object_unref (window); gst_object_unref (window);
gst_object_unref (display); gst_object_unref (display);
return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay, win)); ret = glx->fromNative((GLXContext) handle);
}
} }
#endif #endif
#if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID)) #if GST_GL_HAVE_PLATFORM_EGL && (defined (HAVE_QT_WAYLAND) || defined (HAVE_QT_EGLFS) || defined (HAVE_QT_ANDROID))
if (platform == GST_GL_PLATFORM_EGL) { if (!ret && platform == GST_GL_PLATFORM_EGL) {
auto egl = qt_gl_context->nativeInterface<QNativeInterface::QEGLContext>();
if (!egl) {
GST_WARNING ("Retriving EGL context interface from Qt failed");
} else {
EGLDisplay egl_display = EGL_DEFAULT_DISPLAY; EGLDisplay egl_display = EGL_DEFAULT_DISPLAY;
GstGLDisplay *display = gst_gl_context_get_display (context); GstGLDisplay *display = gst_gl_context_get_display (context);
GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display); GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display);
#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND) #if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND)
if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) { if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) {
#if 1 #if 0
g_warning ("Qt does not support wrapping native OpenGL contexts " g_warning ("Qt does not support wrapping native OpenGL contexts "
"on wayland. See https://bugreports.qt.io/browse/QTBUG-82528"); "on wayland. See https://bugreports.qt.io/browse/QTBUG-82528");
gst_object_unref (display_egl); gst_object_unref (display_egl);
gst_object_unref (display); gst_object_unref (display);
return QVariant::fromValue(nullptr); return NULL;
#else #else
if (display_egl) if (display_egl)
egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl); egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl);
@ -334,26 +347,42 @@ qt_opengl_native_context_from_gst_gl_context (GstGLContext * context)
#endif #endif
gst_object_unref (display_egl); gst_object_unref (display_egl);
gst_object_unref (display); gst_object_unref (display);
return QVariant::fromValue(QEGLNativeContext((EGLContext) handle, egl_display)); GST_ERROR ("creating native context from context %p and display %p", (void *) handle, egl_display);
ret = egl->fromNative((EGLContext) handle, egl_display);
GST_ERROR ("created native context %p", ret);
}
} }
#endif #endif
#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32) #if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32)
if (platform == GST_GL_PLATFORM_WGL) { if (!ret && platform == GST_GL_PLATFORM_WGL) {
auto wgl = qt_gl_context->nativeInterface<QNativeInterface::QWGLContext>();
if (!wgl) {
GST_WARNING ("Retriving WGL context interface from Qt failed");
} else {
GstGLWindow *window = gst_gl_context_get_window (context); GstGLWindow *window = gst_gl_context_get_window (context);
guintptr hwnd = gst_gl_window_get_window_handle (window); guintptr hwnd = gst_gl_window_get_window_handle (window);
gst_object_unref (window); gst_object_unref (window);
return QVariant::fromValue(QWGLNativeContext((HGLRC) handle, (HWND) hwnd)); ret = wgl->fromNative((HGLRC) handle, (HWND) hwnd);
}
} }
#endif #endif
{ if (!ret) {
gchar *platform_s = gst_gl_platform_to_string (platform); gchar *platform_s = gst_gl_platform_to_string (platform);
g_warning ("Unimplemented configuration! This means either:\n" g_warning ("Unimplemented configuration! This means either:\n"
"1. The qmlgl plugin was built without support for your platform.\n" "1. Qt6 wasn't built with support for \'%s\'\n"
"2. The necessary code to convert from a GstGLContext to Qt's " "2. The qmlgl plugin was built without support for your platform.\n"
"native context type for \'%s\' currently does not exist.", "3. The necessary code to convert from a GstGLContext to Qt's "
platform_s); "native context type for \'%s\' currently does not exist."
"4. Qt failed to wrap an existing native context.",
platform_s, platform_s);
g_free (platform_s); g_free (platform_s);
} }
return QVariant::fromValue(nullptr);
qt_gl_context->doneCurrent();
delete qt_gl_context;
gst_gl_context_activate (context, FALSE);
gst_gl_context_activate (context, TRUE);
return ret;
} }
#endif

View file

@ -26,6 +26,7 @@
#include <QVariant> #include <QVariant>
#include <QRunnable> #include <QRunnable>
#include <QOpenGLContext>
G_BEGIN_DECLS G_BEGIN_DECLS
@ -46,10 +47,6 @@ gboolean gst_qml6_get_gl_wrapcontext (GstGLDisplay * display,
G_END_DECLS G_END_DECLS
#if 0 QOpenGLContext * qt_opengl_native_context_from_gst_gl_context (GstGLContext * context);
#if defined(__cplusplus)
QVariant qt_opengl_native_context_from_gst_gl_context (GstGLContext * context);
#endif
#endif
#endif /* __QML6_GL_UTILS_H__ */ #endif /* __QML6_GL_UTILS_H__ */

View file

@ -5,14 +5,17 @@ sources = [
'gstqt6glutility.cc', 'gstqt6glutility.cc',
'gstqml6glsink.cc', 'gstqml6glsink.cc',
'gstqml6glsrc.cc', 'gstqml6glsrc.cc',
'gstqml6gloverlay.cc',
'qt6glitem.cc', 'qt6glitem.cc',
'qt6glwindow.cc', 'qt6glwindow.cc',
'qt6glrenderer.cc',
] ]
moc_headers = [ moc_headers = [
'qt6glitem.h', 'qt6glitem.h',
'gstqsg6glnode.h', 'gstqsg6glnode.h',
'qt6glwindow.h', 'qt6glwindow.h',
'qt6glrenderer.h',
] ]
qt6qml_dep = dependency('', required: false) qt6qml_dep = dependency('', required: false)

View file

@ -0,0 +1,765 @@
#include <QObject>
#include <QQmlEngine>
#include <QQmlComponent>
#include <QWindow>
#include <QQuickRenderControl>
#include <QQuickWindow>
#include <QQuickGraphicsDevice>
#include <QQuickItem>
#include <QQuickRenderTarget>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QAnimationDriver>
#include <gst/gl/gl.h>
#include "gstqt6gl.h"
#include "qt6glrenderer.h"
#include "gstqt6glutility.h"
#define GST_CAT_DEFAULT gst_qt6_gl_renderer_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
static void
init_debug (void)
{
static gsize _debug;
if (g_once_init_enter (&_debug)) {
GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qt6glrenderer", 0,
"Qt6 OpenGL Renderer");
g_once_init_leave (&_debug, 1);
}
}
/* Needs to be based on QWindow otherwise (at least) windows and nvidia
* proprietary on linux does not work
* We also need to override the size handling to get the correct output size
*/
class GstQt6BackingSurface : public QWindow
{
public:
GstQt6BackingSurface();
~GstQt6BackingSurface();
void setSize (int width, int height);
QSize size() const override;
private:
QSize m_size;
};
GstQt6BackingSurface::GstQt6BackingSurface()
: m_size(QSize())
{
/* we do OpenGL things so need an OpenGL surface */
setSurfaceType(QSurface::OpenGLSurface);
}
GstQt6BackingSurface::~GstQt6BackingSurface()
{
}
QSize GstQt6BackingSurface::size () const
{
return m_size;
}
void GstQt6BackingSurface::setSize (int width, int height)
{
m_size = QSize (width, height);
}
class GstQt6AnimationDriver : public QAnimationDriver
{
public:
GstQt6AnimationDriver();
void setNextTime(qint64 ms);
void advance() override;
qint64 elapsed() const override;
private:
qint64 m_elapsed;
qint64 m_next;
};
GstQt6AnimationDriver::GstQt6AnimationDriver()
: m_elapsed(0),
m_next(0)
{
}
void GstQt6AnimationDriver::advance()
{
m_elapsed = m_next;
advanceAnimation();
}
qint64 GstQt6AnimationDriver::elapsed() const
{
return m_elapsed;
}
void GstQt6AnimationDriver::setNextTime(qint64 ms)
{
m_next = ms;
}
typedef enum
{
STATE_ERROR = -1,
STATE_NEW = 0,
STATE_WAITING_FOR_WINDOW,
STATE_WINDOW_CREATED,
STATE_READY,
} SharedRenderDataState;
struct SharedRenderData
{
int refcount;
SharedRenderDataState state;
GMutex lock;
GCond cond;
GstQt6AnimationDriver *m_animationDriver;
QOpenGLContext *m_context;
GstQt6BackingSurface *m_surface;
QThread *m_renderThread;
};
static struct SharedRenderData *
shared_render_data_new (void)
{
struct SharedRenderData *ret = g_new0 (struct SharedRenderData, 1);
g_atomic_int_set (&ret->refcount, 1);
g_mutex_init (&ret->lock);
return ret;
}
static void
shared_render_data_free (struct SharedRenderData * data)
{
GST_DEBUG ("%p freeing shared render data", data);
g_mutex_clear (&data->lock);
if (data->m_animationDriver) {
data->m_animationDriver->uninstall();
delete data->m_animationDriver;
}
data->m_animationDriver = nullptr;
if (data->m_context) {
if (QOpenGLContext::currentContext() == data->m_context)
data->m_context->doneCurrent();
delete data->m_context;
}
data->m_context = nullptr;
if (data->m_surface)
delete data->m_surface;
data->m_surface = nullptr;
}
static struct SharedRenderData *
shared_render_data_ref (struct SharedRenderData * data)
{
GST_TRACE ("%p reffing shared render data", data);
g_atomic_int_inc (&data->refcount);
return data;
}
static void
shared_render_data_unref (struct SharedRenderData * data)
{
GST_TRACE ("%p unreffing shared render data", data);
if (g_atomic_int_dec_and_test (&data->refcount))
shared_render_data_free (data);
}
void
GstQt6QuickRenderer::deactivateContext ()
{
}
void
GstQt6QuickRenderer::activateContext ()
{
}
struct FBOUserData
{
GstGLContext * context;
QOpenGLFramebufferObject * fbo;
};
GstQt6QuickRenderer::GstQt6QuickRenderer()
: gl_context(NULL),
m_quickWindow(nullptr),
m_renderControl(nullptr),
m_qmlEngine(nullptr),
m_qmlComponent(nullptr),
m_rootItem(nullptr),
gl_allocator(NULL),
gl_params(NULL),
gl_mem(NULL),
m_sharedRenderData(NULL)
{
init_debug ();
}
static gpointer
dup_shared_render_data (gpointer data, gpointer user_data)
{
struct SharedRenderData *render_data = (struct SharedRenderData *) data;
if (render_data)
return shared_render_data_ref (render_data);
return NULL;
}
class CreateSurfaceEvent : public QEvent
{
public:
CreateSurfaceEvent (CreateSurfaceWorker * worker)
: QEvent(CreateSurfaceEvent::type())
{
m_worker = worker;
}
~CreateSurfaceEvent()
{
GST_TRACE ("%p destroying create surface event", this);
delete m_worker;
}
static QEvent::Type type()
{
if (customEventType == QEvent::None) {
int generatedType = QEvent::registerEventType();
customEventType = static_cast<QEvent::Type>(generatedType);
}
return customEventType;
}
private:
static QEvent::Type customEventType;
CreateSurfaceWorker *m_worker;
};
QEvent::Type CreateSurfaceEvent::customEventType = QEvent::None;
CreateSurfaceWorker::CreateSurfaceWorker (struct SharedRenderData * rdata)
{
m_sharedRenderData = shared_render_data_ref (rdata);
}
CreateSurfaceWorker::~CreateSurfaceWorker ()
{
shared_render_data_unref (m_sharedRenderData);
}
bool CreateSurfaceWorker::event(QEvent * ev)
{
if (ev->type() == CreateSurfaceEvent::type()) {
GST_TRACE ("%p creating surface", m_sharedRenderData);
/* create the window surface in the main thread */
g_mutex_lock (&m_sharedRenderData->lock);
m_sharedRenderData->m_surface = new GstQt6BackingSurface;
m_sharedRenderData->m_surface->create();
m_sharedRenderData->m_surface->moveToThread (m_sharedRenderData->m_renderThread);
GST_TRACE ("%p created surface %p", m_sharedRenderData,
m_sharedRenderData->m_surface);
g_cond_broadcast (&m_sharedRenderData->cond);
g_mutex_unlock (&m_sharedRenderData->lock);
}
return QObject::event(ev);
}
bool GstQt6QuickRenderer::init (GstGLContext * context, GError ** error)
{
g_return_val_if_fail (GST_IS_GL_CONTEXT (context), false);
g_return_val_if_fail (gst_gl_context_get_current () == context, false);
QOpenGLContext *qt_native_context = qt_opengl_native_context_from_gst_gl_context (context);
if (!qt_native_context) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not convert from the provided GstGLContext to a Qt "
"native context");
return false;
}
struct SharedRenderData *render_data = NULL, *old_render_data;
do {
if (render_data)
shared_render_data_unref (render_data);
old_render_data = render_data = (struct SharedRenderData *)
g_object_dup_data (G_OBJECT (context),
"qt.gl.render.shared.data", dup_shared_render_data, NULL);
if (!render_data)
render_data = shared_render_data_new ();
} while (old_render_data != render_data
&& !g_object_replace_data (G_OBJECT (context),
"qt.gl.render.shared.data", old_render_data, render_data,
NULL, NULL));
m_sharedRenderData = render_data;
GST_TRACE ("%p retrieved shared render data %p", this, m_sharedRenderData);
g_mutex_lock (&m_sharedRenderData->lock);
if (m_sharedRenderData->state == STATE_ERROR) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"In an error state from a previous attempt");
g_mutex_unlock (&m_sharedRenderData->lock);
return false;
}
if (m_sharedRenderData->state != STATE_READY) {
/* this state handling and locking is so that two qtglrenderer's will
* not attempt to create an OpenGL context without freeing the previous
* OpenGL context and cause a leak. It also only allows one
* CreateSurfaceEvent() to be posted to the main thread
* (QCoreApplication::instance()->thread()) while still allowing
* multiple waiters to wait for the window to be created */
if (m_sharedRenderData->state == STATE_NEW) {
QCoreApplication *app = QCoreApplication::instance ();
if (!app) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not retrieve QCoreApplication instance");
m_sharedRenderData->state = STATE_ERROR;
g_mutex_unlock (&m_sharedRenderData->lock);
return false;
}
m_sharedRenderData->m_renderThread = QThread::currentThread();
m_sharedRenderData->m_context = qt_native_context;
GST_TRACE ("%p new QOpenGLContext %p", this, m_sharedRenderData->m_context);
CreateSurfaceWorker *w = new CreateSurfaceWorker (m_sharedRenderData);
GST_TRACE ("%p posting create surface event to main thread with "
"worker %p", this, w);
w->moveToThread (app->thread());
app->postEvent (w, new CreateSurfaceEvent (w));
m_sharedRenderData->state = STATE_WAITING_FOR_WINDOW;
}
if (m_sharedRenderData->state == STATE_WAITING_FOR_WINDOW) {
gint64 end_time = g_get_monotonic_time () + 5 * G_TIME_SPAN_SECOND;
while (!m_sharedRenderData->m_surface) {
/* XXX: This might deadlock with the main thread if the
* QCoreApplication is not running and will not be able to
* execute. We only wait for 5 seconds until a better
* approach can be found here */
if (!g_cond_wait_until (&m_sharedRenderData->cond,
&m_sharedRenderData->lock, end_time)) {
g_set_error (error, GST_RESOURCE_ERROR,
GST_RESOURCE_ERROR_NOT_FOUND,
"Could not create Qt window within 5 seconds");
m_sharedRenderData->state = STATE_ERROR;
g_mutex_unlock (&m_sharedRenderData->lock);
return false;
}
}
GST_TRACE ("%p surface successfully created", this);
m_sharedRenderData->state = STATE_WINDOW_CREATED;
}
if (m_sharedRenderData->state == STATE_WINDOW_CREATED) {
/* Qt does some things that may require the OpenGL context current
* in ->create() so that it has the necessry information to create
* the QOpenGLContext from the native handle. This may fail if the
* OpenGL context is already current in another thread so we need
* to deactivate the context from GStreamer's thread before asking
* Qt to create the QOpenGLContext with ->create().
*/
gst_gl_context_activate (context, FALSE);
//m_sharedRenderData->m_context->create();
//m_sharedRenderData->m_context->doneCurrent();
if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not make Qt OpenGL context current");
/* try to keep the same OpenGL context state */
gst_gl_context_activate (context, TRUE);
m_sharedRenderData->state = STATE_ERROR;
g_mutex_unlock (&m_sharedRenderData->lock);
return false;
}
if (!gst_gl_context_activate (context, TRUE)) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not make GStreamer OpenGL context current again");
m_sharedRenderData->state = STATE_ERROR;
g_mutex_unlock (&m_sharedRenderData->lock);
return false;
}
m_sharedRenderData->state = STATE_READY;
}
}
m_renderControl = new QQuickRenderControl();
/* Create a QQuickWindow that is associated with our render control. Note that this
* window never gets created or shown, meaning that it will never get an underlying
* native (platform) window.
*/
m_quickWindow = new QQuickWindow(m_renderControl);
m_quickWindow->setGraphicsDevice(QQuickGraphicsDevice::fromOpenGLContext(qt_native_context));
/* after QQuickWindow creation as QQuickRenderControl requires it */
m_renderControl->prepareThread (m_sharedRenderData->m_renderThread);
g_mutex_unlock (&m_sharedRenderData->lock);
/* Create a QML engine. */
m_qmlEngine = new QQmlEngine;
if (!m_qmlEngine->incubationController())
m_qmlEngine->setIncubationController(m_quickWindow->incubationController());
/* TODO: use buffer pool */
gl_context = static_cast<GstGLContext*>(gst_object_ref (context));
gl_allocator = (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (gl_context);
gl_params = (GstGLAllocationParams *)
gst_gl_video_allocation_params_new (gl_context,
NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8);
/* This is a gross hack relying on the internals of Qt and GStreamer
* however it's the only way to remove this warning on shutdown of all
* resources.
*
* GLib-CRITICAL **: 17:35:24.988: g_main_context_pop_thread_default: assertion 'g_queue_peek_head (stack) == context' failed
*
* The reason is that libgstgl has a GMainContext that it pushes as the
* thread default context. Then later, Qt pushes a thread default main
* context. The detruction order of the GMainContext's is reversed as
* GStreamer will explicitly pop the thread default main context however
* Qt pops when the thread is about to be destroyed. GMainContext is
* unhappy with the ordering of the pops.
*/
GMainContext *gst_main_context = g_main_context_ref_thread_default ();
/* make Qt allocate and push a thread-default GMainContext if it is
* going to */
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("pending QEvents processed");
GMainContext *qt_main_context = g_main_context_ref_thread_default ();
if (qt_main_context == gst_main_context) {
g_main_context_unref (qt_main_context);
g_main_context_unref (gst_main_context);
} else {
/* We flip the order of the GMainContext's so that the destruction
* order can be preserved. */
g_main_context_pop_thread_default (qt_main_context);
g_main_context_pop_thread_default (gst_main_context);
g_main_context_push_thread_default (qt_main_context);
g_main_context_push_thread_default (gst_main_context);
g_main_context_unref (qt_main_context);
g_main_context_unref (gst_main_context);
}
return true;
}
GstQt6QuickRenderer::~GstQt6QuickRenderer()
{
gst_gl_allocation_params_free (gl_params);
gst_clear_object (&gl_allocator);
}
void GstQt6QuickRenderer::stopGL ()
{
QOpenGLContext *current_qt_context = QOpenGLContext::currentContext();
GST_DEBUG ("%p stop QOpenGLContext current: %p stored: %p", this,
current_qt_context, m_sharedRenderData->m_context);
/* Invalidating the renderer will cause Qt6 to clear the current qt-tracked OpenGL context.
* We however may be using the QOpenGLContext for multiple qml6gloverlay
* elements so need to recurrent it */
if (current_qt_context)
g_assert (current_qt_context == m_sharedRenderData->m_context);
else
m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface);
if (m_renderControl)
m_renderControl->invalidate();
GST_ERROR ("%p %p", this, QOpenGLContext::currentContext());
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("%p pending QEvents processed", this);
}
void GstQt6QuickRenderer::stopAfterGL ()
{
GST_DEBUG ("%p stop QOpenGLContext curent: %p stored: %p", this,
QOpenGLContext::currentContext(), m_sharedRenderData->m_context);
g_assert (QOpenGLContext::currentContext() == nullptr);
if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface))
g_warn_if_reached();
if (m_sharedRenderData)
shared_render_data_unref (m_sharedRenderData);
m_sharedRenderData = NULL;
/* XXX: reset the OpenGL context and drawable as Qt may have clobbered it.
* Fixes any attempt to access OpenGL after shutting down qmlgloverlay. */
gst_gl_context_activate (gl_context, FALSE);
gst_gl_context_activate (gl_context, TRUE);
}
void GstQt6QuickRenderer::cleanup()
{
if (gl_context)
gst_gl_context_thread_add (gl_context,
(GstGLContextThreadFunc) GstQt6QuickRenderer::stop_c, this);
/* Delete the render control first since it will free the scenegraph resources.
* Destroy the QQuickWindow only afterwards. */
if (m_renderControl)
delete m_renderControl;
m_renderControl = nullptr;
if (m_qmlComponent)
delete m_qmlComponent;
m_qmlComponent = nullptr;
if (m_quickWindow)
delete m_quickWindow;
m_quickWindow = nullptr;
if (m_qmlEngine)
delete m_qmlEngine;
m_qmlEngine = nullptr;
if (m_rootItem)
delete m_rootItem;
m_rootItem = nullptr;
if (gl_context)
gst_gl_context_thread_add (gl_context,
(GstGLContextThreadFunc) GstQt6QuickRenderer::stop_after_c, this);
gst_clear_object (&gl_context);
}
static QSize
gl_params_get_QSize(GstGLAllocationParams * gl_params)
{
GstGLVideoAllocationParams * gl_vid_params = (GstGLVideoAllocationParams *) gl_params;
if (!gl_vid_params)
return QSize (0, 0);
return QSize(GST_VIDEO_INFO_WIDTH (gl_vid_params->v_info), GST_VIDEO_INFO_HEIGHT(gl_vid_params->v_info));
}
void
GstQt6QuickRenderer::renderGstGL ()
{
// const GstGLFuncs *gl = gl_context->gl_vtable;
GST_TRACE ("%p current QOpenGLContext %p", this,
QOpenGLContext::currentContext());
m_sharedRenderData->m_animationDriver->advance();
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("pending QEvents processed");
loop.exit();
if (gl_params && gl_params_get_QSize(gl_params) != m_sharedRenderData->m_surface->size()) {
gst_gl_allocation_params_free(gl_params);
gl_params = NULL;
}
if (!gl_params)
gl_params = (GstGLAllocationParams *)
gst_gl_video_allocation_params_new (gl_context,
NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8);
gl_mem = (GstGLMemory *) gst_gl_base_memory_alloc (gl_allocator, gl_params);
m_quickWindow->setRenderTarget(QQuickRenderTarget::fromOpenGLTexture(gst_gl_memory_get_texture_id (gl_mem), gl_params_get_QSize(gl_params)));
m_renderControl->beginFrame();
if (m_renderControl->sync())
GST_LOG ("sync successful");
m_renderControl->render();
m_renderControl->endFrame();
/* Qt doesn't seem to reset this, breaking glimagesink */
// if (gl->DrawBuffer)
// gl->DrawBuffer (GL_BACK);
}
GstGLMemory *GstQt6QuickRenderer::generateOutput(GstClockTime input_ns)
{
m_sharedRenderData->m_animationDriver->setNextTime(input_ns / GST_MSECOND);
/* run an event loop to update any changed values for rendering */
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("pending QEvents processed");
GST_LOG ("generating output for time %" GST_TIME_FORMAT " ms: %"
G_GUINT64_FORMAT, GST_TIME_ARGS (input_ns), input_ns / GST_MSECOND);
m_quickWindow->update();
/* Polishing happens on the gui thread. */
m_renderControl->polishItems();
/* TODO: an async version could be used instead */
gst_gl_context_thread_add (gl_context,
(GstGLContextThreadFunc) GstQt6QuickRenderer::render_gst_gl_c, this);
GstGLMemory *tmp = gl_mem;
gl_mem = NULL;
return tmp;
}
void GstQt6QuickRenderer::initializeGstGL ()
{
GST_TRACE ("current QOpenGLContext %p", QOpenGLContext::currentContext());
if (!m_sharedRenderData->m_context->makeCurrent(m_sharedRenderData->m_surface)) {
m_errorString = "Failed to make Qt's wrapped OpenGL context current";
return;
}
GST_INFO ("current QOpenGLContext %p", QOpenGLContext::currentContext());
/* XXX: Avoid an assertion inside QSGDefaultRenderContext::initialize()
* from an unused (in this scenario) property when using multiple
* QQuickRenderControl's with the same QOpenGLContext.
*
* First noticed with Qt 5.15. Idea from:
* https://forum.qt.io/topic/55888/is-it-impossible-that-2-qquickrendercontrol-use-same-qopenglcontext/2
*
* ASSERT: "!m_gl->property(QSG_RENDERCONTEXT_PROPERTY).isValid()" in file /path/to/qt5/qtdeclarative/src/quick/scenegraph/qsgdefaultrendercontext.cpp, line 121
*/
//m_sharedRenderData->m_context->setProperty("_q_sgrendercontext", QVariant());
m_renderControl->initialize();
/* 1. QAnimationDriver's are thread-specific
* 2. QAnimationDriver controls the 'animation time' that the Qml scene is
* rendered at
*/
/* FIXME: what happens with multiple qmlgloverlay elements? Do we need a
* shared animation driver? */
g_mutex_lock (&m_sharedRenderData->lock);
if (m_sharedRenderData->m_animationDriver == nullptr) {
m_sharedRenderData->m_animationDriver = new GstQt6AnimationDriver;
m_sharedRenderData->m_animationDriver->install();
}
g_mutex_unlock (&m_sharedRenderData->lock);
/* XXX: reset the OpenGL context drawable as Qt may have clobbered it.
* Fixes glimagesink output where Qt replaces the Surface to use in its
* own MakeCurrent call. Qt does this on it's OpenGL initialisation
* the the rendering engine. */
gst_gl_context_activate (gl_context, FALSE);
gst_gl_context_activate (gl_context, TRUE);
}
void GstQt6QuickRenderer::initializeQml()
{
disconnect(m_qmlComponent, &QQmlComponent::statusChanged, this,
&GstQt6QuickRenderer::initializeQml);
if (m_qmlComponent->isError()) {
const QList<QQmlError> errorList = m_qmlComponent->errors();
for (const QQmlError &error : errorList)
m_errorString += error.toString();
return;
}
QObject *rootObject = m_qmlComponent->create();
if (m_qmlComponent->isError()) {
const QList<QQmlError> errorList = m_qmlComponent->errors();
for (const QQmlError &error : errorList)
m_errorString += error.toString();
delete rootObject;
return;
}
m_rootItem = qobject_cast<QQuickItem *>(rootObject);
if (!m_rootItem) {
m_errorString += "root QML item is not a QQuickItem";
delete rootObject;
return;
}
/* The root item is ready. Associate it with the window. */
m_rootItem->setParentItem(m_quickWindow->contentItem());
/* Update item and rendering related geometries. */
updateSizes();
/* Initialize the render control and our OpenGL resources. */
gst_gl_context_thread_add (gl_context,
(GstGLContextThreadFunc) GstQt6QuickRenderer::initialize_gst_gl_c, this);
}
void GstQt6QuickRenderer::updateSizes()
{
GstQt6BackingSurface *surface =
static_cast<GstQt6BackingSurface *>(m_sharedRenderData->m_surface);
/* Behave like SizeRootObjectToView. */
QSize size = surface->size();
m_rootItem->setWidth(size.width());
m_rootItem->setHeight(size.height());
m_quickWindow->setGeometry(0, 0, size.width(), size.height());
gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
size.height());
GstGLVideoAllocationParams *params = (GstGLVideoAllocationParams *) (gl_params);
gst_video_info_set_format (params->v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
size.height());
}
void GstQt6QuickRenderer::setSize(int w, int h)
{
static_cast<GstQt6BackingSurface *>(m_sharedRenderData->m_surface)->setSize(w, h);
updateSizes();
}
bool GstQt6QuickRenderer::setQmlScene (const gchar * scene, GError ** error)
{
/* replacing the scene is not supported */
g_return_val_if_fail (m_qmlComponent == NULL, false);
m_errorString = "";
m_qmlComponent = new QQmlComponent(m_qmlEngine);
/* XXX: do we need to provide a propper base name? */
m_qmlComponent->setData(QByteArray (scene), QUrl(""));
if (m_qmlComponent->isLoading())
/* TODO: handle async properly */
connect(m_qmlComponent, &QQmlComponent::statusChanged, this,
&GstQt6QuickRenderer::initializeQml);
else
initializeQml();
if (m_errorString != "") {
QByteArray string = m_errorString.toUtf8();
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_SETTINGS,
"%s", string.constData());
return FALSE;
}
return TRUE;
}
QQuickItem * GstQt6QuickRenderer::rootItem() const
{
return m_rootItem;
}

View file

@ -0,0 +1,124 @@
/*
* GStreamer
* Copyright (C) 2022 Matthew Waters <matthew@cenricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_QT6_GL_RENDER_H__
#define __GST_QT6_GL_RENDER_H__
#include <QThread>
#include <QMutex>
#include <gst/gl/gl.h>
QT_FORWARD_DECLARE_CLASS(QOpenGLContext)
QT_FORWARD_DECLARE_CLASS(QOpenGLFramebufferObject)
QT_FORWARD_DECLARE_CLASS(QQuickRenderControl)
QT_FORWARD_DECLARE_CLASS(QQuickWindow)
QT_FORWARD_DECLARE_CLASS(QQmlEngine)
QT_FORWARD_DECLARE_CLASS(QQmlComponent)
QT_FORWARD_DECLARE_CLASS(QQuickItem)
QT_FORWARD_DECLARE_CLASS(GstAnimationDriver)
QT_FORWARD_DECLARE_CLASS(GstBackingSurface)
class GstQt6QuickRenderer : public QObject
{
Q_OBJECT
public:
GstQt6QuickRenderer();
~GstQt6QuickRenderer();
/* initialize the GStreamer/Qt integration. On failure returns false
* and fills @error.
* Must be called with @context not wrapped and current in the current
* thread */
bool init (GstGLContext * context, GError ** error);
/* set the qml scene. returns false and fills @error on failure */
bool setQmlScene (const gchar * scene, GError ** error);
void setSize(int w, int h);
GstGLMemory *generateOutput(GstClockTime input_ns);
/* cleanup any resources. Any use of this object after calling this
* function may result in undefined behaviour */
void cleanup();
/* retrieve the rootItem from the qml scene. Only valid after
* setQmlScene() has been successfully called */
QQuickItem *rootItem() const;
private slots:
void initializeQml();
private:
void init();
void ensureFbo();
void updateSizes();
static void render_gst_gl_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->renderGstGL (); }
void renderGstGL ();
static void initialize_gst_gl_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->initializeGstGL (); }
void initializeGstGL ();
static void stop_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->stopGL (); }
void stopGL ();
static void stop_after_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->stopAfterGL (); }
void stopAfterGL ();
static void activate_context_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->activateContext (); }
void activateContext ();
static void deactivate_context_c (GstGLContext * context, GstQt6QuickRenderer * self) { self->deactivateContext (); }
void deactivateContext ();
GstGLContext *gl_context;
QQuickWindow *m_quickWindow;
QQuickRenderControl *m_renderControl;
QQmlEngine *m_qmlEngine;
QQmlComponent *m_qmlComponent;
QQuickItem *m_rootItem;
GstGLBaseMemoryAllocator *gl_allocator;
GstGLAllocationParams *gl_params;
GstVideoInfo v_info;
GstGLMemory *gl_mem;
QString m_errorString;
struct SharedRenderData *m_sharedRenderData;
};
class CreateSurfaceWorker : public QObject
{
Q_OBJECT
public:
CreateSurfaceWorker (struct SharedRenderData * rdata);
~CreateSurfaceWorker ();
bool event(QEvent *ev) override;
private:
struct SharedRenderData *m_sharedRenderData;
};
#endif /* __GST_QT6_GL_RENDER_H__ */

View file

@ -16,3 +16,4 @@ endif
subdir('qmlsink') subdir('qmlsink')
subdir('qmlsrc') subdir('qmlsrc')
subdir('qmloverlay')

View file

@ -0,0 +1,127 @@
#include <QApplication>
#include <QQmlApplicationEngine>
#include <QQuickWindow>
#include <QQuickItem>
#include <QRunnable>
#include <QDirIterator>
#include <gst/gst.h>
class SetPlaying : public QRunnable
{
public:
SetPlaying(GstElement *);
~SetPlaying();
void run ();
private:
GstElement * pipeline_;
};
SetPlaying::SetPlaying (GstElement * pipeline)
{
this->pipeline_ = pipeline ? static_cast<GstElement *> (gst_object_ref (pipeline)) : NULL;
}
SetPlaying::~SetPlaying ()
{
if (this->pipeline_)
gst_object_unref (this->pipeline_);
}
void
SetPlaying::run ()
{
if (this->pipeline_)
gst_element_set_state (this->pipeline_, GST_STATE_PLAYING);
}
static void
on_overlay_scene_initialized (GstElement * overlay, gpointer unused)
{
QQuickItem *rootObject;
GST_INFO ("scene initialized");
g_object_get (overlay, "root-item", &rootObject, NULL);
QQuickItem *videoItem = rootObject->findChild<QQuickItem *> ("inputVideoItem");
g_object_set (overlay, "widget", videoItem, NULL);
}
int main(int argc, char *argv[])
{
int ret;
gst_init (&argc, &argv);
{
QGuiApplication app(argc, argv);
QQuickWindow::setGraphicsApi(QSGRendererInterface::OpenGL);
GstElement *pipeline = gst_pipeline_new (NULL);
GstElement *src = gst_element_factory_make ("videotestsrc", NULL);
GstElement *glupload = gst_element_factory_make ("glupload", NULL);
/* the plugin must be loaded before loading the qml file to register the
* GstGLVideoItem qml item */
GstElement *overlay = gst_element_factory_make ("qml6gloverlay", NULL);
GstElement *overlay2 = gst_element_factory_make ("qml6gloverlay", NULL);
GstElement *sink = gst_element_factory_make ("qml6glsink", NULL);
g_assert (src && glupload && overlay && sink);
gst_bin_add_many (GST_BIN (pipeline), src, glupload, overlay, overlay2, sink, NULL);
gst_element_link_many (src, glupload, overlay, overlay2, sink, NULL);
/* load qmlglsink output */
QQmlApplicationEngine engine;
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
QQuickItem *videoItem;
QQuickWindow *rootObject;
/* find and set the videoItem on the sink */
rootObject = static_cast<QQuickWindow *> (engine.rootObjects().first());
videoItem = rootObject->findChild<QQuickItem *> ("videoItem");
g_assert (videoItem);
g_object_set(sink, "widget", videoItem, NULL);
QDirIterator it(":", QDirIterator::Subdirectories);
while (it.hasNext()) {
qDebug() << it.next();
}
QFile f(":/overlay.qml");
if(!f.open(QIODevice::ReadOnly)) {
qWarning() << "error: " << f.errorString();
return 1;
}
QByteArray overlay_scene = f.readAll();
qDebug() << overlay_scene;
QFile f2(":/overlay2.qml");
if(!f2.open(QIODevice::ReadOnly)) {
qWarning() << "error: " << f2.errorString();
return 1;
}
QByteArray overlay_scene2 = f2.readAll();
qDebug() << overlay_scene2;
/* load qmlgloverlay contents */
g_signal_connect (overlay, "qml-scene-initialized", G_CALLBACK (on_overlay_scene_initialized), NULL);
g_object_set (overlay, "qml-scene", overlay_scene.data(), NULL);
g_signal_connect (overlay2, "qml-scene-initialized", G_CALLBACK (on_overlay_scene_initialized), NULL);
g_object_set (overlay2, "qml-scene", overlay_scene2.data(), NULL);
rootObject->scheduleRenderJob (new SetPlaying (pipeline),
QQuickWindow::BeforeSynchronizingStage);
ret = app.exec();
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
}
gst_deinit ();
return ret;
}

View file

@ -0,0 +1,38 @@
import QtQuick 6.0
import QtQuick.Controls 6.0
import QtQuick.Dialogs 6.0
import QtQuick.Window 6.0
import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
ApplicationWindow {
id: window
visible: true
width: 640
height: 480
x: 30
y: 30
color: "black"
Item {
anchors.fill: parent
GstGLQt6VideoItem {
id: video
objectName: "videoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
anchors.bottom: parent.bottom
anchors.horizontalCenter: parent.horizontalCenter
text: "qmlglsink text"
font.pointSize: 20
color: "yellow"
style: Text.Outline
styleColor: "blue"
}
}
}

View file

@ -0,0 +1,11 @@
sources = [
'main.cpp',
]
qt_preprocessed = qt6_mod.preprocess(qresources : 'qmloverlay.qrc')
executable('qml6gloverlay', sources, qt_preprocessed,
dependencies : [gst_dep, qt6qml_example_deps],
override_options : ['cpp_std=c++17'],
c_args : gst_plugins_good_args,
include_directories : [configinc],
install: false)

View file

@ -0,0 +1,57 @@
import QtQuick 6.0
import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
Item {
/* render upside down for GStreamer */
transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 }
GstGLQt6VideoItem {
id: video
objectName: "inputVideoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
id: rotatingText
anchors.centerIn: parent
text: "Qt Quick\nrendered to\na texture"
font.pointSize: 20
color: "black"
style: Text.Outline
styleColor: "white"
RotationAnimator {
target: rotatingText;
from: 0;
to: 360;
duration: 5000
running: true
loops: Animation.Infinite
}
}
Text {
property int elapsedTime: 0
id: time
anchors.top: rotatingText.bottom
anchors.horizontalCenter: rotatingText.horizontalCenter
font.pointSize: 12
style: Text.Outline
styleColor: "black"
color: "white"
Timer {
interval: 1000
running: true
repeat: true
onTriggered: {
parent.elapsedTime += interval / 1000
parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds"
}
}
}
}

View file

@ -0,0 +1,57 @@
import QtQuick 6.0
import org.freedesktop.gstreamer.Qt6GLVideoItem 1.0
Item {
/* render upside down for GStreamer */
transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 }
GstGLQt6VideoItem {
id: video
objectName: "inputVideoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
id: rotatingText
anchors.centerIn: parent
text: "Qt Quick\nrendered to\na texture"
font.pointSize: 20
color: "black"
style: Text.Outline
styleColor: "white"
RotationAnimator {
target: rotatingText;
from: 0;
to: 360;
duration: 10000
running: true
loops: Animation.Infinite
}
}
Text {
property int elapsedTime: 0
id: time
anchors.bottom: rotatingText.top
anchors.horizontalCenter: rotatingText.horizontalCenter
font.pointSize: 12
style: Text.Outline
styleColor: "red"
color: "black"
Timer {
interval: 1000
running: true
repeat: true
onTriggered: {
parent.elapsedTime += interval / 1000
parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds"
}
}
}
}

View file

@ -0,0 +1,7 @@
<RCC>
<qresource prefix="/">
<file>main.qml</file>
<file>overlay.qml</file>
<file>overlay2.qml</file>
</qresource>
</RCC>