mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-10-02 08:42:32 +00:00
Code Issues 1 Projects Releases Wiki Activity

qt: add a qml overlay filter element

Browse source 
It takes a qml scene description and renders it using a possible input
stream.

Currently supported on GLX and WGL.
This commit is contained in:
Matthew Waters 2020-02-26 18:29:06 +11:00 committed by GStreamer Merge Bot
parent 21acf312f0
commit 4778d7166a
14 changed files with 1276 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
ext/qt/gstplugin.cc
View file

@ -22,6 +22,7 @@
#include "config.h"
#endif
#include "gstqtoverlay.h"
#include "gstqtsink.h"
#include "gstqtsrc.h"
#include <QtQml/QQmlApplicationEngine>
@ -33,11 +34,16 @@ plugin_init (GstPlugin * plugin)
GST_RANK_NONE, GST_TYPE_QT_SINK)) {
return FALSE;
}
if (!gst_element_register (plugin, "qmlglsrc",
GST_RANK_NONE, GST_TYPE_QT_SRC)) {
return FALSE;
}
if (!gst_element_register (plugin, "qmlgloverlay",
GST_RANK_NONE, GST_TYPE_QT_OVERLAY)) {
return FALSE;
}
/* this means the plugin must be loaded before the qml engine is loaded */
qmlRegisterType<QtGLVideoItem> ("org.freedesktop.gstreamer.GLVideoItem", 1, 0, "GstGLVideoItem");

88
ext/qt/gstqtglutility.cc
View file

@ -28,10 +28,13 @@
#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
#include <QX11Info>
#include <gst/gl/x11/gstgldisplay_x11.h>
#include <QtPlatformHeaders/QGLXNativeContext>
#endif
#if GST_GL_HAVE_WINDOW_WAYLAND && GST_GL_HAVE_PLATFORM_EGL && defined (HAVE_QT_WAYLAND)
#include <gst/gl/egl/gstegl.h>
#include <qpa/qplatformnativeinterface.h>
#include <QtPlatformHeaders/QEGLNativeContext>
#include <gst/gl/wayland/gstgldisplay_wayland.h>
#endif
@ -48,11 +51,19 @@
#endif
#endif
#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32)
#include <windows.h>
#include <QtPlatformHeaders/QWGLNativeContext>
#endif
#include <gst/gl/gstglfuncs.h>
#define GST_CAT_DEFAULT qt_gl_utils_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
G_LOCK_DEFINE_STATIC (display_lock);
static GWeakRef qt_display;
GstGLDisplay *
gst_qt_get_gl_display ()
{
@ -67,6 +78,16 @@ gst_qt_get_gl_display ()
"Qt gl utility functions");
g_once_init_leave (&_debug, 1);
}
G_LOCK (display_lock);
/* XXX: this assumes that only one display will ever be created by Qt */
display = static_cast<GstGLDisplay *>(g_weak_ref_get (&qt_display));
if (display) {
GST_INFO ("returning previously created display");
G_UNLOCK (display_lock);
return display;
}
GST_INFO ("QGuiApplication::instance()->platformName() %s", app->platformName().toUtf8().data());
#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
@ -145,6 +166,9 @@ gst_qt_get_gl_display ()
if (!display)
display = gst_gl_display_new ();
g_weak_ref_set (&qt_display, display);
G_UNLOCK (display_lock);
return display;
}
@ -159,6 +183,17 @@ gst_qt_get_gl_wrapcontext (GstGLDisplay * display,
g_return_val_if_fail (display != NULL && wrap_glcontext != NULL, FALSE);
/* see if we already have a current GL context in GStreamer for this thread */
{
GstGLContext *current = gst_gl_context_get_current ();
if (current) {
if (current->display == display) {
*wrap_glcontext = static_cast<GstGLContext *> (gst_object_ref (current));
return TRUE;
}
}
}
#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
if (GST_IS_GL_DISPLAY_X11 (display)) {
#if GST_GL_HAVE_PLATFORM_GLX
@ -262,3 +297,56 @@ gst_qt_get_gl_wrapcontext (GstGLDisplay * display,
return TRUE;
}
QVariant
qt_opengl_native_context_from_gst_gl_context (GstGLContext * context)
{
guintptr handle;
GstGLPlatform platform;
handle = gst_gl_context_get_gl_context (context);
platform = gst_gl_context_get_gl_platform (context);
#if GST_GL_HAVE_WINDOW_X11 && defined (HAVE_QT_X11)
if (platform == GST_GL_PLATFORM_GLX) {
GstGLDisplay *display = gst_gl_context_get_display (context);
Display *xdisplay = (Display *) gst_gl_display_get_handle (display);
gst_object_unref (display);
return QVariant::fromValue(QGLXNativeContext((GLXContext) handle, xdisplay));
}
#endif
#if GST_GL_HAVE_PLATFORM_EGL
if (platform == GST_GL_PLATFORM_EGL) {
#if GST_GL_HAVE_WINDOW_WAYLAND && defined (HAVE_QT_WAYLAND)
GstGLDisplay *display = gst_gl_context_get_display (context);
if (gst_gl_display_get_handle_type (display) == GST_GL_DISPLAY_TYPE_WAYLAND) {
GstGLDisplayEGL *display_egl = gst_gl_display_egl_from_gl_display (display);
if (display_egl) {
EGLDisplay egl_display = (EGLDisplay) gst_gl_display_get_handle ((GstGLDisplay *) display_egl);
gst_object_unref (display_egl);
gst_object_unref (display);
return QVariant::fromValue(QEGLNativeContext((EGLContext) handle, egl_display));
}
}
#endif
}
#endif
#if GST_GL_HAVE_WINDOW_WIN32 && GST_GL_HAVE_PLATFORM_WGL && defined (HAVE_QT_WIN32)
if (platform == GST_GL_PLATFORM_WGL) {
GstGLWindow *window = gst_gl_context_get_window (context);
guintptr hwnd = gst_gl_window_get_window_handle (window);
gst_object_unref (window);
return QVariant::fromValue(QWGLNativeContext((HGLRC) handle, (HWND) hwnd));
}
#endif
{
gchar *platform_s = gst_gl_platform_to_string (platform);
g_warning ("Unimplemented configuration! This means either:\n"
"1. The qmlgl plugin was built without support for your platform.\n"
"2. The necessary code to convert from a GstGLContext to Qt's "
"native context type for \'%s\' currently does not exist.",
platform_s);
g_free (platform_s);
}
return QVariant::fromValue(nullptr);
}

4
ext/qt/gstqtglutility.h
View file

@ -24,11 +24,15 @@
#include <gst/gst.h>
#include <gst/gl/gl.h>
#include <QVariant>
G_BEGIN_DECLS
GstGLDisplay * gst_qt_get_gl_display ();
gboolean gst_qt_get_gl_wrapcontext (GstGLDisplay * display,
GstGLContext **wrap_glcontext, GstGLContext **context);
QVariant qt_opengl_native_context_from_gst_gl_context (GstGLContext * context);
G_END_DECLS
#endif /* __QT_GL_UTILS_H__ */

305
ext/qt/gstqtoverlay.cc Normal file
View file

@ -0,0 +1,305 @@
/*
* GStreamer
* Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:gstqtoverlay
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstqtoverlay.h"
#include "qtglrenderer.h"
#include <gst/gl/gstglfuncs.h>
#define GST_CAT_DEFAULT gst_debug_qt_gl_overlay
GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
static void gst_qt_overlay_finalize (GObject * object);
static void gst_qt_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * param_spec);
static void gst_qt_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * param_spec);
static gboolean gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter);
static void gst_qt_overlay_gl_stop (GstGLBaseFilter * bfilter);
static gboolean gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter,
GstCaps * in_caps, GstCaps * out_caps);
static GstFlowReturn gst_qt_overlay_prepare_output_buffer (GstBaseTransform * btrans,
GstBuffer * buffer, GstBuffer ** outbuf);
static GstFlowReturn gst_qt_overlay_transform (GstBaseTransform * btrans,
GstBuffer * inbuf, GstBuffer * outbuf);
enum
{
PROP_0,
PROP_WIDGET,
PROP_QML_SCENE,
};
enum
{
SIGNAL_0,
SIGNAL_QML_SCENE_INITIALIZED,
LAST_SIGNAL
};
static guint gst_qt_overlay_signals[LAST_SIGNAL] = { 0 };
#define gst_qt_overlay_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstQtOverlay, gst_qt_overlay,
GST_TYPE_GL_FILTER, GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT,
"qtoverlay", 0, "Qt Video Overlay"));
static void
gst_qt_overlay_class_init (GstQtOverlayClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *btrans_class;
GstGLBaseFilterClass *glbasefilter_class;
GstGLFilterClass *glfilter_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
glbasefilter_class = (GstGLBaseFilterClass *) klass;
glfilter_class = (GstGLFilterClass *) klass;
btrans_class = (GstBaseTransformClass *) klass;
gobject_class->set_property = gst_qt_overlay_set_property;
gobject_class->get_property = gst_qt_overlay_get_property;
gobject_class->finalize = gst_qt_overlay_finalize;
gst_element_class_set_metadata (gstelement_class, "Qt Video Overlay",
"Filter/QML/Overlay", "A filter that renders a QML scene onto a video stream",
"Matthew Waters <matthew@centricular.com>");
g_object_class_install_property (gobject_class, PROP_QML_SCENE,
g_param_spec_string ("qml-scene", "QML Scene",
"The contents of the QML scene", NULL,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_object_class_install_property (gobject_class, PROP_WIDGET,
g_param_spec_pointer ("widget", "QQuickItem",
"The QQuickItem to place the input video in the object hierarchy",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
/**
* GstQmlGLOverlay::qml-scene-initialized
* @element: the #GstQmlGLOverlay
* @root_item: the `QQuickItem` found to be the root item
* @user_data: user provided data
*/
gst_qt_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED] =
g_signal_new ("qml-scene-initialized", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 1, G_TYPE_POINTER);
gst_gl_filter_add_rgba_pad_templates (glfilter_class);
btrans_class->prepare_output_buffer = gst_qt_overlay_prepare_output_buffer;
btrans_class->transform = gst_qt_overlay_transform;
glbasefilter_class->gl_start = gst_qt_overlay_gl_start;
glbasefilter_class->gl_stop = gst_qt_overlay_gl_stop;
glbasefilter_class->gl_set_caps = gst_qt_overlay_gl_set_caps;
}
static void
gst_qt_overlay_init (GstQtOverlay * qt_overlay)
{
qt_overlay->widget = QSharedPointer<QtGLVideoItemInterface>();
}
static void
gst_qt_overlay_finalize (GObject * object)
{
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object);
qt_overlay->widget.clear();
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_qt_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object);
switch (prop_id) {
case PROP_WIDGET: {
QtGLVideoItem *qt_item = static_cast<QtGLVideoItem *> (g_value_get_pointer (value));
if (qt_item)
qt_overlay->widget = qt_item->getInterface();
else
qt_overlay->widget.clear();
break;
}
case PROP_QML_SCENE:
g_free (qt_overlay->qml_scene);
qt_overlay->qml_scene = g_value_dup_string (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_qt_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (object);
switch (prop_id) {
case PROP_WIDGET:
/* This is not really safe - the app needs to be
* sure the widget is going to be kept alive or
* this can crash */
if (qt_overlay->widget)
g_value_set_pointer (value, qt_overlay->widget->videoItem());
else
g_value_set_pointer (value, NULL);
break;
case PROP_QML_SCENE:
g_value_set_string (value, qt_overlay->qml_scene);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter)
{
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter);
QQuickItem *root;
GError *error = NULL;
GST_TRACE_OBJECT (bfilter, "using scene:\n%s", qt_overlay->qml_scene);
if (!qt_overlay->qml_scene || g_strcmp0 (qt_overlay->qml_scene, "") == 0) {
GST_ELEMENT_ERROR (bfilter, RESOURCE, NOT_FOUND, ("qml-scene property not set"), (NULL));
return FALSE;
}
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (bfilter))
return FALSE;
qt_overlay->renderer = new GstQuickRenderer;
if (!qt_overlay->renderer->init (bfilter->context, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
return FALSE;
}
/* FIXME: Qml may do async loading and we need to propagate qml errors in that case as well */
if (!qt_overlay->renderer->setQmlScene (qt_overlay->qml_scene, &error)) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("%s", error->message), (NULL));
return FALSE;
}
root = qt_overlay->renderer->rootItem();
if (!root) {
GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND,
("Qml scene does not have a root item"), (NULL));
return FALSE;
}
g_signal_emit (qt_overlay, gst_qt_overlay_signals[SIGNAL_QML_SCENE_INITIALIZED], 0, root);
return TRUE;
}
static void
gst_qt_overlay_gl_stop (GstGLBaseFilter * bfilter)
{
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter);
if (qt_overlay->renderer) {
qt_overlay->renderer->cleanup();
delete qt_overlay->renderer;
}
qt_overlay->renderer = NULL;
GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (bfilter);
}
static gboolean
gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter, GstCaps * in_caps,
GstCaps * out_caps)
{
GstGLFilter *filter = GST_GL_FILTER (bfilter);
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (bfilter);
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_set_caps (bfilter, in_caps, out_caps))
return FALSE;
qt_overlay->renderer->setSize (GST_VIDEO_INFO_WIDTH (&filter->out_info),
GST_VIDEO_INFO_HEIGHT (&filter->out_info));
return TRUE;
}
static GstFlowReturn
gst_qt_overlay_prepare_output_buffer (GstBaseTransform * btrans,
GstBuffer * buffer, GstBuffer ** outbuf)
{
GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (btrans);
GstGLBaseFilter *bfilter = GST_GL_BASE_FILTER (btrans);
GstGLFilter *filter = GST_GL_FILTER (btrans);
GstQtOverlay *qt_overlay = GST_QT_OVERLAY (btrans);
GstGLMemory *out_mem;
if (qt_overlay->widget) {
qt_overlay->widget->setCaps (bfilter->in_caps);
qt_overlay->widget->setBuffer (buffer);
}
/* XXX: is this the correct ts to drive the animation */
out_mem = qt_overlay->renderer->generateOutput (GST_BUFFER_PTS (buffer));
if (!out_mem) {
GST_ERROR_OBJECT (qt_overlay, "Failed to generate output");
return GST_FLOW_ERROR;
}
*outbuf = gst_buffer_new ();
gst_buffer_append_memory (*outbuf, (GstMemory *) out_mem);
gst_buffer_add_video_meta (*outbuf, (GstVideoFrameFlags) 0,
GST_VIDEO_INFO_FORMAT (&filter->out_info),
GST_VIDEO_INFO_WIDTH (&filter->in_info),
GST_VIDEO_INFO_HEIGHT (&filter->out_info));
bclass->copy_metadata (btrans, buffer, *outbuf);
return GST_FLOW_OK;
}
static GstFlowReturn
gst_qt_overlay_transform (GstBaseTransform * btrans, GstBuffer * inbuf,
GstBuffer * outbuf)
{
return GST_FLOW_OK;
}

76
ext/qt/gstqtoverlay.h Normal file
View file

@ -0,0 +1,76 @@
/*
* GStreamer
* Copyright (C) 2020 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_QT_OVERLAY_H__
#define __GST_QT_OVERLAY_H__
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include "qtglrenderer.h"
#include "qtitem.h"
typedef struct _GstQtOverlay GstQtOverlay;
typedef struct _GstQtOverlayClass GstQtOverlayClass;
typedef struct _GstQtOverlayPrivate GstQtOverlayPrivate;
G_BEGIN_DECLS
GType gst_qt_overlay_get_type (void);
#define GST_TYPE_QT_OVERLAY (gst_qt_overlay_get_type())
#define GST_QT_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_QT_OVERLAY,GstQtOverlay))
#define GST_QT_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_QT_OVERLAY,GstQtOverlayClass))
#define GST_IS_QT_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_QT_OVERLAY))
#define GST_IS_QT_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_QT_OVERLAY))
#define GST_QT_OVERLAY_CAST(obj) ((GstQtOverlay*)(obj))
/**
* GstQtOverlay:
*
* Opaque #GstQtOverlay object
*/
struct _GstQtOverlay
{
/* <private> */
GstGLFilter parent;
gchar *qml_scene;
GstQuickRenderer *renderer;
QSharedPointer<QtGLVideoItemInterface> widget;
};
/**
* GstQtOverlayClass:
*
* The #GstQtOverlayClass struct only contains private data
*/
struct _GstQtOverlayClass
{
/* <private> */
GstGLFilterClass parent_class;
};
GstQtOverlay * gst_qt_overlay_new (void);
G_END_DECLS
#endif /* __GST_QT_OVERLAY_H__ */

3
ext/qt/meson.build
View file

@ -2,8 +2,10 @@ sources = [
'gstplugin.cc',
'gstqsgtexture.cc',
'gstqtglutility.cc',
'gstqtoverlay.cc',
'gstqtsink.cc',
'gstqtsrc.cc',
'qtglrenderer.cc',
'qtitem.cc',
'qtwindow.cc',
]
@ -12,6 +14,7 @@ moc_headers = [
'qtitem.h',
'qtwindow.h',
'gstqsgtexture.h',
'qtglrenderer.h',
]
# FIXME: -Dqt5=enabled is silently ignored if a c++ compiler is not found

446
ext/qt/qtglrenderer.cc Normal file
View file

@ -0,0 +1,446 @@
#include <QObject>
#include <QQmlEngine>
#include <QQmlComponent>
#include <QWindow>
#include <QQuickRenderControl>
#include <QQuickWindow>
#include <QQuickItem>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QOpenGLFramebufferObject>
#include <QAnimationDriver>
#include <gst/gl/gl.h>
#include <gst/gl/gstglfuncs.h>
#include "qtglrenderer.h"
#include "gstqtglutility.h"
#define GST_CAT_DEFAULT gst_qt_gl_renderer_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
static void
init_debug (void)
{
static volatile gsize _debug;
if (g_once_init_enter (&_debug)) {
GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtglrenderer", 0,
"Qt OpenGL Renderer");
g_once_init_leave (&_debug, 1);
}
}
/* Needs to be based on QWindow otherwise (at least) windows and nvidia
* proprietary on linux does not work
* We also need to override the size handling to get the correct output size
*/
class GstBackingSurface : public QWindow
{
public:
GstBackingSurface();
~GstBackingSurface();
void setSize (int width, int height);
QSize size() const override;
private:
QSize m_size;
};
GstBackingSurface::GstBackingSurface()
: m_size(QSize())
{
/* we do OpenGL things so need an OpenGL surface */
setSurfaceType(QSurface::OpenGLSurface);
}
GstBackingSurface::~GstBackingSurface()
{
}
QSize GstBackingSurface::size () const
{
return m_size;
}
void GstBackingSurface::setSize (int width, int height)
{
m_size = QSize (width, height);
}
class GstAnimationDriver : public QAnimationDriver
{
public:
GstAnimationDriver();
void setNextTime(qint64 ms);
void advance() override;
qint64 elapsed() const override;
private:
qint64 m_elapsed;
qint64 m_next;
};
GstAnimationDriver::GstAnimationDriver()
: m_elapsed(0),
m_next(0)
{
}
void GstAnimationDriver::advance()
{
m_elapsed = m_next;
advanceAnimation();
}
qint64 GstAnimationDriver::elapsed() const
{
return m_elapsed;
}
void GstAnimationDriver::setNextTime(qint64 ms)
{
m_next = ms;
}
void
GstQuickRenderer::deactivateContext ()
{
}
void
GstQuickRenderer::activateContext ()
{
}
static void
delete_cxx (QOpenGLFramebufferObject * cxx)
{
GST_TRACE ("freeing Qfbo %p", cxx);
delete cxx;
}
GstQuickRenderer::GstQuickRenderer()
: gl_context(NULL),
m_context(nullptr),
m_renderThread(nullptr),
m_surface(nullptr),
m_fbo(nullptr),
m_quickWindow(nullptr),
m_renderControl(nullptr),
m_qmlEngine(nullptr),
m_qmlComponent(nullptr),
m_rootItem(nullptr),
m_animationDriver(nullptr),
gl_allocator(NULL),
gl_params(NULL),
gl_mem(NULL)
{
init_debug ();
}
bool GstQuickRenderer::init (GstGLContext * context, GError ** error)
{
g_return_val_if_fail (gst_gl_context_get_current () == context, false);
QVariant qt_native_context = qt_opengl_native_context_from_gst_gl_context (context);
if (qt_native_context.isNull()) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not convert from the provided GstGLContext to a Qt "
"native context");
return false;
}
m_context = new QOpenGLContext;
m_context->setNativeHandle(qt_native_context);
m_surface = new GstBackingSurface;
m_surface->create(); /* FIXME: may need to be called on Qt's main thread */
m_renderThread = QThread::currentThread();
gst_gl_context_activate (context, FALSE);
/* Qt does some things that it may require the OpenGL context current in
* ->create() so that it has the necessry information to create the
* QOpenGLContext from the native handle. This may fail if the OpenGL
* context is already current in another thread so we need to deactivate
* the context from GStreamer's thread before asking Qt to create the
* QOpenGLContext with ->create().
*/
m_context->create();
m_context->doneCurrent();
m_context->moveToThread (m_renderThread);
if (!m_context->makeCurrent(m_surface)) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not make Qt OpenGL context current");
/* try to keep the same OpenGL context state */
gst_gl_context_activate (context, TRUE);
return false;
}
if (!gst_gl_context_activate (context, TRUE)) {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_NOT_FOUND,
"Could not make OpenGL context current again");
return false;
}
m_renderControl = new QQuickRenderControl();
/* Create a QQuickWindow that is associated with our render control. Note that this
* window never gets created or shown, meaning that it will never get an underlying
* native (platform) window.
*/
m_quickWindow = new QQuickWindow(m_renderControl);
/* after QQuickWindow creation as QQuickRenderControl requires it */
m_renderControl->prepareThread (m_renderThread);
/* Create a QML engine. */
m_qmlEngine = new QQmlEngine;
if (!m_qmlEngine->incubationController())
m_qmlEngine->setIncubationController(m_quickWindow->incubationController());
gl_context = static_cast<GstGLContext*>(gst_object_ref (context));
gl_allocator = (GstGLBaseMemoryAllocator *) gst_gl_memory_allocator_get_default (gl_context);
gl_params = (GstGLAllocationParams *) (gst_gl_video_allocation_params_new_wrapped_texture (gl_context,
NULL, &this->v_info, 0, NULL, GST_GL_TEXTURE_TARGET_2D, GST_GL_RGBA8,
0, NULL, (GDestroyNotify) delete_cxx));
return true;
}
GstQuickRenderer::~GstQuickRenderer()
{
gst_gl_allocation_params_free (gl_params);
}
void GstQuickRenderer::stop ()
{
g_assert (QOpenGLContext::currentContext() == m_context);
if (m_renderControl)
m_renderControl->invalidate();
if (m_fbo)
delete m_fbo;
m_fbo = nullptr;
m_context->doneCurrent();
if (m_animationDriver)
delete m_animationDriver;
m_animationDriver = nullptr;
}
void GstQuickRenderer::cleanup()
{
if (gl_context)
gst_gl_context_thread_add (gl_context,
(GstGLContextThreadFunc) GstQuickRenderer::stop_c, this);
/* Delete the render control first since it will free the scenegraph resources.
* Destroy the QQuickWindow only afterwards. */
if (m_renderControl)
delete m_renderControl;
m_renderControl = nullptr;
if (m_qmlComponent)
delete m_qmlComponent;
m_qmlComponent = nullptr;
if (m_quickWindow)
delete m_quickWindow;
m_quickWindow = nullptr;
if (m_qmlEngine)
delete m_qmlEngine;
m_qmlEngine = nullptr;
gst_clear_object (&gl_context);
if (m_context)
delete m_context;
m_context = nullptr;
}
void GstQuickRenderer::ensureFbo()
{
if (m_fbo && m_fbo->size() != m_surface->size()) {
GST_INFO ("removing old framebuffer created with size %ix%i",
m_fbo->size().width(), m_fbo->size().height());
delete m_fbo;
m_fbo = nullptr;
}
if (!m_fbo) {
m_fbo = new QOpenGLFramebufferObject(m_surface->size(),
QOpenGLFramebufferObject::CombinedDepthStencil);
m_quickWindow->setRenderTarget(m_fbo);
GST_DEBUG ("new framebuffer created with size %ix%i",
m_fbo->size().width(), m_fbo->size().height());
}
}
void
GstQuickRenderer::renderGstGL ()
{
GST_DEBUG ("current QOpenGLContext %p", QOpenGLContext::currentContext());
m_quickWindow->resetOpenGLState();
m_animationDriver->advance();
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("pending QEvents processed");
ensureFbo();
/* Synchronization and rendering happens here on the render thread. */
if (m_renderControl->sync())
GST_LOG ("sync successful");
/* Meanwhile on this thread continue with the actual rendering. */
m_renderControl->render();
GST_DEBUG ("wrapping Qfbo %p with texture %u", m_fbo, m_fbo->texture());
gl_params->user_data = static_cast<gpointer> (m_fbo);
gl_params->gl_handle = GINT_TO_POINTER (m_fbo->texture());
gl_mem = (GstGLMemory *) gst_gl_base_memory_alloc (gl_allocator, gl_params);
m_fbo = nullptr;
}
GstGLMemory *GstQuickRenderer::generateOutput(GstClockTime input_ns)
{
m_animationDriver->setNextTime(input_ns / GST_MSECOND);
/* run an event loop to update any changed values for rendering */
QEventLoop loop;
if (loop.processEvents())
GST_LOG ("pending QEvents processed");
GST_LOG ("generating output for time %" GST_TIME_FORMAT " ms: %"
G_GUINT64_FORMAT, GST_TIME_ARGS (input_ns), input_ns / GST_MSECOND);
m_quickWindow->update();
/* Polishing happens on the gui thread. */
m_renderControl->polishItems();
/* TODO: an async version could be used where */
gst_gl_context_thread_add (gl_context, (GstGLContextThreadFunc) GstQuickRenderer::render_gst_gl_c, this);
GstGLMemory *tmp = gl_mem;
gl_mem = NULL;
return tmp;
}
void GstQuickRenderer::initializeGstGL ()
{
GST_TRACE ("current QOpenGLContext %p", QOpenGLContext::currentContext());
if (!m_context->makeCurrent(m_surface)) {
m_errorString = "Failed to make Qt's wrapped OpenGL context current";
return;
}
GST_INFO ("current QOpenGLContext %p", QOpenGLContext::currentContext());
m_renderControl->initialize(m_context);
/* 1. QAnimationDriver's are thread-specific
* 2. QAnimationDriver controls the 'animation time' that the Qml scene is
* rendered at
*/
/* FIXME: what happens with multiple qmlgloverlay elements? Do we need a
* shared animation driver? */
m_animationDriver = new GstAnimationDriver;
m_animationDriver->install();
}
void GstQuickRenderer::initializeQml()
{
disconnect(m_qmlComponent, &QQmlComponent::statusChanged, this, &GstQuickRenderer::initializeQml);
if (m_qmlComponent->isError()) {
const QList<QQmlError> errorList = m_qmlComponent->errors();
for (const QQmlError &error : errorList)
m_errorString += error.toString();
return;
}
QObject *rootObject = m_qmlComponent->create();
if (m_qmlComponent->isError()) {
const QList<QQmlError> errorList = m_qmlComponent->errors();
for (const QQmlError &error : errorList)
m_errorString += error.toString();
delete rootObject;
return;
}
m_rootItem = qobject_cast<QQuickItem *>(rootObject);
if (!m_rootItem) {
m_errorString += "root QML item is not a QQuickItem";
delete rootObject;
return;
}
/* The root item is ready. Associate it with the window. */
m_rootItem->setParentItem(m_quickWindow->contentItem());
/* Update item and rendering related geometries. */
updateSizes();
/* Initialize the render control and our OpenGL resources. */
gst_gl_context_thread_add (gl_context, (GstGLContextThreadFunc) GstQuickRenderer::initialize_gst_gl_c, this);
}
void GstQuickRenderer::updateSizes()
{
GstBackingSurface *surface = static_cast<GstBackingSurface *>(m_surface);
/* Behave like SizeRootObjectToView. */
QSize size = surface->size();
m_rootItem->setWidth(size.width());
m_rootItem->setHeight(size.height());
m_quickWindow->setGeometry(0, 0, size.width(), size.height());
gst_video_info_set_format (&v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
size.height());
GstGLVideoAllocationParams *params = (GstGLVideoAllocationParams *) (gl_params);
gst_video_info_set_format (params->v_info, GST_VIDEO_FORMAT_RGBA, size.width(),
size.height());
}
void GstQuickRenderer::setSize(int w, int h)
{
static_cast<GstBackingSurface *>(m_surface)->setSize(w, h);
updateSizes();
}
bool GstQuickRenderer::setQmlScene (const gchar * scene, GError ** error)
{
/* replacing the scene is not supported */
g_return_val_if_fail (m_qmlComponent == NULL, false);
m_errorString = "";
m_qmlComponent = new QQmlComponent(m_qmlEngine);
/* XXX: do we need to provide a propper base name? */
m_qmlComponent->setData(QByteArray (scene), QUrl(""));
if (m_qmlComponent->isLoading())
/* TODO: handle async properly */
connect(m_qmlComponent, &QQmlComponent::statusChanged, this, &GstQuickRenderer::initializeQml);
else
initializeQml();
if (m_errorString != "") {
g_set_error (error, GST_RESOURCE_ERROR, GST_RESOURCE_ERROR_SETTINGS,
m_errorString.toUtf8());
return FALSE;
}
return TRUE;
}
QQuickItem * GstQuickRenderer::rootItem() const
{
return m_rootItem;
}

112
ext/qt/qtglrenderer.h Normal file
View file

@ -0,0 +1,112 @@
/*
* GStreamer
* Copyright (C) 2020 Matthew Waters <matthew@cenricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __QT_QUICK_RENDER_H__
#define __QT_QUICK_RENDER_H__
#include <QThread>
#include <QMutex>
#include <gst/gl/gl.h>
QT_FORWARD_DECLARE_CLASS(QOpenGLContext)
QT_FORWARD_DECLARE_CLASS(QOpenGLFramebufferObject)
QT_FORWARD_DECLARE_CLASS(QQuickRenderControl)
QT_FORWARD_DECLARE_CLASS(QQuickWindow)
QT_FORWARD_DECLARE_CLASS(QQmlEngine)
QT_FORWARD_DECLARE_CLASS(QQmlComponent)
QT_FORWARD_DECLARE_CLASS(QQuickItem)
QT_FORWARD_DECLARE_CLASS(GstAnimationDriver)
QT_FORWARD_DECLARE_CLASS(GstBackingSurface)
class GstQuickRenderer : public QObject
{
Q_OBJECT
public:
GstQuickRenderer();
~GstQuickRenderer();
/* initialize the GStreamer/Qt integration. On failure returns false
* and fills @error.
* Must be called with @context not wrapped and current in the current
* thread */
bool init (GstGLContext * context, GError ** error);
/* set the qml scene. returns false and fills @error on failure */
bool setQmlScene (const gchar * scene, GError ** error);
void setSize(int w, int h);
GstGLMemory *generateOutput(GstClockTime input_ns);
/* cleanup any resources. Any use of this object after calling this
* function may result in undefined behaviour */
void cleanup();
/* retrieve the rootItem from the qml scene. Only valid after
* setQmlScene() has been successfully called */
QQuickItem *rootItem() const;
private slots:
void initializeQml();
private:
void init();
void ensureFbo();
void updateSizes();
static void render_gst_gl_c (GstGLContext * context, GstQuickRenderer * self) { self->renderGstGL (); }
void renderGstGL ();
static void initialize_gst_gl_c (GstGLContext * context, GstQuickRenderer * self) { self->initializeGstGL (); }
void initializeGstGL ();
static void stop_c (GstGLContext * context, GstQuickRenderer * self) { self->stop (); }
void stop ();
static void activate_context_c (GstGLContext * context, GstQuickRenderer * self) { self->activateContext (); }
void activateContext ();
static void deactivate_context_c (GstGLContext * context, GstQuickRenderer * self) { self->deactivateContext (); }
void deactivateContext ();
GstGLContext *gl_context;
QOpenGLContext *m_context;
QThread *m_renderThread;
GstBackingSurface *m_surface;
QOpenGLFramebufferObject *m_fbo;
QQuickWindow *m_quickWindow;
QQuickRenderControl *m_renderControl;
QQmlEngine *m_qmlEngine;
QQmlComponent *m_qmlComponent;
QQuickItem *m_rootItem;
GstAnimationDriver *m_animationDriver;
GstGLBaseMemoryAllocator *gl_allocator;
GstGLAllocationParams *gl_params;
GstVideoInfo v_info;
GstGLMemory *gl_mem;
QString m_errorString;
};
#endif /* __QT_QUICK_RENDER_H__ */

1
tests/examples/qt/meson.build
View file

@ -1,3 +1,4 @@
subdir('qmloverlay')
subdir('qmlsink')
subdir('qmlsink-dynamically-added')
subdir('qmlsrc')

112
tests/examples/qt/qmloverlay/main.cpp Normal file
View file

@ -0,0 +1,112 @@
#include <QApplication>
#include <QQmlApplicationEngine>
#include <QQuickWindow>
#include <QQuickItem>
#include <QRunnable>
#include <QDirIterator>
#include <gst/gst.h>
class SetPlaying : public QRunnable
{
public:
SetPlaying(GstElement *);
~SetPlaying();
void run ();
private:
GstElement * pipeline_;
};
SetPlaying::SetPlaying (GstElement * pipeline)
{
this->pipeline_ = pipeline ? static_cast<GstElement *> (gst_object_ref (pipeline)) : NULL;
}
SetPlaying::~SetPlaying ()
{
if (this->pipeline_)
gst_object_unref (this->pipeline_);
}
void
SetPlaying::run ()
{
if (this->pipeline_)
gst_element_set_state (this->pipeline_, GST_STATE_PLAYING);
}
static void
on_overlay_scene_initialized (GstElement * overlay, gpointer root_item, gpointer unused)
{
GST_INFO ("scene initialized");
QQuickItem *rootObject = static_cast<QQuickItem *> (root_item);
QQuickItem *videoItem = rootObject->findChild<QQuickItem *> ("inputVideoItem");
g_object_set (overlay, "widget", videoItem, NULL);
}
int main(int argc, char *argv[])
{
int ret;
gst_init (&argc, &argv);
{
QGuiApplication app(argc, argv);
GstElement *pipeline = gst_pipeline_new (NULL);
GstElement *src = gst_element_factory_make ("videotestsrc", NULL);
GstElement *glupload = gst_element_factory_make ("glupload", NULL);
/* the plugin must be loaded before loading the qml file to register the
* GstGLVideoItem qml item */
GstElement *overlay = gst_element_factory_make ("qmlgloverlay", NULL);
GstElement *sink = gst_element_factory_make ("qmlglsink", NULL);
g_assert (src && glupload && overlay && sink);
gst_bin_add_many (GST_BIN (pipeline), src, glupload, overlay, sink, NULL);
gst_element_link_many (src, glupload, overlay, sink, NULL);
/* load qmlglsink output */
QQmlApplicationEngine engine;
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
QQuickItem *videoItem;
QQuickWindow *rootObject;
/* find and set the videoItem on the sink */
rootObject = static_cast<QQuickWindow *> (engine.rootObjects().first());
videoItem = rootObject->findChild<QQuickItem *> ("videoItem");
g_assert (videoItem);
g_object_set(sink, "widget", videoItem, NULL);
QDirIterator it(":", QDirIterator::Subdirectories);
while (it.hasNext()) {
qDebug() << it.next();
}
QFile f(":/overlay.qml");
if(!f.open(QIODevice::ReadOnly)) {
qWarning() << "error: " << f.errorString();
return 1;
}
QByteArray overlay_scene = f.readAll();
qDebug() << overlay_scene;
/* load qmlgloverlay contents */
g_signal_connect (overlay, "qml-scene-initialized", G_CALLBACK (on_overlay_scene_initialized), NULL);
g_object_set (overlay, "qml-scene", overlay_scene.data(), NULL);
rootObject->scheduleRenderJob (new SetPlaying (pipeline),
QQuickWindow::BeforeSynchronizingStage);
ret = app.exec();
gst_element_set_state (pipeline, GST_STATE_NULL);
gst_object_unref (pipeline);
}
gst_deinit ();
return ret;
}

39
tests/examples/qt/qmloverlay/main.qml Normal file
View file

@ -0,0 +1,39 @@
import QtQuick 2.4
import QtQuick.Controls 1.1
import QtQuick.Controls.Styles 1.3
import QtQuick.Dialogs 1.2
import QtQuick.Window 2.1
import org.freedesktop.gstreamer.GLVideoItem 1.0
ApplicationWindow {
id: window
visible: true
width: 640
height: 480
x: 30
y: 30
color: "black"
Item {
anchors.fill: parent
GstGLVideoItem {
id: video
objectName: "videoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
anchors.bottom: parent.bottom
anchors.horizontalCenter: parent.horizontalCenter
text: "qmlglsink text"
font.pointSize: 20
color: "yellow"
style: Text.Outline
styleColor: "blue"
}
}
}

20
tests/examples/qt/qmloverlay/meson.build Normal file
View file

@ -0,0 +1,20 @@
sources = [
'main.cpp',
]
if have_cxx and build_gstgl and gstgl_dep.found()
qt5_mod = import('qt5')
qt5qml_deps = dependency('qt5', modules : ['Core', 'Gui', 'Widgets', 'Qml', 'Quick'],
required: get_option('examples'))
# FIXME Add a way to get that information out of the qt5 module
moc = find_program('moc-qt5', 'moc', required : get_option('examples'))
if qt5qml_deps.found() and moc.found()
qt_preprocessed = qt5_mod.preprocess(qresources : 'qmloverlay.qrc')
executable('qmlgloverlay', sources, qt_preprocessed,
dependencies : [gst_dep, qt5qml_deps],
c_args : gst_plugins_good_args,
include_directories : [configinc],
install: false)
endif
endif

57
tests/examples/qt/qmloverlay/overlay.qml Normal file
View file

@ -0,0 +1,57 @@
import QtQuick 2.4
import org.freedesktop.gstreamer.GLVideoItem 1.0
Item {
/* render upside down for GStreamer */
transform: Scale { origin.x : 0; origin.y : height / 2.; yScale : -1 }
GstGLVideoItem {
id: video
objectName: "inputVideoItem"
anchors.centerIn: parent
width: parent.width
height: parent.height
}
Text {
id: rotatingText
anchors.centerIn: parent
text: "Qt Quick\nrendered to\na texture"
font.pointSize: 20
color: "black"
style: Text.Outline
styleColor: "white"
RotationAnimator {
target: rotatingText;
from: 0;
to: 360;
duration: 5000
running: true
loops: Animation.Infinite
}
}
Text {
property int elapsedTime: 0
id: time
anchors.top: rotatingText.bottom
anchors.horizontalCenter: rotatingText.horizontalCenter
font.pointSize: 12
style: Text.Outline
styleColor: "black"
color: "white"
Timer {
interval: 1000
running: true
repeat: true
onTriggered: {
parent.elapsedTime += interval / 1000
parent.text = "overlay: " + parent.elapsedTime.toString() + " seconds"
}
}
}
}

6
tests/examples/qt/qmloverlay/qmloverlay.qrc Normal file
View file

@ -0,0 +1,6 @@
<RCC>
<qresource prefix="/">
<file>main.qml</file>
<file>overlay.qml</file>
</qresource>
</RCC>