From 41ad50fb48d15da5502b867f8b0e1696c536b08f Mon Sep 17 00:00:00 2001 From: Andreas Wittmann Date: Thu, 13 Jun 2024 09:59:24 +0200 Subject: [PATCH] qmlgloverlay: make depth buffer optional allow dropping depth/stencil buffer for qmlgloverlay element. Shows considerable drop in memory bandwidth consumption of the element on windows platforms. Part-of: --- .../gst/gl/glprototypes/all_functions.h | 1 + .../gst/gl/glprototypes/invalidate_subdata.h | 66 +++++++++++++++++++ .../gst-libs/gst/gl/gstglfuncs.h | 2 +- .../gst-libs/gst/gl/meson.build | 1 + .../docs/gst_plugins_cache.json | 12 ++++ .../gst-plugins-good/ext/qt/gstqtoverlay.cc | 27 +++++++- .../gst-plugins-good/ext/qt/gstqtoverlay.h | 2 + .../gst-plugins-good/ext/qt/qtglrenderer.cc | 34 +++++++++- .../gst-plugins-good/ext/qt/qtglrenderer.h | 11 ++-- 9 files changed, 146 insertions(+), 10 deletions(-) create mode 100644 subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/invalidate_subdata.h diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/all_functions.h b/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/all_functions.h index 48007e9548..a62efb8913 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/all_functions.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/all_functions.h @@ -32,3 +32,4 @@ #include "buffers.h" #include "query.h" #include "buffer_storage.h" +#include "invalidate_subdata.h" \ No newline at end of file diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/invalidate_subdata.h b/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/invalidate_subdata.h new file mode 100644 index 0000000000..53449cae14 --- /dev/null +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/glprototypes/invalidate_subdata.h @@ -0,0 +1,66 @@ +/* + * GStreamer + * Copyright (C) 2024 Andreas Wittmann + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +GST_GL_EXT_BEGIN (invalidate_subdata, + GST_GL_API_OPENGL3 | + GST_GL_API_GLES2, + 4, 3, + 3, 0, + "ARB:\0", + "invalidate_subdata\0") +GST_GL_EXT_FUNCTION (void, InvalidateFramebuffer, + (GLenum target, + GLsizei numAttachments, + const GLenum* attachments)) +GST_GL_EXT_FUNCTION (void, InvalidateSubFramebuffer, + (GLenum target, + GLsizei numAttachments, + const GLenum* attachments, + GLint x, + GLint y, + GLsizei width, + GLsizei height)) +GST_GL_EXT_END () + +GST_GL_EXT_BEGIN (invalidate_subdata_no_gles, + GST_GL_API_OPENGL3, + 4, 3, + 255, 255, + "ARB:\0", + "invalidate_subdata\0") +GST_GL_EXT_FUNCTION (void, InvalidateTexSubImage, + (GLuint texture, + GLint level, + GLint xoffset, + GLint yoffset, + GLint zoffset, + GLsizei width, + GLsizei height, + GLsizei depth)) +GST_GL_EXT_FUNCTION (void, InvalidateTexImage, + (GLuint texture, + GLint level)) +GST_GL_EXT_FUNCTION (void, InvalidateBufferSubData, + (GLuint buffer, + GLintptr offset, + GLsizeiptr length)) +GST_GL_EXT_FUNCTION (void, InvalidateBufferData, + (GLuint buffer)) +GST_GL_EXT_END () \ No newline at end of file diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglfuncs.h b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglfuncs.h index 81ba4d5f99..1a8f44aac5 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglfuncs.h +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/gstglfuncs.h @@ -99,7 +99,7 @@ G_BEGIN_DECLS struct _GstGLFuncs { #include - gpointer padding[GST_PADDING_LARGE*6-2]; + gpointer padding[GST_PADDING_LARGE*6-8]; }; #undef GST_GL_EXT_BEGIN diff --git a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build index 4e4087fba0..8d9899be96 100644 --- a/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build +++ b/subprojects/gst-plugins-base/gst-libs/gst/gl/meson.build @@ -103,6 +103,7 @@ gl_prototype_headers = files([ 'glprototypes/gles.h', 'glprototypes/gstgl_compat.h', 'glprototypes/gstgl_gles2compat.h', + 'glprototypes/invalidate_subdata.h', 'glprototypes/opengl.h', 'glprototypes/query.h', 'glprototypes/shaders.h', diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index 04b8cd634f..c194448216 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -12450,6 +12450,18 @@ "type": "GstGLContext", "writable": false }, + "depth-buffer": { + "blurb": "Use depth and stencil buffer for the rendering of the scene. Might corrupt the rendering when set to FALSE! Only set to FALSE after carefully checking the targeted QML scene.", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "true", + "mutable": "ready", + "readable": true, + "type": "gboolean", + "writable": true + }, "qml-scene": { "blurb": "The contents of the QML scene", "conditionally-available": false, diff --git a/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.cc b/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.cc index 001bc2e6e4..4bb4173f58 100644 --- a/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.cc +++ b/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.cc @@ -145,6 +145,7 @@ enum PROP_WIDGET, PROP_QML_SCENE, PROP_ROOT_ITEM, + PROP_DEPTH_AND_STENCIL_BUFFER, }; enum @@ -204,6 +205,23 @@ gst_qt_overlay_class_init (GstQtOverlayClass * klass) "The root QQuickItem from the qml-scene used to render", (GParamFlags) (G_PARAM_READABLE | G_PARAM_STATIC_STRINGS))); + /** + * GstQtOverlay:depth-buffer: + * + * Instructs QML to not use a depth or stencil buffer when rendering. + * + * WARNING: Disabling the depth and stencil buffer may cause QML to + * incorrectly render the provided scene. + * + * Since: 1.26 + */ + g_object_class_install_property (gobject_class, PROP_DEPTH_AND_STENCIL_BUFFER, + g_param_spec_boolean ("depth-buffer", "Depth and Stencil Buffer", + "Use depth and stencil buffer for the rendering of the scene. " + "Might corrupt the rendering when set to FALSE! " + "Only set to FALSE after carefully checking the targeted QML scene.", TRUE, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + /** * GstQmlGLOverlay::qml-scene-initialized * @element: the #GstQmlGLOverlay @@ -243,6 +261,7 @@ static void gst_qt_overlay_init (GstQtOverlay * qt_overlay) { qt_overlay->widget = QSharedPointer(); + qt_overlay->depth_buffer = TRUE; qt_overlay->qml_scene = NULL; } @@ -278,6 +297,9 @@ gst_qt_overlay_set_property (GObject * object, guint prop_id, g_free (qt_overlay->qml_scene); qt_overlay->qml_scene = g_value_dup_string (value); break; + case PROP_DEPTH_AND_STENCIL_BUFFER: + qt_overlay->depth_buffer = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -316,6 +338,9 @@ gst_qt_overlay_get_property (GObject * object, guint prop_id, } GST_OBJECT_UNLOCK (qt_overlay); break; + case PROP_DEPTH_AND_STENCIL_BUFFER: + g_value_set_boolean(value, qt_overlay->depth_buffer); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -341,7 +366,7 @@ gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter) GST_OBJECT_LOCK (bfilter); qt_overlay->renderer = new GstQuickRenderer; - if (!qt_overlay->renderer->init (bfilter->context, &error)) { + if (!qt_overlay->renderer->init (bfilter->context, qt_overlay->depth_buffer, &error)) { GST_ELEMENT_ERROR (GST_ELEMENT (bfilter), RESOURCE, NOT_FOUND, ("%s", error->message), (NULL)); delete qt_overlay->renderer; diff --git a/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.h b/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.h index 238f81d5d0..f15fcfa069 100644 --- a/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.h +++ b/subprojects/gst-plugins-good/ext/qt/gstqtoverlay.h @@ -51,6 +51,8 @@ struct _GstQtOverlay /* */ GstGLFilter parent; + gboolean depth_buffer; + gchar *qml_scene; GstQuickRenderer *renderer; diff --git a/subprojects/gst-plugins-good/ext/qt/qtglrenderer.cc b/subprojects/gst-plugins-good/ext/qt/qtglrenderer.cc index 6eacf2d947..5992ed5f16 100644 --- a/subprojects/gst-plugins-good/ext/qt/qtglrenderer.cc +++ b/subprojects/gst-plugins-good/ext/qt/qtglrenderer.cc @@ -21,6 +21,14 @@ #define GST_CAT_DEFAULT gst_qt_gl_renderer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +// from GL_EXT_discard_framebuffer +#ifndef GL_DEPTH_EXT +#define GL_DEPTH_EXT 0x1801 +#endif +#ifndef GL_STENCIL_EXT +#define GL_STENCIL_EXT 0x1802 +#endif + static void init_debug (void) { @@ -217,6 +225,7 @@ GstQuickRenderer::GstQuickRenderer() gl_allocator(NULL), gl_params(NULL), gl_mem(NULL), + m_useDepthBuffer(TRUE), m_sharedRenderData(NULL) { init_debug (); @@ -292,7 +301,7 @@ bool CreateSurfaceWorker::event(QEvent * ev) return QObject::event(ev); } -bool GstQuickRenderer::init (GstGLContext * context, GError ** error) +bool GstQuickRenderer::init (GstGLContext * context, const gboolean use_depth_buffer, GError ** error) { g_return_val_if_fail (GST_IS_GL_CONTEXT (context), false); g_return_val_if_fail (gst_gl_context_get_current () == context, false); @@ -306,6 +315,9 @@ bool GstQuickRenderer::init (GstGLContext * context, GError ** error) return false; } + m_useDepthBuffer = use_depth_buffer; + GST_DEBUG ("%s", m_useDepthBuffer ? "Use depth and stencil buffer" : "Discard depth and stencil buffer"); + struct SharedRenderData *render_data = NULL, *old_render_data; do { if (render_data) @@ -551,7 +563,8 @@ void GstQuickRenderer::ensureFbo() if (!m_fbo) { m_fbo = new QOpenGLFramebufferObject(m_sharedRenderData->m_surface->size(), - QOpenGLFramebufferObject::CombinedDepthStencil); + m_useDepthBuffer ? QOpenGLFramebufferObject::CombinedDepthStencil : + QOpenGLFramebufferObject::NoAttachment); m_quickWindow->setRenderTarget(m_fbo); GST_DEBUG ("%p new framebuffer created with size %ix%i", this, m_fbo->size().width(), m_fbo->size().height()); @@ -584,6 +597,23 @@ GstQuickRenderer::renderGstGL () /* Meanwhile on this thread continue with the actual rendering. */ m_renderControl->render(); + { + GLint currentFbo = -1; + gl->GetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, ¤tFbo); + + if ((GLuint)currentFbo == m_fbo->handle()) { + if (gl->InvalidateFramebuffer) { + const GLenum attachments[] = { GL_DEPTH_ATTACHMENT, GL_STENCIL_ATTACHMENT }; + gl->InvalidateFramebuffer(GL_FRAMEBUFFER, sizeof(attachments) / sizeof(*attachments), attachments); + } else if (gl->DiscardFramebuffer) { + const GLenum attachments[] = { GL_DEPTH_EXT, GL_STENCIL_EXT }; + gl->DiscardFramebuffer(GL_FRAMEBUFFER, sizeof(attachments) / sizeof(*attachments), attachments); + } else { + GST_DEBUG ("Context lacks both - GL_ARB_invalidate_subdata and GL_EXT_discard_framebuffer, cannot discard"); + } + } + } + GST_DEBUG ("wrapping Qfbo %p with texture %u", m_fbo, m_fbo->texture()); struct FBOUserData *data = g_new0 (struct FBOUserData, 1); data->context = (GstGLContext *) gst_object_ref (gl_context); diff --git a/subprojects/gst-plugins-good/ext/qt/qtglrenderer.h b/subprojects/gst-plugins-good/ext/qt/qtglrenderer.h index b271b0bee4..fc2b126077 100644 --- a/subprojects/gst-plugins-good/ext/qt/qtglrenderer.h +++ b/subprojects/gst-plugins-good/ext/qt/qtglrenderer.h @@ -44,11 +44,10 @@ public: GstQuickRenderer(); ~GstQuickRenderer(); - /* initialize the GStreamer/Qt integration. On failure returns false - * and fills @error. - * Must be called with @context not wrapped and current in the current - * thread */ - bool init (GstGLContext * context, GError ** error); + /* initialize the GStreamer/Qt integration and determine wether to attach depth/stencil buffer to fbo. On failure + * returns false and fills @error. Must be called with @context not wrapped and current in the current thread + */ + bool init (GstGLContext * context, const gboolean use_depth_buffer, GError ** error); /* set the qml scene. returns false and fills @error on failure */ bool setQmlScene (const gchar * scene, GError ** error); @@ -101,7 +100,7 @@ private: GstGLAllocationParams *gl_params; GstVideoInfo v_info; GstGLMemory *gl_mem; - + gboolean m_useDepthBuffer; QString m_errorString; struct SharedRenderData *m_sharedRenderData; };