From 6efccf0ee18a3cf7f426a5acceaa716e7c41f5dd Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 4 Sep 2023 12:57:26 +1000 Subject: [PATCH] qml6/sink: add support for non-RGBA input Part-of: --- .../gst-plugins-good/ext/qt6/RGBA.frag | 24 + .../ext/qt6/YUV_TRIPLANAR.frag | 37 ++ .../gst-plugins-good/ext/qt6/gstqml6glsink.cc | 2 +- .../gst-plugins-good/ext/qt6/gstqsg6glnode.cc | 186 ------ .../gst-plugins-good/ext/qt6/gstqsg6glnode.h | 58 -- .../ext/qt6/gstqsg6material.cc | 611 ++++++++++++++++++ .../ext/qt6/gstqsg6material.h | 75 +++ .../gst-plugins-good/ext/qt6/meson.build | 30 +- .../gst-plugins-good/ext/qt6/qt6glitem.cc | 56 +- .../gst-plugins-good/ext/qt6/resources.qrc | 7 + .../gst-plugins-good/ext/qt6/vertex.vert | 23 + .../tests/examples/qt6/qmlsink/main.cpp | 8 +- 12 files changed, 848 insertions(+), 269 deletions(-) create mode 100644 subprojects/gst-plugins-good/ext/qt6/RGBA.frag create mode 100644 subprojects/gst-plugins-good/ext/qt6/YUV_TRIPLANAR.frag delete mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.cc delete mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.h create mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc create mode 100644 subprojects/gst-plugins-good/ext/qt6/gstqsg6material.h create mode 100644 subprojects/gst-plugins-good/ext/qt6/resources.qrc create mode 100644 subprojects/gst-plugins-good/ext/qt6/vertex.vert diff --git a/subprojects/gst-plugins-good/ext/qt6/RGBA.frag b/subprojects/gst-plugins-good/ext/qt6/RGBA.frag new file mode 100644 index 0000000000..c1ab120107 --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/RGBA.frag @@ -0,0 +1,24 @@ +#version 440 + +layout(location = 0) in vec2 vTexCoord; + +layout(location = 0) out vec4 fragColor; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +layout(binding = 1) uniform sampler2D tex; + +vec4 swizzle(in vec4 texel, in ivec4 swizzle) { + return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]); +} + +void main() +{ + vec4 texel = swizzle(texture(tex, vTexCoord), ubuf.swizzle); + fragColor = texel * ubuf.qt_Opacity; +} diff --git a/subprojects/gst-plugins-good/ext/qt6/YUV_TRIPLANAR.frag b/subprojects/gst-plugins-good/ext/qt6/YUV_TRIPLANAR.frag new file mode 100644 index 0000000000..6c09684893 --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/YUV_TRIPLANAR.frag @@ -0,0 +1,37 @@ +#version 440 + +layout(location = 0) in vec2 vTexCoord; + +layout(location = 0) out vec4 fragColor; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +layout(binding = 1) uniform sampler2D Ytex; +layout(binding = 2) uniform sampler2D Utex; +layout(binding = 3) uniform sampler2D Vtex; + +vec4 swizzle(in vec4 texel, in ivec4 swizzle) { + return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]); +} + +vec4 yuva_to_rgba(in vec4 yuva, in mat4 color_matrix) { + return yuva * color_matrix; +} + +void main() +{ + vec4 yuva; + yuva.x = texture(Ytex, vTexCoord).r; + yuva.y = texture(Utex, vTexCoord).r; + yuva.z = texture(Vtex, vTexCoord).r; + yuva.a = 1.0; + yuva = swizzle(yuva, ubuf.swizzle); + vec4 rgba = yuva_to_rgba (yuva, ubuf.color_matrix); + fragColor = rgba * ubuf.qt_Opacity; +} + diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc index 531bcea2ed..068d343a33 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc @@ -109,7 +109,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " - "format = (string) { RGB, RGBA }, " + "format = (string) { RGBA, BGRA, RGB, YV12 }, " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.cc b/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.cc deleted file mode 100644 index f36cb2a7d4..0000000000 --- a/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.cc +++ /dev/null @@ -1,186 +0,0 @@ -/* - * GStreamer - * Copyright (C) 2022 Matthew Waters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#include "gstqsg6glnode.h" - -#include -#include -#include -#include - -#define GST_CAT_DEFAULT gst_qsg_texture_debug -GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); - -GstQSG6OpenGLNode::GstQSG6OpenGLNode(QQuickItem * item) -{ - static gsize _debug; - - if (g_once_init_enter (&_debug)) { - GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsgtexture", 0, - "Qt Scenegraph Texture"); - g_once_init_leave (&_debug, 1); - } - - gst_video_info_init (&this->v_info); - - this->buffer_ = NULL; - this->sync_buffer_ = gst_buffer_new (); - this->dummy_tex_ = nullptr; - // TODO; handle windowChanged? - this->window_ = item->window(); -} - -GstQSG6OpenGLNode::~GstQSG6OpenGLNode() -{ - gst_buffer_replace (&this->buffer_, NULL); - gst_buffer_replace (&this->sync_buffer_, NULL); - this->buffer_was_bound = FALSE; - delete this->dummy_tex_; - this->dummy_tex_ = nullptr; -} - -QSGTexture * -GstQSG6OpenGLNode::texture() const -{ - return QSGSimpleTextureNode::texture(); -} - -/* only called from the streaming thread with scene graph thread blocked */ -void -GstQSG6OpenGLNode::setCaps (GstCaps * caps) -{ - GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps); - - if (caps) - gst_video_info_from_caps (&this->v_info, caps); - else - gst_video_info_init (&this->v_info); -} - -/* only called from the streaming thread with scene graph thread blocked */ -GstBuffer * -GstQSG6OpenGLNode::getBuffer () -{ - GstBuffer *buffer = NULL; - - if (this->buffer_) - buffer = gst_buffer_ref (this->buffer_); - - return buffer; -} - -/* only called from the streaming thread with scene graph thread blocked */ -void -GstQSG6OpenGLNode::setBuffer (GstBuffer * buffer) -{ - GstGLContext *qt_context = NULL; - gboolean buffer_changed; - - GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer); - /* FIXME: update more state here */ - buffer_changed = gst_buffer_replace (&this->buffer_, buffer); - - if (buffer_changed) { - GstGLContext *context; - GstGLSyncMeta *sync_meta; - GstMemory *mem; - guint tex_id; - QQuickWindow::CreateTextureOptions options = QQuickWindow::TextureHasAlphaChannel; - QSGTexture *texture = nullptr; - QSize texSize; - - qt_context = gst_gl_context_get_current(); - if (!qt_context) - goto use_dummy_tex; - - if (!this->buffer_) - goto use_dummy_tex; - if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN) - goto use_dummy_tex; - - this->mem_ = gst_buffer_peek_memory (this->buffer_, 0); - if (!this->mem_) - goto use_dummy_tex; - - /* FIXME: should really lock the memory to prevent write access */ - if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_, - (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) { - g_assert_not_reached (); - goto use_dummy_tex; - } - - mem = gst_buffer_peek_memory (this->buffer_, 0); - g_assert (gst_is_gl_memory (mem)); - - context = ((GstGLBaseMemory *)mem)->context; - - sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_); - if (!sync_meta) - sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_); - - gst_gl_sync_meta_set_sync_point (sync_meta, context); - - gst_gl_sync_meta_wait (sync_meta, qt_context); - - tex_id = *(guint *) this->v_frame.data[0]; - GST_LOG ("%p binding Qt texture %u", this, tex_id); - - texSize = QSize(GST_VIDEO_FRAME_WIDTH (&this->v_frame), GST_VIDEO_FRAME_HEIGHT (&this->v_frame)); - // XXX: ideally, we would like to subclass the relevant texture object - // ourselves but this is good enough for now - texture = QNativeInterface::QSGOpenGLTexture::fromNative(tex_id, this->window_, texSize, options); - - setTexture(texture); - setOwnsTexture(true); - markDirty(QSGNode::DirtyMaterial); - - gst_video_frame_unmap (&this->v_frame); - - /* Texture was successfully bound, so we do not need - * to use the dummy texture */ - } - - if (!texture()) { -use_dummy_tex: - /* Create dummy texture if not already present. */ - if (this->dummy_tex_ == nullptr) { - /* Make this a black 64x64 pixel RGBA texture. - * This size and format is supported pretty much everywhere, so these - * are a safe pick. (64 pixel sidelength must be supported according - * to the GLES2 spec, table 6.18.) - * Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */ - const int tex_sidelength = 64; - QImage image(tex_sidelength, tex_sidelength, QImage::Format_ARGB32); - image.fill(QColor(0, 0, 0, 255)); - - this->dummy_tex_ = this->window_->createTextureFromImage(image); - } - - g_assert (this->dummy_tex_ != nullptr); - - if (texture() != this->dummy_tex_) { - setTexture(this->dummy_tex_); - setOwnsTexture(false); - markDirty(QSGNode::DirtyMaterial); - } - - GST_LOG ("%p binding fallback dummy Qt texture %p", this, this->dummy_tex_); - } -} diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.h b/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.h deleted file mode 100644 index 0428fa5dea..0000000000 --- a/subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * GStreamer - * Copyright (C) 2022 Matthew Waters - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#pragma once - -#include -#include - -#include "gstqt6gl.h" -#include -#include -#include -#include -#include - -class GstQSG6OpenGLNode : public QSGTextureProvider, public QSGSimpleTextureNode, protected QOpenGLFunctions -{ - Q_OBJECT - -public: - GstQSG6OpenGLNode(QQuickItem *item); - ~GstQSG6OpenGLNode(); - - QSGTexture *texture() const override; - - void setCaps(GstCaps *caps); - void setBuffer(GstBuffer *buffer); - GstBuffer *getBuffer(); - - void updateQSGTexture(); - -private: - QQuickWindow *window_; - GstBuffer * buffer_; - gboolean buffer_was_bound; - GstBuffer * sync_buffer_; - GstMemory * mem_; - QSGTexture *dummy_tex_; - GstVideoInfo v_info; - GstVideoFrame v_frame; -}; diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc new file mode 100644 index 0000000000..9c078bbf6d --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc @@ -0,0 +1,611 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include +#include +#include +#include "gstqsg6material.h" +#include + +#define GST_CAT_DEFAULT gst_qsg_texture_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +/* matrix colour conversion code from vkvideoconvert.c */ +typedef struct +{ + double dm[4][4]; +} Matrix4; + +static void +matrix_debug (const Matrix4 * s) +{ + GST_DEBUG ("[%f %f %f %f]", s->dm[0][0], s->dm[0][1], s->dm[0][2], + s->dm[0][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[1][0], s->dm[1][1], s->dm[1][2], + s->dm[1][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[2][0], s->dm[2][1], s->dm[2][2], + s->dm[2][3]); + GST_DEBUG ("[%f %f %f %f]", s->dm[3][0], s->dm[3][1], s->dm[3][2], + s->dm[3][3]); +} + +static void +matrix_to_float (const Matrix4 * m, float *ret) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + ret[j * 4 + i] = m->dm[i][j]; + } + } +} + +static void +matrix_set_identity (Matrix4 * m) +{ + int i, j; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + m->dm[i][j] = (i == j); + } + } +} + +static void +matrix_copy (Matrix4 * d, const Matrix4 * s) +{ + gint i, j; + + for (i = 0; i < 4; i++) + for (j = 0; j < 4; j++) + d->dm[i][j] = s->dm[i][j]; +} + +/* Perform 4x4 matrix multiplication: + * - @dst@ = @a@ * @b@ + * - @dst@ may be a pointer to @a@ andor @b@ + */ +static void +matrix_multiply (Matrix4 * dst, Matrix4 * a, Matrix4 * b) +{ + Matrix4 tmp; + int i, j, k; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + double x = 0; + for (k = 0; k < 4; k++) { + x += a->dm[i][k] * b->dm[k][j]; + } + tmp.dm[i][j] = x; + } + } + matrix_copy (dst, &tmp); +} + +static void +matrix_offset_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][3] = a1; + a.dm[1][3] = a2; + a.dm[2][3] = a3; + matrix_debug (&a); + matrix_multiply (m, &a, m); +} + +static void +matrix_scale_components (Matrix4 * m, double a1, double a2, double a3) +{ + Matrix4 a; + + matrix_set_identity (&a); + a.dm[0][0] = a1; + a.dm[1][1] = a2; + a.dm[2][2] = a3; + matrix_multiply (m, &a, m); +} + +static void +matrix_YCbCr_to_RGB (Matrix4 * m, double Kr, double Kb) +{ + double Kg = 1.0 - Kr - Kb; + Matrix4 k = { + { + {1., 0., 2 * (1 - Kr), 0.}, + {1., -2 * Kb * (1 - Kb) / Kg, -2 * Kr * (1 - Kr) / Kg, 0.}, + {1., 2 * (1 - Kb), 0., 0.}, + {0., 0., 0., 1.}, + } + }; + + matrix_multiply (m, &k, m); +} + +static void +convert_to_RGB (GstVideoInfo *info, Matrix4 * m) +{ + { + const GstVideoFormatInfo *uinfo; + gint offset[4], scale[4], depth[4]; + guint i; + + uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info)); + + /* bring color components to [0..1.0] range */ + gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset, + scale); + + for (i = 0; i < uinfo->n_components; i++) + depth[i] = (1 << uinfo->depth[i]) - 1; + + matrix_offset_components (m, -offset[0] / (float) depth[0], + -offset[1] / (float) depth[1], -offset[2] / (float) depth[2]); + matrix_scale_components (m, depth[0] / ((float) scale[0]), + depth[1] / ((float) scale[1]), depth[2] / ((float) scale[2])); + GST_DEBUG ("to RGB scale/offset matrix"); + matrix_debug (m); + } + + if (GST_VIDEO_INFO_IS_YUV (info)) { + gdouble Kr, Kb; + + if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb)) + matrix_YCbCr_to_RGB (m, Kr, Kb); + GST_DEBUG ("to RGB matrix"); + matrix_debug (m); + } +} + +class GstQSGTexture : public QSGTexture { +public: + GstQSGTexture(QRhiTexture *); + ~GstQSGTexture(); + + qint64 comparisonKey() const override; + bool hasAlphaChannel() const override; + bool hasMipmaps() const override { return false; }; + bool isAtlasTexture() const override { return false; }; + QSize textureSize() const override; + + QRhiTexture *rhiTexture() const override; + +private: + QRhiTexture *m_texture; + bool m_has_alpha; +}; + +GstQSGTexture::GstQSGTexture(QRhiTexture * texture) + : m_texture(texture) +{ + switch (texture->format()) { + case QRhiTexture::RGBA8: + case QRhiTexture::RGB10A2: + case QRhiTexture::RGBA16F: + case QRhiTexture::RGBA32F: + this->m_has_alpha = true; + break; + default: + this->m_has_alpha = false; + } +} + +GstQSGTexture::~GstQSGTexture() +{ +} + +qint64 +GstQSGTexture::comparisonKey() const +{ + if (this->m_texture) + return qint64(qintptr(this->m_texture)); + + return qint64(qintptr(this)); +} + +bool +GstQSGTexture::hasAlphaChannel() const +{ + return m_has_alpha; +} + +QSize +GstQSGTexture::textureSize() const +{ + // XXX: currently unused + return QSize(0, 0); +} + +QRhiTexture * +GstQSGTexture::rhiTexture() const +{ + return m_texture; +} + +class GstQSGMaterialShader : public QSGMaterialShader { +public: + GstQSGMaterialShader(GstVideoFormat v_format); + ~GstQSGMaterialShader(); + + bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; + void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, QSGMaterial *newMaterial, QSGMaterial *) override; + +private: + GstVideoFormat v_format; + QSGTexture *m_textures[GST_VIDEO_MAX_PLANES]; +}; + +GstQSGMaterialShader::GstQSGMaterialShader(GstVideoFormat v_format) + : v_format(v_format) +{ + setShaderFileName(VertexStage, ":/org/freedesktop/gstreamer/qml6/vertex.vert.qsb"); + + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGB: + setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb"); + break; + case GST_VIDEO_FORMAT_YV12: + setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/YUV_TRIPLANAR.frag.qsb"); + break; + default: + g_assert_not_reached (); + } + + m_textures[0] = nullptr; + m_textures[1] = nullptr; + m_textures[2] = nullptr; + m_textures[3] = nullptr; +} + +GstQSGMaterialShader::~GstQSGMaterialShader() +{ +} + +bool +GstQSGMaterialShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format); + bool changed = false; + QByteArray *buf = state.uniformData(); + Q_ASSERT(buf->size() >= 84); + + GST_TRACE ("%p new material %p old material %p", this, newMaterial, oldMaterial); + + if (state.isMatrixDirty()) { + const QMatrix4x4 m = state.combinedMatrix(); + memcpy(buf->data(), m.constData(), 64); + changed = true; + } + + if (state.isOpacityDirty()) { + const float opacity = state.opacity(); + memcpy(buf->data() + 144, &opacity, 4); + changed = true; + } + + auto *mat = static_cast(newMaterial); + if (oldMaterial != newMaterial || mat->uniforms.dirty) { + memcpy(buf->data() + 64, &mat->uniforms.input_swizzle, 4 * sizeof (int)); + memcpy(buf->data() + 80, mat->uniforms.color_matrix.constData(), 64); + mat->uniforms.dirty = false; + changed = true; + } + + for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) { + if (this->m_textures[i]) { + delete this->m_textures[i]; + this->m_textures[i] = nullptr; + } + if (i < finfo->n_planes) + this->m_textures[i] = mat->bind(this, state.rhi(), state.resourceUpdateBatch(), i, v_format); + } + + return changed; +} + +void +GstQSGMaterialShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, + QSGMaterial *newMaterial, QSGMaterial *) +{ + *texture = this->m_textures[binding - 1]; + GST_TRACE ("%p binding:%d texture %p", this, binding, *texture); +} + +#define DEFINE_MATERIAL(format) \ +class G_PASTE(GstQSGMaterial_,format) : public GstQSGMaterial { \ +public: \ + G_PASTE(GstQSGMaterial_,format)(); \ + ~G_PASTE(GstQSGMaterial_,format)(); \ + QSGMaterialType *type() const override { static QSGMaterialType type; return &type; }; \ +}; \ +G_PASTE(GstQSGMaterial_,format)::G_PASTE(GstQSGMaterial_,format)() {} \ +G_PASTE(GstQSGMaterial_,format)::~G_PASTE(GstQSGMaterial_,format)() {} + +DEFINE_MATERIAL(RGBA_SWIZZLE); +DEFINE_MATERIAL(YUV_TRIPLANAR); + +GstQSGMaterial * +GstQSGMaterial::new_for_format(GstVideoFormat format) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (format); + + if (GST_VIDEO_FORMAT_INFO_IS_RGB (finfo) && finfo->n_planes == 1) { + return static_cast(new GstQSGMaterial_RGBA_SWIZZLE()); + } + + switch (format) { + case GST_VIDEO_FORMAT_YV12: + return static_cast(new GstQSGMaterial_YUV_TRIPLANAR()); + default: + g_assert_not_reached (); + } +} + +GstQSGMaterial::GstQSGMaterial () +{ + static gsize _debug; + + if (g_once_init_enter (&_debug)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsg6material", 0, + "Qt6 Scenegraph Material"); + g_once_init_leave (&_debug, 1); + } + + g_weak_ref_init (&this->qt_context_ref_, NULL); + gst_video_info_init (&this->v_info); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + + this->buffer_ = NULL; + this->buffer_was_bound = false; + this->sync_buffer_ = gst_buffer_new (); + + this->uniforms.dirty = true; +} + +GstQSGMaterial::~GstQSGMaterial () +{ + g_weak_ref_clear (&this->qt_context_ref_); + gst_buffer_replace (&this->buffer_, NULL); + gst_buffer_replace (&this->sync_buffer_, NULL); + this->buffer_was_bound = false; + + if (this->v_frame.buffer) { + gst_video_frame_unmap (&this->v_frame); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + } +} + +bool +GstQSGMaterial::compatibleWith(GstVideoInfo * v_info) +{ + if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info)) + return false; + + return true; +} + +QSGMaterialShader * +GstQSGMaterial::createShader(QSGRendererInterface::RenderMode renderMode) const +{ + GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info); + + return new GstQSGMaterialShader(v_format); +} + +/* only called from the streaming thread with scene graph thread blocked */ +void +GstQSGMaterial::setCaps (GstCaps * caps) +{ + GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps); + + gst_video_info_from_caps (&this->v_info, caps); +} + +/* only called from the streaming thread with scene graph thread blocked */ +gboolean +GstQSGMaterial::setBuffer (GstBuffer * buffer) +{ + GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer); + /* FIXME: update more state here */ + if (!gst_buffer_replace (&this->buffer_, buffer)) + return FALSE; + + this->buffer_was_bound = false; + + g_weak_ref_set (&this->qt_context_ref_, gst_gl_context_get_current ()); + + if (this->v_frame.buffer) { + gst_video_frame_unmap (&this->v_frame); + memset (&this->v_frame, 0, sizeof (this->v_frame)); + } + + if (this->buffer_) { + if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_, + (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) { + g_assert_not_reached (); + return FALSE; + } + gst_gl_video_format_swizzle(GST_VIDEO_INFO_FORMAT (&this->v_info), this->uniforms.input_swizzle); + + Matrix4 m; + float matrix_data[16] = { 0.0, }; + + matrix_set_identity (&m); + convert_to_RGB (&this->v_info, &m); + matrix_debug (&m); + matrix_to_float (&m, matrix_data); + + this->uniforms.color_matrix = QMatrix4x4(matrix_data); + this->uniforms.dirty = true; + } + + return TRUE; +} + +/* only called from the streaming thread with scene graph thread blocked */ +GstBuffer * +GstQSGMaterial::getBuffer (bool * was_bound) +{ + GstBuffer *buffer = NULL; + + if (this->buffer_) + buffer = gst_buffer_ref (this->buffer_); + if (was_bound) + *was_bound = this->buffer_was_bound; + + return buffer; +} + +void +GstQSGMaterial::setFiltering(QSGTexture::Filtering filtering) +{ + m_filtering = filtering; +} + +static QRhiTexture::Format +video_format_to_rhi_format (GstVideoFormat format, guint plane) +{ + switch (format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + return QRhiTexture::RGBA8; + case GST_VIDEO_FORMAT_YV12: + return QRhiTexture::RED_OR_ALPHA8; + default: + g_assert_not_reached (); + } +} + +QSGTexture * +GstQSGMaterial::bind(GstQSGMaterialShader *shader, QRhi * rhi, QRhiResourceUpdateBatch *res_updates, guint plane, GstVideoFormat v_format) +{ + GstGLContext *qt_context, *context; + GstMemory *mem; + GstGLMemory *gl_mem; + GstGLSyncMeta *sync_meta; + gboolean use_dummy_tex = TRUE; + guint tex_id; + GstQSGTexture *ret; + QRhiTexture *rhi_tex; + QSize tex_size; + + qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_)); + if (!qt_context) + goto out; + + if (!this->buffer_) + goto out; + if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN) + goto out; + + mem = gst_buffer_peek_memory (this->buffer_, plane); + g_assert (gst_is_gl_memory (mem)); + gl_mem = (GstGLMemory *) mem; + context = ((GstGLBaseMemory *)mem)->context; + + /* Texture was successfully bound, so we do not need + * to use the dummy texture */ + use_dummy_tex = FALSE; + + this->buffer_was_bound = true; + tex_id = *(guint *) this->v_frame.data[plane]; + + tex_size = QSize(gst_gl_memory_get_texture_width(gl_mem), gst_gl_memory_get_texture_height (gl_mem)); + + rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), tex_size, 1, {}); + rhi_tex->createFrom({(guint64) tex_id, 0}); + + sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_); + if (!sync_meta) + sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_); + + gst_gl_sync_meta_set_sync_point (sync_meta, context); + + gst_gl_sync_meta_wait (sync_meta, qt_context); + + GST_LOG ("%p binding GL texture %u for plane %d", this, tex_id, plane); + +out: + if (G_UNLIKELY (use_dummy_tex)) { + /* Create dummy texture if not already present. + * Use the Qt RHI functions instead of the GstGL ones. + */ + + /* Make this a black 64x64 pixel RGBA texture. + * This size and format is supported pretty much everywhere, so these + * are a safe pick. (64 pixel sidelength must be supported according + * to the GLES2 spec, table 6.18.) + * Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */ + const int tex_sidelength = 64; + std::vector < char > dummy_data (tex_sidelength * tex_sidelength * 4, 0); + + rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), QSize(tex_sidelength, tex_sidelength), 1, {}); + + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_RGB: + break; + case GST_VIDEO_FORMAT_YV12: + if (plane == 1 || plane == 2) { + char *data = dummy_data.data(); + for (gsize j = 0; j < tex_sidelength; j++) { + for (gsize k = 0; k < tex_sidelength; k++) { + data[(j * tex_sidelength + k) * 4 + 0] = 0x7F; + } + } + } + break; + default: + g_assert_not_reached (); + break; + } + + QRhiTextureSubresourceUploadDescription sub_desc; + + sub_desc.setData(QByteArray::fromRawData(dummy_data.data(), dummy_data.size())); + + QRhiTextureUploadEntry entry(0, 0, sub_desc); + QRhiTextureUploadDescription desc({ entry }); + res_updates->uploadTexture(rhi_tex, desc); + + GST_LOG ("%p binding for plane %d fallback dummy Qt texture", this, plane); + } + + ret = new GstQSGTexture(rhi_tex); + ret->setFiltering(m_filtering); + + gst_clear_object (&qt_context); + + return static_cast(ret); +} diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.h b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.h new file mode 100644 index 0000000000..42a2a87768 --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.h @@ -0,0 +1,75 @@ +/* + * GStreamer + * Copyright (C) 2023 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_QSG6_MATERIAL_H__ +#define __GST_QSG6_MATERIAL_H__ + +#include +#include +#include + +#include "gstqt6gl.h" +#include +#include +#include +#include + +class QRhi; +class QRhiResourceUpdateBatch; +class GstQSGMaterialShader; + +class GstQSGMaterial : public QSGMaterial +{ +protected: + GstQSGMaterial(); + ~GstQSGMaterial(); +public: + static GstQSGMaterial *new_for_format (GstVideoFormat format); + + void setCaps (GstCaps * caps); + gboolean setBuffer (GstBuffer * buffer); + GstBuffer * getBuffer (bool * was_bound); + bool compatibleWith(GstVideoInfo *v_info); + + void setFiltering(QSGTexture::Filtering); + + QSGTexture * bind(GstQSGMaterialShader *, QRhi *, QRhiResourceUpdateBatch *, guint binding, GstVideoFormat); + + /* QSGMaterial */ + QSGMaterialShader *createShader(QSGRendererInterface::RenderMode renderMode) const override; + + struct { + int input_swizzle[4]; + QMatrix4x4 color_matrix; + bool dirty; + } uniforms; + +private: + GstBuffer * buffer_; + bool buffer_was_bound; + GWeakRef qt_context_ref_; + GstBuffer * sync_buffer_; + GstMemory * mem_; + GstVideoInfo v_info; + GstVideoFrame v_frame; + QSGTexture::Filtering m_filtering; +}; + +#endif /* __GST_QSG6_MATERIAL_H__ */ diff --git a/subprojects/gst-plugins-good/ext/qt6/meson.build b/subprojects/gst-plugins-good/ext/qt6/meson.build index fc5f4909b0..71344e65e9 100644 --- a/subprojects/gst-plugins-good/ext/qt6/meson.build +++ b/subprojects/gst-plugins-good/ext/qt6/meson.build @@ -1,7 +1,7 @@ sources = [ 'gstplugin.cc', 'gstqt6element.cc', - 'gstqsg6glnode.cc', + 'gstqsg6material.cc', 'gstqt6glutility.cc', 'gstqml6glsink.cc', 'gstqml6glsrc.cc', @@ -14,11 +14,16 @@ sources = [ moc_headers = [ 'qt6glitem.h', - 'gstqsg6glnode.h', 'qt6glwindow.h', 'qt6glrenderer.h', ] +shader_sources = [ + 'vertex.vert', + 'RGBA.frag', + 'YUV_TRIPLANAR.frag', +] + qt6qml_dep = dependency('', required: false) qt6_option = get_option('qt6') qt6_egl = get_option('qt-egl') @@ -50,7 +55,7 @@ if not qt6_mod.has_tools(method: qt6_method) endif qt6qml_dep = dependency('qt6', modules : ['Core', 'Gui', 'Qml', 'Quick'], - method: qt6_method, required: qt6_option, static: host_system == 'ios') + method: qt6_method, required: qt6_option, static: host_system == 'ios', private_headers: true) if not qt6qml_dep.found() subdir_done() endif @@ -153,7 +158,24 @@ endif if qt6_option.require(have_qt_windowing).allowed() # Build it! moc_files = qt6_mod.preprocess(moc_headers : moc_headers, method: qt6_method) - gstqml6gl = library('gstqml6', sources, moc_files, + # TODO: dist backup qsb shaders? + qsb = find_program('qsb-qt6', 'qsb') + shaders = [] + foreach shader: shader_sources + qsb_shader = shader + '.qsb' + dist_shader = shader + '-dist.qsb' + + compiled_shader = custom_target(qsb_shader, + input: shader, + output: qsb_shader, + command: [qsb, '--glsl=100 es,120,330', '--batchable', '--output', '@OUTPUT@', '@INPUT@'] + ) + shaders += [compiled_shader] + endforeach + resource_file = configure_file(input: 'resources.qrc', output: 'resources.qrc', copy: true) + qresources = qt6_mod.compile_resources(sources: resource_file, method: qt6_method) + + gstqml6gl = library('gstqml6', sources, moc_files, qresources, cpp_args : gst_plugins_good_args + qt_defines, link_args : noseh_link_args, include_directories: [configinc, libsinc], diff --git a/subprojects/gst-plugins-good/ext/qt6/qt6glitem.cc b/subprojects/gst-plugins-good/ext/qt6/qt6glitem.cc index 26d2385ba0..c43e40d59a 100644 --- a/subprojects/gst-plugins-good/ext/qt6/qt6glitem.cc +++ b/subprojects/gst-plugins-good/ext/qt6/qt6glitem.cc @@ -26,14 +26,13 @@ #include #include "qt6glitem.h" -#include "gstqsg6glnode.h" #include "gstqt6glutility.h" +#include "gstqsg6material.h" #include #include #include #include -#include /** * SECTION:Qt6GLVideoItem @@ -88,8 +87,6 @@ struct _Qt6GLVideoItemPrivate * FIXME: Ideally we would use fences for this but there seems to be no * way to reliably "try wait" on a fence */ GQueue potentially_unbound_buffers; - - GstQSG6OpenGLNode *m_node; }; Qt6GLVideoItem::Qt6GLVideoItem() @@ -276,32 +273,55 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, UpdatePaintNodeData * updatePaintNodeData) { GstBuffer *old_buffer; + GstQSGMaterial *tex = nullptr; + QSGGeometry *geometry = nullptr; + bool was_bound = false; if (!this->priv->initted) return oldNode; - GstQSG6OpenGLNode *texNode = static_cast (oldNode); + QSGGeometryNode *texNode = static_cast (oldNode); GstVideoRectangle src, dst, result; g_mutex_lock (&this->priv->lock); GST_TRACE ("%p updatePaintNode", this); + if (!this->priv->caps) { + GST_LOG ("%p no caps yet", this); + g_mutex_unlock (&this->priv->lock); + return NULL; + } + if (gst_gl_context_get_current() == NULL) gst_gl_context_activate (this->priv->other_context, TRUE); - if (!texNode) { - bool is_smooth = this->smooth (); - texNode = new GstQSG6OpenGLNode (this); - texNode->setFiltering (is_smooth ? QSGTexture::Filtering::Linear : - QSGTexture::Filtering::Nearest); - this->priv->m_node = texNode; + if (texNode) { + tex = static_cast(texNode->material()); + if (tex && !tex->compatibleWith(&this->priv->v_info)) { + delete texNode; + texNode = nullptr; + } } - if ((old_buffer = texNode->getBuffer())) { + if (!texNode) { + bool is_smooth = this->smooth (); + texNode = new QSGGeometryNode(); + geometry = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4); + texNode->setGeometry(geometry); + tex = GstQSGMaterial::new_for_format(GST_VIDEO_INFO_FORMAT (&this->priv->v_info)); + tex->setFiltering(is_smooth ? QSGTexture::Filtering::Linear : + QSGTexture::Filtering::Nearest); + texNode->setMaterial(tex); + } + + if ((old_buffer = tex->getBuffer(&was_bound))) { if (old_buffer == this->priv->buffer) { /* same buffer */ gst_buffer_unref (old_buffer); + } else if (!was_bound) { + GST_TRACE ("old buffer %p was not bound yet, unreffing", old_buffer); + gst_buffer_unref (old_buffer); } else { GstBuffer *tmp_buffer; @@ -326,8 +346,8 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, old_buffer = NULL; } - texNode->setCaps (this->priv->caps); - texNode->setBuffer (this->priv->buffer); + tex->setCaps (this->priv->caps); + tex->setBuffer (this->priv->buffer); if (this->priv->force_aspect_ratio && this->priv->caps) { src.w = this->priv->display_width; @@ -346,7 +366,10 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode, result.h = boundingRect().height(); } - texNode->setRect (QRectF (result.x, result.y, result.w, result.h)); + geometry = texNode->geometry(); + QRectF rect(result.x, result.y, result.w, result.h); + QRectF sourceRect(0, 0, 1, 1); + QSGGeometry::updateTexturedRectGeometry(geometry, rect, sourceRect); g_mutex_unlock (&this->priv->lock); @@ -713,7 +736,6 @@ Qt6GLVideoItem::onSceneGraphInitialized () void Qt6GLVideoItem::onSceneGraphInvalidated () { - this->priv->m_node = nullptr; GST_FIXME ("%p scene graph invalidated", this); } @@ -790,13 +812,11 @@ Qt6GLVideoItem::handleWindowChanged (QQuickWindow * win) this->priv->qt_context = NULL; this->priv->initted = FALSE; } - this->priv->m_node = nullptr; } void Qt6GLVideoItem::releaseResources() { - this->priv->m_node = nullptr; } gboolean diff --git a/subprojects/gst-plugins-good/ext/qt6/resources.qrc b/subprojects/gst-plugins-good/ext/qt6/resources.qrc new file mode 100644 index 0000000000..7a01e20f68 --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/resources.qrc @@ -0,0 +1,7 @@ + + + vertex.vert.qsb + RGBA.frag.qsb + YUV_TRIPLANAR.frag.qsb + + diff --git a/subprojects/gst-plugins-good/ext/qt6/vertex.vert b/subprojects/gst-plugins-good/ext/qt6/vertex.vert new file mode 100644 index 0000000000..f67117731d --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/vertex.vert @@ -0,0 +1,23 @@ +#version 440 + +layout(location = 0) in vec4 aVertex; +layout(location = 1) in vec2 aTexCoord; + +layout(location = 0) out vec2 vTexCoord; + +layout(std140, binding = 0) uniform buf { + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +} ubuf; + +out gl_PerVertex { + vec4 gl_Position; +}; + +void main() +{ + gl_Position = ubuf.qt_Matrix * aVertex; + vTexCoord = aTexCoord; +} diff --git a/subprojects/gst-plugins-good/tests/examples/qt6/qmlsink/main.cpp b/subprojects/gst-plugins-good/tests/examples/qt6/qmlsink/main.cpp index 81f305252b..38c681dd06 100644 --- a/subprojects/gst-plugins-good/tests/examples/qt6/qmlsink/main.cpp +++ b/subprojects/gst-plugins-good/tests/examples/qt6/qmlsink/main.cpp @@ -48,6 +48,10 @@ int main(int argc, char *argv[]) GstElement *pipeline = gst_pipeline_new (NULL); GstElement *src = gst_element_factory_make ("videotestsrc", NULL); + GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL); + GstCaps *caps = gst_caps_from_string ("video/x-raw,format=YV12"); + g_object_set (capsfilter, "caps", caps, NULL); + gst_clear_caps (&caps); GstElement *glupload = gst_element_factory_make ("glupload", NULL); /* the plugin must be loaded before loading the qml file to register the * GstGLVideoItem qml item */ @@ -55,8 +59,8 @@ int main(int argc, char *argv[]) g_assert (src && glupload && sink); - gst_bin_add_many (GST_BIN (pipeline), src, glupload, sink, NULL); - gst_element_link_many (src, glupload, sink, NULL); + gst_bin_add_many (GST_BIN (pipeline), src, capsfilter, glupload, sink, NULL); + gst_element_link_many (src, capsfilter, glupload, sink, NULL); QQmlApplicationEngine engine; engine.load(QUrl(QStringLiteral("qrc:/main.qml")));