qml6/sink: add support for non-RGBA input

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5290>

subprojects/gst-plugins-good/ext/qt6/RGBA.frag

@@ -0,0 +1,24 @@
+#version 440
+
+layout(location = 0) in vec2 vTexCoord;
+
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+  mat4 qt_Matrix;
+  ivec4 swizzle;
+  mat4 color_matrix;
+  float qt_Opacity;
+} ubuf;
+
+layout(binding = 1) uniform sampler2D tex;
+
+vec4 swizzle(in vec4 texel, in ivec4 swizzle) {
+  return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]);
+}
+
+void main()
+{
+  vec4 texel = swizzle(texture(tex, vTexCoord), ubuf.swizzle);
+  fragColor = texel * ubuf.qt_Opacity;
+}

subprojects/gst-plugins-good/ext/qt6/YUV_TRIPLANAR.frag

@@ -0,0 +1,37 @@
+#version 440
+
+layout(location = 0) in vec2 vTexCoord;
+
+layout(location = 0) out vec4 fragColor;
+
+layout(std140, binding = 0) uniform buf {
+  mat4 qt_Matrix;
+  ivec4 swizzle;
+  mat4 color_matrix;
+  float qt_Opacity;
+} ubuf;
+
+layout(binding = 1) uniform sampler2D Ytex;
+layout(binding = 2) uniform sampler2D Utex;
+layout(binding = 3) uniform sampler2D Vtex;
+
+vec4 swizzle(in vec4 texel, in ivec4 swizzle) {
+  return vec4(texel[swizzle[0]], texel[swizzle[1]], texel[swizzle[2]], texel[swizzle[3]]);
+}
+
+vec4 yuva_to_rgba(in vec4 yuva, in mat4 color_matrix) {
+  return yuva * color_matrix;
+}
+
+void main()
+{
+  vec4 yuva;
+  yuva.x = texture(Ytex, vTexCoord).r;
+  yuva.y = texture(Utex, vTexCoord).r;
+  yuva.z = texture(Vtex, vTexCoord).r;
+  yuva.a = 1.0;
+  yuva = swizzle(yuva, ubuf.swizzle);
+  vec4 rgba = yuva_to_rgba (yuva, ubuf.color_matrix);
+  fragColor = rgba * ubuf.qt_Opacity;
+}
+

subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc

@@ -109,7 +109,7 @@ GST_STATIC_PAD_TEMPLATE ("sink",
     GST_PAD_SINK,
     GST_PAD_ALWAYS,
     GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
-    "format = (string) { RGB, RGBA }, "
+    "format = (string) { RGBA, BGRA, RGB, YV12 }, "
     "width = " GST_VIDEO_SIZE_RANGE ", "
     "height = " GST_VIDEO_SIZE_RANGE ", "
     "framerate = " GST_VIDEO_FPS_RANGE ", "

subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.cc

@@ -1,186 +0,0 @@
-/*
- * GStreamer
- * Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#include "gstqsg6glnode.h"
-
-#include <QtQuick/QSGTextureProvider>
-#include <QtQuick/QSGSimpleTextureNode>
-#include <QtQuick/QQuickWindow>
-#include <QtQuick/QSGTexture>
-
-#define GST_CAT_DEFAULT gst_qsg_texture_debug
-GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
-
-GstQSG6OpenGLNode::GstQSG6OpenGLNode(QQuickItem * item)
-{
-  static gsize _debug;
-
-  if (g_once_init_enter (&_debug)) {
-    GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsgtexture", 0,
-        "Qt Scenegraph Texture");
-    g_once_init_leave (&_debug, 1);
-  }
-
-  gst_video_info_init (&this->v_info);
-
-  this->buffer_ = NULL;
-  this->sync_buffer_ = gst_buffer_new ();
-  this->dummy_tex_ = nullptr;
-  // TODO; handle windowChanged?
-  this->window_ = item->window();
-}
-
-GstQSG6OpenGLNode::~GstQSG6OpenGLNode()
-{
-  gst_buffer_replace (&this->buffer_, NULL);
-  gst_buffer_replace (&this->sync_buffer_, NULL);
-  this->buffer_was_bound = FALSE;
-  delete this->dummy_tex_;
-  this->dummy_tex_ = nullptr;
-}
-
-QSGTexture *
-GstQSG6OpenGLNode::texture() const
-{
-  return QSGSimpleTextureNode::texture();
-}
-
-/* only called from the streaming thread with scene graph thread blocked */
-void
-GstQSG6OpenGLNode::setCaps (GstCaps * caps)
-{
-  GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps);
-
-  if (caps)
-    gst_video_info_from_caps (&this->v_info, caps);
-  else
-    gst_video_info_init (&this->v_info);
-}
-
-/* only called from the streaming thread with scene graph thread blocked */
-GstBuffer *
-GstQSG6OpenGLNode::getBuffer ()
-{
-  GstBuffer *buffer = NULL;
-
-  if (this->buffer_)
-    buffer = gst_buffer_ref (this->buffer_);
-
-  return buffer;
-}
-
-/* only called from the streaming thread with scene graph thread blocked */
-void
-GstQSG6OpenGLNode::setBuffer (GstBuffer * buffer)
-{
-  GstGLContext *qt_context = NULL;
-  gboolean buffer_changed;
-
-  GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer);
-  /* FIXME: update more state here */
-  buffer_changed = gst_buffer_replace (&this->buffer_, buffer);
-
-  if (buffer_changed) {
-    GstGLContext *context;
-    GstGLSyncMeta *sync_meta;
-    GstMemory *mem;
-    guint tex_id;
-    QQuickWindow::CreateTextureOptions options = QQuickWindow::TextureHasAlphaChannel;
-    QSGTexture *texture = nullptr;
-    QSize texSize;
-
-    qt_context = gst_gl_context_get_current();
-    if (!qt_context)
-      goto use_dummy_tex;
-
-    if (!this->buffer_)
-      goto use_dummy_tex;
-    if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN)
-      goto use_dummy_tex;
-
-    this->mem_ = gst_buffer_peek_memory (this->buffer_, 0);
-    if (!this->mem_)
-      goto use_dummy_tex;
-
-    /* FIXME: should really lock the memory to prevent write access */
-    if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_,
-          (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) {
-      g_assert_not_reached ();
-      goto use_dummy_tex;
-    }
-
-    mem = gst_buffer_peek_memory (this->buffer_, 0);
-    g_assert (gst_is_gl_memory (mem));
-
-    context = ((GstGLBaseMemory *)mem)->context;
-
-    sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_);
-    if (!sync_meta)
-      sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_);
-
-    gst_gl_sync_meta_set_sync_point (sync_meta, context);
-
-    gst_gl_sync_meta_wait (sync_meta, qt_context);
-
-    tex_id = *(guint *) this->v_frame.data[0];
-    GST_LOG ("%p binding Qt texture %u", this, tex_id);
-
-    texSize = QSize(GST_VIDEO_FRAME_WIDTH (&this->v_frame), GST_VIDEO_FRAME_HEIGHT (&this->v_frame));
-    // XXX: ideally, we would like to subclass the relevant texture object
-    // ourselves but this is good enough for now
-    texture = QNativeInterface::QSGOpenGLTexture::fromNative(tex_id, this->window_, texSize, options);
-
-    setTexture(texture);
-    setOwnsTexture(true);
-    markDirty(QSGNode::DirtyMaterial);
-
-    gst_video_frame_unmap (&this->v_frame);
-
-    /* Texture was successfully bound, so we do not need
-     * to use the dummy texture */
-  }
-
-  if (!texture()) {
-use_dummy_tex:
-    /* Create dummy texture if not already present. */
-    if (this->dummy_tex_ == nullptr) {
-      /* Make this a black 64x64 pixel RGBA texture.
-       * This size and format is supported pretty much everywhere, so these
-       * are a safe pick. (64 pixel sidelength must be supported according
-       * to the GLES2 spec, table 6.18.)
-       * Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */
-      const int tex_sidelength = 64;
-      QImage image(tex_sidelength, tex_sidelength, QImage::Format_ARGB32);
-      image.fill(QColor(0, 0, 0, 255));
-
-      this->dummy_tex_ = this->window_->createTextureFromImage(image);
-    }
-
-    g_assert (this->dummy_tex_ != nullptr);
-
-    if (texture() != this->dummy_tex_) {
-      setTexture(this->dummy_tex_);
-      setOwnsTexture(false);
-      markDirty(QSGNode::DirtyMaterial);
-    }
-
-    GST_LOG ("%p binding fallback dummy Qt texture %p", this, this->dummy_tex_);
-  }
-}

subprojects/gst-plugins-good/ext/qt6/gstqsg6glnode.h

@@ -1,58 +0,0 @@
-/*
- * GStreamer
- * Copyright (C) 2022 Matthew Waters <matthew@centricular.com>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#pragma once
-
-#include <gst/gst.h>
-#include <gst/gl/gl.h>
-
-#include "gstqt6gl.h"
-#include <QtQuick/QQuickItem>
-#include <QtQuick/QSGTexture>
-#include <QtQuick/QSGTextureProvider>
-#include <QtQuick/QSGSimpleTextureNode>
-#include <QtGui/QOpenGLFunctions>
-
-class GstQSG6OpenGLNode : public QSGTextureProvider, public QSGSimpleTextureNode, protected QOpenGLFunctions
-{
-  Q_OBJECT
-
-public:
-  GstQSG6OpenGLNode(QQuickItem *item);
-  ~GstQSG6OpenGLNode();
-
-  QSGTexture *texture() const override;
-
-  void setCaps(GstCaps *caps);
-  void setBuffer(GstBuffer *buffer);
-  GstBuffer *getBuffer();
-
-  void updateQSGTexture();
-
-private:
-  QQuickWindow *window_;
-  GstBuffer * buffer_;
-  gboolean buffer_was_bound;
-  GstBuffer * sync_buffer_;
-  GstMemory * mem_;
-  QSGTexture *dummy_tex_;
-  GstVideoInfo v_info;
-  GstVideoFrame v_frame;
-};

subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc

@@ -0,0 +1,611 @@
+/*
+ * GStreamer
+ * Copyright (C) 2023 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <vector>
+#include <stdio.h>
+
+#include <gst/video/video.h>
+#include <gst/gl/gl.h>
+#include <gst/gl/gstglfuncs.h>
+#include "gstqsg6material.h"
+#include <private/qrhi_p.h>
+
+#define GST_CAT_DEFAULT gst_qsg_texture_debug
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
+
+/* matrix colour conversion code from vkvideoconvert.c */
+typedef struct
+{
+  double dm[4][4];
+} Matrix4;
+
+static void
+matrix_debug (const Matrix4 * s)
+{
+  GST_DEBUG ("[%f %f %f %f]", s->dm[0][0], s->dm[0][1], s->dm[0][2],
+      s->dm[0][3]);
+  GST_DEBUG ("[%f %f %f %f]", s->dm[1][0], s->dm[1][1], s->dm[1][2],
+      s->dm[1][3]);
+  GST_DEBUG ("[%f %f %f %f]", s->dm[2][0], s->dm[2][1], s->dm[2][2],
+      s->dm[2][3]);
+  GST_DEBUG ("[%f %f %f %f]", s->dm[3][0], s->dm[3][1], s->dm[3][2],
+      s->dm[3][3]);
+}
+
+static void
+matrix_to_float (const Matrix4 * m, float *ret)
+{
+  int i, j;
+
+  for (i = 0; i < 4; i++) {
+    for (j = 0; j < 4; j++) {
+      ret[j * 4 + i] = m->dm[i][j];
+    }
+  }
+}
+
+static void
+matrix_set_identity (Matrix4 * m)
+{
+  int i, j;
+
+  for (i = 0; i < 4; i++) {
+    for (j = 0; j < 4; j++) {
+      m->dm[i][j] = (i == j);
+    }
+  }
+}
+
+static void
+matrix_copy (Matrix4 * d, const Matrix4 * s)
+{
+  gint i, j;
+
+  for (i = 0; i < 4; i++)
+    for (j = 0; j < 4; j++)
+      d->dm[i][j] = s->dm[i][j];
+}
+
+/* Perform 4x4 matrix multiplication:
+ *  - @dst@ = @a@ * @b@
+ *  - @dst@ may be a pointer to @a@ andor @b@
+ */
+static void
+matrix_multiply (Matrix4 * dst, Matrix4 * a, Matrix4 * b)
+{
+  Matrix4 tmp;
+  int i, j, k;
+
+  for (i = 0; i < 4; i++) {
+    for (j = 0; j < 4; j++) {
+      double x = 0;
+      for (k = 0; k < 4; k++) {
+        x += a->dm[i][k] * b->dm[k][j];
+      }
+      tmp.dm[i][j] = x;
+    }
+  }
+  matrix_copy (dst, &tmp);
+}
+
+static void
+matrix_offset_components (Matrix4 * m, double a1, double a2, double a3)
+{
+  Matrix4 a;
+
+  matrix_set_identity (&a);
+  a.dm[0][3] = a1;
+  a.dm[1][3] = a2;
+  a.dm[2][3] = a3;
+  matrix_debug (&a);
+  matrix_multiply (m, &a, m);
+}
+
+static void
+matrix_scale_components (Matrix4 * m, double a1, double a2, double a3)
+{
+  Matrix4 a;
+
+  matrix_set_identity (&a);
+  a.dm[0][0] = a1;
+  a.dm[1][1] = a2;
+  a.dm[2][2] = a3;
+  matrix_multiply (m, &a, m);
+}
+
+static void
+matrix_YCbCr_to_RGB (Matrix4 * m, double Kr, double Kb)
+{
+  double Kg = 1.0 - Kr - Kb;
+  Matrix4 k = {
+    {
+          {1., 0., 2 * (1 - Kr), 0.},
+          {1., -2 * Kb * (1 - Kb) / Kg, -2 * Kr * (1 - Kr) / Kg, 0.},
+          {1., 2 * (1 - Kb), 0., 0.},
+          {0., 0., 0., 1.},
+        }
+  };
+
+  matrix_multiply (m, &k, m);
+}
+
+static void
+convert_to_RGB (GstVideoInfo *info, Matrix4 * m)
+{
+  {
+    const GstVideoFormatInfo *uinfo;
+    gint offset[4], scale[4], depth[4];
+    guint i;
+
+    uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info));
+
+    /* bring color components to [0..1.0] range */
+    gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset,
+        scale);
+
+    for (i = 0; i < uinfo->n_components; i++)
+      depth[i] = (1 << uinfo->depth[i]) - 1;
+
+    matrix_offset_components (m, -offset[0] / (float) depth[0],
+        -offset[1] / (float) depth[1], -offset[2] / (float) depth[2]);
+    matrix_scale_components (m, depth[0] / ((float) scale[0]),
+        depth[1] / ((float) scale[1]), depth[2] / ((float) scale[2]));
+    GST_DEBUG ("to RGB scale/offset matrix");
+    matrix_debug (m);
+  }
+
+  if (GST_VIDEO_INFO_IS_YUV (info)) {
+    gdouble Kr, Kb;
+
+    if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb))
+      matrix_YCbCr_to_RGB (m, Kr, Kb);
+    GST_DEBUG ("to RGB matrix");
+    matrix_debug (m);
+  }
+}
+
+class GstQSGTexture : public QSGTexture {
+public:
+  GstQSGTexture(QRhiTexture *);
+  ~GstQSGTexture();
+
+  qint64 comparisonKey() const override;
+  bool hasAlphaChannel() const override;
+  bool hasMipmaps() const override { return false; };
+  bool isAtlasTexture() const override { return false; };
+  QSize textureSize() const override;
+
+  QRhiTexture *rhiTexture() const override;
+
+private:
+  QRhiTexture *m_texture;
+  bool m_has_alpha;
+};
+
+GstQSGTexture::GstQSGTexture(QRhiTexture * texture)
+  : m_texture(texture)
+{
+  switch (texture->format()) {
+    case QRhiTexture::RGBA8:
+    case QRhiTexture::RGB10A2:
+    case QRhiTexture::RGBA16F:
+    case QRhiTexture::RGBA32F:
+      this->m_has_alpha = true;
+      break;
+    default:
+      this->m_has_alpha = false;
+  }
+}
+
+GstQSGTexture::~GstQSGTexture()
+{
+}
+
+qint64
+GstQSGTexture::comparisonKey() const
+{
+  if (this->m_texture)
+      return qint64(qintptr(this->m_texture));
+
+  return qint64(qintptr(this));
+}
+
+bool
+GstQSGTexture::hasAlphaChannel() const
+{
+  return m_has_alpha;
+}
+
+QSize
+GstQSGTexture::textureSize() const
+{
+  // XXX: currently unused
+  return QSize(0, 0);
+}
+
+QRhiTexture *
+GstQSGTexture::rhiTexture() const
+{
+  return m_texture;
+}
+
+class GstQSGMaterialShader : public QSGMaterialShader {
+public:
+  GstQSGMaterialShader(GstVideoFormat v_format);
+  ~GstQSGMaterialShader();
+
+  bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override;
+  void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, QSGMaterial *newMaterial, QSGMaterial *) override;
+
+private:
+  GstVideoFormat v_format;
+  QSGTexture *m_textures[GST_VIDEO_MAX_PLANES];
+};
+
+GstQSGMaterialShader::GstQSGMaterialShader(GstVideoFormat v_format)
+  : v_format(v_format)
+{
+  setShaderFileName(VertexStage, ":/org/freedesktop/gstreamer/qml6/vertex.vert.qsb");
+
+  switch (v_format) {
+    case GST_VIDEO_FORMAT_RGBA:
+    case GST_VIDEO_FORMAT_BGRA:
+    case GST_VIDEO_FORMAT_RGB:
+      setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb");
+      break;
+    case GST_VIDEO_FORMAT_YV12:
+      setShaderFileName(FragmentStage, ":/org/freedesktop/gstreamer/qml6/YUV_TRIPLANAR.frag.qsb");
+      break;
+    default:
+      g_assert_not_reached ();
+  }
+
+  m_textures[0] = nullptr;
+  m_textures[1] = nullptr;
+  m_textures[2] = nullptr;
+  m_textures[3] = nullptr;
+}
+
+GstQSGMaterialShader::~GstQSGMaterialShader()
+{
+}
+
+bool
+GstQSGMaterialShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial)
+{
+  const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format);
+  bool changed = false;
+  QByteArray *buf = state.uniformData();
+  Q_ASSERT(buf->size() >= 84);
+
+  GST_TRACE ("%p new material %p old material %p", this, newMaterial, oldMaterial);
+
+  if (state.isMatrixDirty()) {
+    const QMatrix4x4 m = state.combinedMatrix();
+    memcpy(buf->data(), m.constData(), 64);
+    changed = true;
+  }
+
+  if (state.isOpacityDirty()) {
+    const float opacity = state.opacity();
+    memcpy(buf->data() + 144, &opacity, 4);
+    changed = true;
+  }
+
+  auto *mat = static_cast<GstQSGMaterial *>(newMaterial);
+  if (oldMaterial != newMaterial || mat->uniforms.dirty) {
+    memcpy(buf->data() + 64, &mat->uniforms.input_swizzle, 4 * sizeof (int));
+    memcpy(buf->data() + 80, mat->uniforms.color_matrix.constData(), 64);
+    mat->uniforms.dirty = false;
+    changed = true;
+  }
+
+  for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) {
+    if (this->m_textures[i]) {
+      delete this->m_textures[i];
+      this->m_textures[i] = nullptr;
+    }
+    if (i < finfo->n_planes)
+      this->m_textures[i] = mat->bind(this, state.rhi(), state.resourceUpdateBatch(), i, v_format);
+  }
+
+  return changed;
+}
+
+void
+GstQSGMaterialShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture,
+    QSGMaterial *newMaterial, QSGMaterial *)
+{
+  *texture = this->m_textures[binding - 1];
+  GST_TRACE ("%p binding:%d texture %p", this, binding, *texture);
+}
+
+#define DEFINE_MATERIAL(format) \
+class G_PASTE(GstQSGMaterial_,format) : public GstQSGMaterial { \
+public: \
+  G_PASTE(GstQSGMaterial_,format)(); \
+  ~G_PASTE(GstQSGMaterial_,format)(); \
+  QSGMaterialType *type() const override { static QSGMaterialType type; return &type; }; \
+}; \
+G_PASTE(GstQSGMaterial_,format)::G_PASTE(GstQSGMaterial_,format)() {} \
+G_PASTE(GstQSGMaterial_,format)::~G_PASTE(GstQSGMaterial_,format)() {}
+
+DEFINE_MATERIAL(RGBA_SWIZZLE);
+DEFINE_MATERIAL(YUV_TRIPLANAR);
+
+GstQSGMaterial *
+GstQSGMaterial::new_for_format(GstVideoFormat format)
+{
+  const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
+
+  if (GST_VIDEO_FORMAT_INFO_IS_RGB (finfo) && finfo->n_planes == 1) {
+    return static_cast<GstQSGMaterial *>(new GstQSGMaterial_RGBA_SWIZZLE());
+  }
+
+  switch (format) {
+    case GST_VIDEO_FORMAT_YV12:
+      return static_cast<GstQSGMaterial *>(new GstQSGMaterial_YUV_TRIPLANAR());
+    default:
+      g_assert_not_reached ();
+  }
+}
+
+GstQSGMaterial::GstQSGMaterial ()
+{
+  static gsize _debug;
+
+  if (g_once_init_enter (&_debug)) {
+    GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsg6material", 0,
+        "Qt6 Scenegraph Material");
+    g_once_init_leave (&_debug, 1);
+  }
+
+  g_weak_ref_init (&this->qt_context_ref_, NULL);
+  gst_video_info_init (&this->v_info);
+  memset (&this->v_frame, 0, sizeof (this->v_frame));
+
+  this->buffer_ = NULL;
+  this->buffer_was_bound = false;
+  this->sync_buffer_ = gst_buffer_new ();
+
+  this->uniforms.dirty = true;
+}
+
+GstQSGMaterial::~GstQSGMaterial ()
+{
+  g_weak_ref_clear (&this->qt_context_ref_);
+  gst_buffer_replace (&this->buffer_, NULL);
+  gst_buffer_replace (&this->sync_buffer_, NULL);
+  this->buffer_was_bound = false;
+
+  if (this->v_frame.buffer) {
+    gst_video_frame_unmap (&this->v_frame);
+    memset (&this->v_frame, 0, sizeof (this->v_frame));
+  }
+}
+
+bool
+GstQSGMaterial::compatibleWith(GstVideoInfo * v_info)
+{
+  if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info))
+    return false;
+
+  return true;
+}
+
+QSGMaterialShader *
+GstQSGMaterial::createShader(QSGRendererInterface::RenderMode renderMode) const
+{
+  GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info);
+
+  return new GstQSGMaterialShader(v_format);
+}
+
+/* only called from the streaming thread with scene graph thread blocked */
+void
+GstQSGMaterial::setCaps (GstCaps * caps)
+{
+  GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps);
+
+  gst_video_info_from_caps (&this->v_info, caps);
+}
+
+/* only called from the streaming thread with scene graph thread blocked */
+gboolean
+GstQSGMaterial::setBuffer (GstBuffer * buffer)
+{
+  GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer);
+  /* FIXME: update more state here */
+  if (!gst_buffer_replace (&this->buffer_, buffer))
+    return FALSE;
+
+  this->buffer_was_bound = false;
+
+  g_weak_ref_set (&this->qt_context_ref_, gst_gl_context_get_current ());
+
+  if (this->v_frame.buffer) {
+    gst_video_frame_unmap (&this->v_frame);
+    memset (&this->v_frame, 0, sizeof (this->v_frame));
+  }
+
+  if (this->buffer_) {
+    if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_,
+          (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) {
+      g_assert_not_reached ();
+      return FALSE;
+    }
+    gst_gl_video_format_swizzle(GST_VIDEO_INFO_FORMAT (&this->v_info), this->uniforms.input_swizzle);
+
+    Matrix4 m;
+    float matrix_data[16] = { 0.0, };
+
+    matrix_set_identity (&m);
+    convert_to_RGB (&this->v_info, &m);
+    matrix_debug (&m);
+    matrix_to_float (&m, matrix_data);
+
+    this->uniforms.color_matrix = QMatrix4x4(matrix_data);
+    this->uniforms.dirty = true;
+  }
+
+  return TRUE;
+}
+
+/* only called from the streaming thread with scene graph thread blocked */
+GstBuffer *
+GstQSGMaterial::getBuffer (bool * was_bound)
+{
+  GstBuffer *buffer = NULL;
+
+  if (this->buffer_)
+    buffer = gst_buffer_ref (this->buffer_);
+  if (was_bound)
+    *was_bound = this->buffer_was_bound;
+
+  return buffer;
+}
+
+void
+GstQSGMaterial::setFiltering(QSGTexture::Filtering filtering)
+{
+  m_filtering = filtering;
+}
+
+static QRhiTexture::Format
+video_format_to_rhi_format (GstVideoFormat format, guint plane)
+{
+  switch (format) {
+    case GST_VIDEO_FORMAT_RGBA:
+    case GST_VIDEO_FORMAT_BGRA:
+      return QRhiTexture::RGBA8;
+    case GST_VIDEO_FORMAT_YV12:
+      return QRhiTexture::RED_OR_ALPHA8;
+    default:
+      g_assert_not_reached ();
+  }
+}
+
+QSGTexture *
+GstQSGMaterial::bind(GstQSGMaterialShader *shader, QRhi * rhi, QRhiResourceUpdateBatch *res_updates, guint plane, GstVideoFormat v_format)
+{
+  GstGLContext *qt_context, *context;
+  GstMemory *mem;
+  GstGLMemory *gl_mem;
+  GstGLSyncMeta *sync_meta;
+  gboolean use_dummy_tex = TRUE;
+  guint tex_id;
+  GstQSGTexture *ret;
+  QRhiTexture *rhi_tex;
+  QSize tex_size;
+
+  qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_));
+  if (!qt_context)
+    goto out;
+
+  if (!this->buffer_)
+    goto out;
+  if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN)
+    goto out;
+
+  mem = gst_buffer_peek_memory (this->buffer_, plane);
+  g_assert (gst_is_gl_memory (mem));
+  gl_mem = (GstGLMemory *) mem;
+  context = ((GstGLBaseMemory *)mem)->context;
+
+  /* Texture was successfully bound, so we do not need
+   * to use the dummy texture */
+  use_dummy_tex = FALSE;
+
+  this->buffer_was_bound = true;
+  tex_id = *(guint *) this->v_frame.data[plane];
+
+  tex_size = QSize(gst_gl_memory_get_texture_width(gl_mem), gst_gl_memory_get_texture_height (gl_mem));
+
+  rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), tex_size, 1, {});
+  rhi_tex->createFrom({(guint64) tex_id, 0});
+
+  sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_);
+  if (!sync_meta)
+    sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_);
+
+  gst_gl_sync_meta_set_sync_point (sync_meta, context);
+
+  gst_gl_sync_meta_wait (sync_meta, qt_context);
+
+  GST_LOG ("%p binding GL texture %u for plane %d", this, tex_id, plane);
+
+out:
+  if (G_UNLIKELY (use_dummy_tex)) {
+    /* Create dummy texture if not already present.
+     * Use the Qt RHI functions instead of the GstGL ones.
+     */
+
+    /* Make this a black 64x64 pixel RGBA texture.
+     * This size and format is supported pretty much everywhere, so these
+     * are a safe pick. (64 pixel sidelength must be supported according
+     * to the GLES2 spec, table 6.18.)
+     * Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */
+    const int tex_sidelength = 64;
+    std::vector < char > dummy_data (tex_sidelength * tex_sidelength * 4, 0);
+
+    rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), QSize(tex_sidelength, tex_sidelength), 1, {});
+
+    switch (v_format) {
+      case GST_VIDEO_FORMAT_RGBA:
+      case GST_VIDEO_FORMAT_BGRA:
+      case GST_VIDEO_FORMAT_RGB:
+        break;
+      case GST_VIDEO_FORMAT_YV12:
+        if (plane == 1 || plane == 2) {
+          char *data = dummy_data.data();
+          for (gsize j = 0; j < tex_sidelength; j++) {
+            for (gsize k = 0; k < tex_sidelength; k++) {
+              data[(j * tex_sidelength + k) * 4 + 0] = 0x7F;
+            }
+          }
+        }
+        break;
+      default:
+        g_assert_not_reached ();
+        break;
+    }
+
+    QRhiTextureSubresourceUploadDescription sub_desc;
+
+    sub_desc.setData(QByteArray::fromRawData(dummy_data.data(), dummy_data.size()));
+
+    QRhiTextureUploadEntry entry(0, 0, sub_desc);
+    QRhiTextureUploadDescription desc({ entry });
+    res_updates->uploadTexture(rhi_tex, desc);
+
+    GST_LOG ("%p binding for plane %d fallback dummy Qt texture", this, plane);
+  }
+
+  ret = new GstQSGTexture(rhi_tex);
+  ret->setFiltering(m_filtering);
+
+  gst_clear_object (&qt_context);
+
+  return static_cast<QSGTexture *>(ret);
+}

subprojects/gst-plugins-good/ext/qt6/gstqsg6material.h

@@ -0,0 +1,75 @@
+/*
+ * GStreamer
+ * Copyright (C) 2023 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_QSG6_MATERIAL_H__
+#define __GST_QSG6_MATERIAL_H__
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/gl/gl.h>
+
+#include "gstqt6gl.h"
+#include <QtQuick/QSGMaterial>
+#include <QtQuick/QSGMaterialShader>
+#include <QtGui/QOpenGLFunctions>
+#include <QtQuick/QSGTexture>
+
+class QRhi;
+class QRhiResourceUpdateBatch;
+class GstQSGMaterialShader;
+
+class GstQSGMaterial : public QSGMaterial
+{
+protected:
+    GstQSGMaterial();
+    ~GstQSGMaterial();
+public:
+    static GstQSGMaterial *new_for_format (GstVideoFormat format);
+
+    void setCaps (GstCaps * caps);
+    gboolean setBuffer (GstBuffer * buffer);
+    GstBuffer * getBuffer (bool * was_bound);
+    bool compatibleWith(GstVideoInfo *v_info);
+
+    void setFiltering(QSGTexture::Filtering);
+
+    QSGTexture * bind(GstQSGMaterialShader *, QRhi *, QRhiResourceUpdateBatch *, guint binding, GstVideoFormat);
+
+    /* QSGMaterial */
+    QSGMaterialShader *createShader(QSGRendererInterface::RenderMode renderMode) const override;
+
+    struct {
+        int input_swizzle[4];
+        QMatrix4x4 color_matrix;
+        bool dirty;
+    } uniforms;
+
+private:
+    GstBuffer * buffer_;
+    bool buffer_was_bound;
+    GWeakRef qt_context_ref_;
+    GstBuffer * sync_buffer_;
+    GstMemory * mem_;
+    GstVideoInfo v_info;
+    GstVideoFrame v_frame;
+    QSGTexture::Filtering m_filtering;
+};
+
+#endif /* __GST_QSG6_MATERIAL_H__ */

subprojects/gst-plugins-good/ext/qt6/meson.build

@@ -1,7 +1,7 @@
 sources = [
   'gstplugin.cc',
   'gstqt6element.cc',
-  'gstqsg6glnode.cc',
+  'gstqsg6material.cc',
   'gstqt6glutility.cc',
   'gstqml6glsink.cc',
   'gstqml6glsrc.cc',
@@ -14,11 +14,16 @@ sources = [
 
 moc_headers = [
   'qt6glitem.h',
-  'gstqsg6glnode.h',
   'qt6glwindow.h',
   'qt6glrenderer.h',
 ]
 
+shader_sources = [
+  'vertex.vert',
+  'RGBA.frag',
+  'YUV_TRIPLANAR.frag',
+]
+
 qt6qml_dep = dependency('', required: false)
 qt6_option = get_option('qt6')
 qt6_egl = get_option('qt-egl')
@@ -50,7 +55,7 @@ if not qt6_mod.has_tools(method: qt6_method)
 endif
 
 qt6qml_dep = dependency('qt6', modules : ['Core', 'Gui', 'Qml', 'Quick'],
-                        method: qt6_method, required: qt6_option, static: host_system == 'ios')
+                        method: qt6_method, required: qt6_option, static: host_system == 'ios', private_headers: true)
 if not qt6qml_dep.found()
   subdir_done()
 endif
@@ -153,7 +158,24 @@ endif
 if qt6_option.require(have_qt_windowing).allowed()
   # Build it!
   moc_files = qt6_mod.preprocess(moc_headers : moc_headers, method: qt6_method)
-  gstqml6gl = library('gstqml6', sources, moc_files,
+  # TODO: dist backup qsb shaders?
+  qsb = find_program('qsb-qt6', 'qsb')
+  shaders = []
+  foreach shader: shader_sources
+    qsb_shader = shader + '.qsb'
+    dist_shader = shader + '-dist.qsb'
+
+    compiled_shader = custom_target(qsb_shader,
+      input: shader,
+      output: qsb_shader,
+      command: [qsb, '--glsl=100 es,120,330', '--batchable', '--output', '@OUTPUT@', '@INPUT@']
+    )
+    shaders += [compiled_shader]
+  endforeach
+  resource_file = configure_file(input: 'resources.qrc', output: 'resources.qrc', copy: true)
+  qresources = qt6_mod.compile_resources(sources: resource_file, method: qt6_method)
+
+  gstqml6gl = library('gstqml6', sources, moc_files, qresources,
     cpp_args : gst_plugins_good_args + qt_defines,
     link_args : noseh_link_args,
     include_directories: [configinc, libsinc],

subprojects/gst-plugins-good/ext/qt6/qt6glitem.cc

@@ -26,14 +26,13 @@
 
 #include <gst/video/video.h>
 #include "qt6glitem.h"
-#include "gstqsg6glnode.h"
 #include "gstqt6glutility.h"
+#include "gstqsg6material.h"
 
 #include <QtCore/QMutexLocker>
 #include <QtCore/QPointer>
 #include <QtGui/QGuiApplication>
 #include <QtQuick/QQuickWindow>
-#include <QtQuick/QSGSimpleTextureNode>
 
 /**
  * SECTION:Qt6GLVideoItem
@@ -88,8 +87,6 @@ struct _Qt6GLVideoItemPrivate
    * FIXME: Ideally we would use fences for this but there seems to be no
    * way to reliably "try wait" on a fence */
   GQueue potentially_unbound_buffers;
-
-  GstQSG6OpenGLNode *m_node;
 };
 
 Qt6GLVideoItem::Qt6GLVideoItem()
@@ -276,32 +273,55 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode,
     UpdatePaintNodeData * updatePaintNodeData)
 {
   GstBuffer *old_buffer;
+  GstQSGMaterial *tex = nullptr;
+  QSGGeometry *geometry = nullptr;
+  bool was_bound = false;
 
   if (!this->priv->initted)
     return oldNode;
 
-  GstQSG6OpenGLNode *texNode = static_cast<GstQSG6OpenGLNode *> (oldNode);
+  QSGGeometryNode *texNode = static_cast<QSGGeometryNode *> (oldNode);
   GstVideoRectangle src, dst, result;
 
   g_mutex_lock (&this->priv->lock);
 
   GST_TRACE ("%p updatePaintNode", this);
 
+  if (!this->priv->caps) {
+    GST_LOG ("%p no caps yet", this);
+    g_mutex_unlock (&this->priv->lock);
+    return NULL;
+  }
+
   if (gst_gl_context_get_current() == NULL)
     gst_gl_context_activate (this->priv->other_context, TRUE);
 
-  if (!texNode) {
+  if (texNode) {
-    bool is_smooth = this->smooth ();
+    tex = static_cast<GstQSGMaterial *>(texNode->material());
-    texNode = new GstQSG6OpenGLNode (this);
+    if (tex && !tex->compatibleWith(&this->priv->v_info)) {
-    texNode->setFiltering (is_smooth ? QSGTexture::Filtering::Linear :
+      delete texNode;
-        QSGTexture::Filtering::Nearest);
+      texNode = nullptr;
-    this->priv->m_node = texNode;
+    }
   }
 
-  if ((old_buffer = texNode->getBuffer())) {
+  if (!texNode) {
+    bool is_smooth = this->smooth ();
+    texNode = new QSGGeometryNode();
+    geometry = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
+    texNode->setGeometry(geometry);
+    tex = GstQSGMaterial::new_for_format(GST_VIDEO_INFO_FORMAT (&this->priv->v_info));
+    tex->setFiltering(is_smooth ? QSGTexture::Filtering::Linear :
+        QSGTexture::Filtering::Nearest);
+    texNode->setMaterial(tex);
+  }
+
+  if ((old_buffer = tex->getBuffer(&was_bound))) {
     if (old_buffer == this->priv->buffer) {
       /* same buffer */
       gst_buffer_unref (old_buffer);
+    } else if (!was_bound) {
+      GST_TRACE ("old buffer %p was not bound yet, unreffing", old_buffer);
+      gst_buffer_unref (old_buffer);
     } else {
       GstBuffer *tmp_buffer;
 
@@ -326,8 +346,8 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode,
     old_buffer = NULL;
   }
 
-  texNode->setCaps (this->priv->caps);
+  tex->setCaps (this->priv->caps);
-  texNode->setBuffer (this->priv->buffer);
+  tex->setBuffer (this->priv->buffer);
 
   if (this->priv->force_aspect_ratio && this->priv->caps) {
     src.w = this->priv->display_width;
@@ -346,7 +366,10 @@ Qt6GLVideoItem::updatePaintNode(QSGNode * oldNode,
     result.h = boundingRect().height();
   }
 
-  texNode->setRect (QRectF (result.x, result.y, result.w, result.h));
+  geometry = texNode->geometry();
+  QRectF rect(result.x, result.y, result.w, result.h);
+  QRectF sourceRect(0, 0, 1, 1);
+  QSGGeometry::updateTexturedRectGeometry(geometry, rect, sourceRect);
 
   g_mutex_unlock (&this->priv->lock);
 
@@ -713,7 +736,6 @@ Qt6GLVideoItem::onSceneGraphInitialized ()
 void
 Qt6GLVideoItem::onSceneGraphInvalidated ()
 {
-  this->priv->m_node = nullptr;
   GST_FIXME ("%p scene graph invalidated", this);
 }
 
@@ -790,13 +812,11 @@ Qt6GLVideoItem::handleWindowChanged (QQuickWindow * win)
     this->priv->qt_context = NULL;
     this->priv->initted = FALSE;
   }
-  this->priv->m_node = nullptr;
 }
 
 void
 Qt6GLVideoItem::releaseResources()
 {
-  this->priv->m_node = nullptr;
 }
 
 gboolean

subprojects/gst-plugins-good/ext/qt6/resources.qrc

@@ -0,0 +1,7 @@
+<RCC>
+    <qresource prefix="/org/freedesktop/gstreamer/qml6">
+        <file>vertex.vert.qsb</file>
+        <file>RGBA.frag.qsb</file>
+        <file>YUV_TRIPLANAR.frag.qsb</file>
+    </qresource>
+</RCC>

subprojects/gst-plugins-good/ext/qt6/vertex.vert

@@ -0,0 +1,23 @@
+#version 440
+
+layout(location = 0) in vec4 aVertex;
+layout(location = 1) in vec2 aTexCoord;
+
+layout(location = 0) out vec2 vTexCoord;
+
+layout(std140, binding = 0) uniform buf {
+  mat4 qt_Matrix;
+  ivec4 swizzle;
+  mat4 color_matrix;
+  float qt_Opacity;
+} ubuf;
+
+out gl_PerVertex {
+  vec4 gl_Position;
+};
+
+void main()
+{
+  gl_Position = ubuf.qt_Matrix * aVertex;
+  vTexCoord = aTexCoord;
+}

subprojects/gst-plugins-good/tests/examples/qt6/qmlsink/main.cpp

@@ -48,6 +48,10 @@ int main(int argc, char *argv[])
 
     GstElement *pipeline = gst_pipeline_new (NULL);
     GstElement *src = gst_element_factory_make ("videotestsrc", NULL);
+    GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL);
+    GstCaps *caps = gst_caps_from_string ("video/x-raw,format=YV12");
+    g_object_set (capsfilter, "caps", caps, NULL);
+    gst_clear_caps (&caps);
     GstElement *glupload = gst_element_factory_make ("glupload", NULL);
     /* the plugin must be loaded before loading the qml file to register the
      * GstGLVideoItem qml item */
@@ -55,8 +59,8 @@ int main(int argc, char *argv[])
 
     g_assert (src && glupload && sink);
 
-    gst_bin_add_many (GST_BIN (pipeline), src, glupload, sink, NULL);
+    gst_bin_add_many (GST_BIN (pipeline), src, capsfilter, glupload, sink, NULL);
-    gst_element_link_many (src, glupload, sink, NULL);
+    gst_element_link_many (src, capsfilter, glupload, sink, NULL);
 
     QQmlApplicationEngine engine;
     engine.load(QUrl(QStringLiteral("qrc:/main.qml")));