/* * GStreamer * Copyright (C) 2023 Matthew Waters * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include #include #include #include #include #include "gstqsg6material.h" #include #define GST_CAT_DEFAULT gst_qsg_texture_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); /* matrix colour conversion code from vkvideoconvert.c */ typedef struct { double dm[4][4]; } Matrix4; static void matrix_debug (const Matrix4 * s) { GST_DEBUG ("[%f %f %f %f]", s->dm[0][0], s->dm[0][1], s->dm[0][2], s->dm[0][3]); GST_DEBUG ("[%f %f %f %f]", s->dm[1][0], s->dm[1][1], s->dm[1][2], s->dm[1][3]); GST_DEBUG ("[%f %f %f %f]", s->dm[2][0], s->dm[2][1], s->dm[2][2], s->dm[2][3]); GST_DEBUG ("[%f %f %f %f]", s->dm[3][0], s->dm[3][1], s->dm[3][2], s->dm[3][3]); } static void matrix_to_float (const Matrix4 * m, float *ret) { int i, j; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) { ret[j * 4 + i] = m->dm[i][j]; } } } static void matrix_set_identity (Matrix4 * m) { int i, j; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) { m->dm[i][j] = (i == j); } } } static void matrix_copy (Matrix4 * d, const Matrix4 * s) { gint i, j; for (i = 0; i < 4; i++) for (j = 0; j < 4; j++) d->dm[i][j] = s->dm[i][j]; } /* Perform 4x4 matrix multiplication: * - @dst@ = @a@ * @b@ * - @dst@ may be a pointer to @a@ andor @b@ */ static void matrix_multiply (Matrix4 * dst, Matrix4 * a, Matrix4 * b) { Matrix4 tmp; int i, j, k; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) { double x = 0; for (k = 0; k < 4; k++) { x += a->dm[i][k] * b->dm[k][j]; } tmp.dm[i][j] = x; } } matrix_copy (dst, &tmp); } static void matrix_offset_components (Matrix4 * m, double a1, double a2, double a3) { Matrix4 a; matrix_set_identity (&a); a.dm[0][3] = a1; a.dm[1][3] = a2; a.dm[2][3] = a3; matrix_debug (&a); matrix_multiply (m, &a, m); } static void matrix_scale_components (Matrix4 * m, double a1, double a2, double a3) { Matrix4 a; matrix_set_identity (&a); a.dm[0][0] = a1; a.dm[1][1] = a2; a.dm[2][2] = a3; matrix_multiply (m, &a, m); } static void matrix_YCbCr_to_RGB (Matrix4 * m, double Kr, double Kb) { double Kg = 1.0 - Kr - Kb; Matrix4 k = { { {1., 0., 2 * (1 - Kr), 0.}, {1., -2 * Kb * (1 - Kb) / Kg, -2 * Kr * (1 - Kr) / Kg, 0.}, {1., 2 * (1 - Kb), 0., 0.}, {0., 0., 0., 1.}, } }; matrix_multiply (m, &k, m); } static void convert_to_RGB (GstVideoInfo *info, Matrix4 * m) { { const GstVideoFormatInfo *uinfo; gint offset[4], scale[4], depth[4]; guint i; uinfo = gst_video_format_get_info (GST_VIDEO_INFO_FORMAT (info)); /* bring color components to [0..1.0] range */ gst_video_color_range_offsets (info->colorimetry.range, uinfo, offset, scale); for (i = 0; i < uinfo->n_components; i++) depth[i] = (1 << uinfo->depth[i]) - 1; matrix_offset_components (m, -offset[0] / (float) depth[0], -offset[1] / (float) depth[1], -offset[2] / (float) depth[2]); matrix_scale_components (m, depth[0] / ((float) scale[0]), depth[1] / ((float) scale[1]), depth[2] / ((float) scale[2])); GST_DEBUG ("to RGB scale/offset matrix"); matrix_debug (m); } if (GST_VIDEO_INFO_IS_YUV (info)) { gdouble Kr, Kb; if (gst_video_color_matrix_get_Kr_Kb (info->colorimetry.matrix, &Kr, &Kb)) matrix_YCbCr_to_RGB (m, Kr, Kb); GST_DEBUG ("to RGB matrix"); matrix_debug (m); } } class GstQSGTexture : public QSGTexture { public: GstQSGTexture(QRhiTexture *); ~GstQSGTexture(); qint64 comparisonKey() const override; bool hasAlphaChannel() const override; bool hasMipmaps() const override { return false; }; bool isAtlasTexture() const override { return false; }; QSize textureSize() const override; QRhiTexture *rhiTexture() const override; private: QRhiTexture *m_texture; bool m_has_alpha; }; GstQSGTexture::GstQSGTexture(QRhiTexture * texture) : m_texture(texture) { switch (texture->format()) { case QRhiTexture::RGBA8: #if (QT_VERSION >= QT_VERSION_CHECK(6, 4, 0)) case QRhiTexture::RGB10A2: #endif case QRhiTexture::RGBA16F: case QRhiTexture::RGBA32F: this->m_has_alpha = true; break; default: this->m_has_alpha = false; } } GstQSGTexture::~GstQSGTexture() { if (m_texture) { m_texture->deleteLater(); m_texture = nullptr; } } qint64 GstQSGTexture::comparisonKey() const { if (this->m_texture) return qint64(qintptr(this->m_texture)); return qint64(qintptr(this)); } bool GstQSGTexture::hasAlphaChannel() const { return m_has_alpha; } QSize GstQSGTexture::textureSize() const { // XXX: currently unused return QSize(0, 0); } QRhiTexture * GstQSGTexture::rhiTexture() const { return m_texture; } class GstQSG6MaterialShader : public QSGMaterialShader { public: GstQSG6MaterialShader(GstVideoFormat v_format, GstGLTextureTarget target); ~GstQSG6MaterialShader(); bool updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override; void updateSampledImage(RenderState &state, int binding, QSGTexture **texture, QSGMaterial *newMaterial, QSGMaterial *) override; private: GstVideoFormat v_format; QSGTexture *m_textures[GST_VIDEO_MAX_PLANES]; }; GstQSG6MaterialShader::GstQSG6MaterialShader(GstVideoFormat v_format, GstGLTextureTarget target) : v_format(v_format) { const gchar *frag_shader; setShaderFileName(VertexStage, ":/org/freedesktop/gstreamer/qml6/vertex.vert.qsb"); switch (v_format) { case GST_VIDEO_FORMAT_RGBA: case GST_VIDEO_FORMAT_BGRA: case GST_VIDEO_FORMAT_RGB: if (target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) frag_shader = ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb.external"; else frag_shader = ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb"; break; case GST_VIDEO_FORMAT_YV12: frag_shader = ":/org/freedesktop/gstreamer/qml6/YUV_TRIPLANAR.frag.qsb"; break; case GST_VIDEO_FORMAT_NV12: frag_shader = ":/org/freedesktop/gstreamer/qml6/YUV_BIPLANAR.frag.qsb"; break; default: g_assert_not_reached (); } GST_DEBUG("load fragment shader: %s", frag_shader); setShaderFileName(FragmentStage, frag_shader); m_textures[0] = nullptr; m_textures[1] = nullptr; m_textures[2] = nullptr; m_textures[3] = nullptr; } GstQSG6MaterialShader::~GstQSG6MaterialShader() { for (int i = 0; i < 4; i++) { if (m_textures[i]) { delete m_textures[i]; m_textures[i] = nullptr; } } } bool GstQSG6MaterialShader::updateUniformData(RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) { const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format); bool changed = false; QByteArray *buf = state.uniformData(); Q_ASSERT(buf->size() >= 84); GST_TRACE ("%p new material %p old material %p", this, newMaterial, oldMaterial); if (state.isMatrixDirty()) { const QMatrix4x4 m = state.combinedMatrix(); memcpy(buf->data(), m.constData(), 64); changed = true; } if (state.isOpacityDirty()) { const float opacity = state.opacity(); memcpy(buf->data() + 144, &opacity, 4); changed = true; } auto *mat = static_cast(newMaterial); if (oldMaterial != newMaterial || mat->uniforms.dirty) { memcpy(buf->data() + 64, &mat->uniforms.input_swizzle, 4 * sizeof (int)); memcpy(buf->data() + 80, mat->uniforms.color_matrix.constData(), 64); mat->uniforms.dirty = false; changed = true; } for (guint i = 0; i < GST_VIDEO_MAX_PLANES; i++) { if (this->m_textures[i]) { delete this->m_textures[i]; this->m_textures[i] = nullptr; } if (i < finfo->n_planes) this->m_textures[i] = mat->bind(this, state.rhi(), state.resourceUpdateBatch(), i, v_format); } return changed; } void GstQSG6MaterialShader::updateSampledImage(RenderState &state, int binding, QSGTexture **texture, QSGMaterial *newMaterial, QSGMaterial *) { *texture = this->m_textures[binding - 1]; GST_TRACE ("%p binding:%d texture %p", this, binding, *texture); } #define DEFINE_MATERIAL(format) \ class G_PASTE(GstQSG6Material_,format) : public GstQSG6Material { \ public: \ G_PASTE(GstQSG6Material_,format)(); \ ~G_PASTE(GstQSG6Material_,format)(); \ QSGMaterialType *type() const override { static QSGMaterialType type; return &type; }; \ }; \ G_PASTE(GstQSG6Material_,format)::G_PASTE(GstQSG6Material_,format)() {} \ G_PASTE(GstQSG6Material_,format)::~G_PASTE(GstQSG6Material_,format)() {} DEFINE_MATERIAL(RGBA_SWIZZLE); DEFINE_MATERIAL(YUV_TRIPLANAR); DEFINE_MATERIAL(YUV_BIPLANAR); GstQSG6Material * GstQSG6Material::new_for_format(GstVideoFormat format) { const GstVideoFormatInfo *finfo = gst_video_format_get_info (format); if (GST_VIDEO_FORMAT_INFO_IS_RGB (finfo) && finfo->n_planes == 1) { return static_cast(new GstQSG6Material_RGBA_SWIZZLE()); } switch (format) { case GST_VIDEO_FORMAT_YV12: return static_cast(new GstQSG6Material_YUV_TRIPLANAR()); case GST_VIDEO_FORMAT_NV12: return static_cast(new GstQSG6Material_YUV_BIPLANAR()); default: g_assert_not_reached (); } } GstQSG6Material::GstQSG6Material () { static gsize _debug; if (g_once_init_enter (&_debug)) { GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsg6material", 0, "Qt6 Scenegraph Material"); g_once_init_leave (&_debug, 1); } g_weak_ref_init (&this->qt_context_ref_, NULL); gst_video_info_init (&this->v_info); memset (&this->v_frame, 0, sizeof (this->v_frame)); this->buffer_ = NULL; this->buffer_was_bound = false; this->sync_buffer_ = gst_buffer_new (); this->uniforms.dirty = true; } GstQSG6Material::~GstQSG6Material () { g_weak_ref_clear (&this->qt_context_ref_); gst_buffer_replace (&this->buffer_, NULL); gst_buffer_replace (&this->sync_buffer_, NULL); this->buffer_was_bound = false; if (this->v_frame.buffer) { gst_video_frame_unmap (&this->v_frame); memset (&this->v_frame, 0, sizeof (this->v_frame)); } } bool GstQSG6Material::compatibleWith(GstVideoInfo * v_info) { if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info)) return false; return true; } QSGMaterialShader * GstQSG6Material::createShader(QSGRendererInterface::RenderMode renderMode) const { GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info); GstGLTextureTarget target = this->tex_target; return new GstQSG6MaterialShader(v_format, target); } /* only called from the streaming thread with scene graph thread blocked */ void GstQSG6Material::setCaps (GstCaps * caps) { GstStructure *s; const gchar *target_str; GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps); gst_video_info_from_caps (&this->v_info, caps); s = gst_caps_get_structure (caps, 0); target_str = gst_structure_get_string (s, "texture-target"); if (!target_str) target_str = GST_GL_TEXTURE_TARGET_2D_STR; this->tex_target = gst_gl_texture_target_from_string(target_str); } /* only called from the streaming thread with scene graph thread blocked */ gboolean GstQSG6Material::setBuffer (GstBuffer * buffer) { GstGLContext *qt_context = gst_gl_context_get_current (); GST_LOG ("%p setBuffer %" GST_PTR_FORMAT " with qt context %" GST_PTR_FORMAT, this, buffer, qt_context); /* FIXME: update more state here */ g_weak_ref_set (&this->qt_context_ref_, qt_context); if (!gst_buffer_replace (&this->buffer_, buffer)) return FALSE; this->buffer_was_bound = false; if (this->v_frame.buffer) { gst_video_frame_unmap (&this->v_frame); memset (&this->v_frame, 0, sizeof (this->v_frame)); } if (this->buffer_) { if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_, (GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) { g_assert_not_reached (); return FALSE; } gst_gl_video_format_swizzle(GST_VIDEO_INFO_FORMAT (&this->v_info), this->uniforms.input_swizzle); Matrix4 m; float matrix_data[16] = { 0.0, }; matrix_set_identity (&m); convert_to_RGB (&this->v_info, &m); matrix_debug (&m); matrix_to_float (&m, matrix_data); this->uniforms.color_matrix = QMatrix4x4(matrix_data); this->uniforms.dirty = true; } return TRUE; } /* only called from the streaming thread with scene graph thread blocked */ GstBuffer * GstQSG6Material::getBuffer (bool * was_bound) { GstBuffer *buffer = NULL; if (this->buffer_) buffer = gst_buffer_ref (this->buffer_); if (was_bound) *was_bound = this->buffer_was_bound; return buffer; } void GstQSG6Material::setFiltering(QSGTexture::Filtering filtering) { m_filtering = filtering; } static QRhiTexture::Format video_format_to_rhi_format (GstVideoFormat format, guint plane) { switch (format) { case GST_VIDEO_FORMAT_RGBA: case GST_VIDEO_FORMAT_BGRA: case GST_VIDEO_FORMAT_RGB: return QRhiTexture::RGBA8; case GST_VIDEO_FORMAT_YV12: return QRhiTexture::R8; case GST_VIDEO_FORMAT_NV12: return (plane == 0 ? QRhiTexture::R8 : QRhiTexture::RG8); default: g_assert_not_reached (); } } static int video_format_to_texel_size (GstVideoFormat format, guint plane) { switch (format) { case GST_VIDEO_FORMAT_RGBA: case GST_VIDEO_FORMAT_BGRA: case GST_VIDEO_FORMAT_RGB: return 4; case GST_VIDEO_FORMAT_YV12: return 1; case GST_VIDEO_FORMAT_NV12: return (plane == 0 ? 1 : 2); default: g_assert_not_reached (); } } QSGTexture * GstQSG6Material::bind(GstQSG6MaterialShader *shader, QRhi * rhi, QRhiResourceUpdateBatch *res_updates, guint plane, GstVideoFormat v_format) { GstGLContext *qt_context = NULL, *context; GstMemory *mem; GstGLMemory *gl_mem; GstGLSyncMeta *sync_meta; gboolean use_dummy_tex = TRUE; guint tex_id; GstQSGTexture *ret; QRhiTexture *rhi_tex; QSize tex_size; QRhiTexture::Flags flags = {}; if (!this->buffer_) goto out; if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN) goto out; qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_)); if (!qt_context) goto out; GST_DEBUG ("%p attempting to bind with context %" GST_PTR_FORMAT, this, qt_context); mem = gst_buffer_peek_memory (this->buffer_, plane); g_assert (gst_is_gl_memory (mem)); gl_mem = (GstGLMemory *) mem; context = ((GstGLBaseMemory *)mem)->context; /* Texture was successfully bound, so we do not need * to use the dummy texture */ use_dummy_tex = FALSE; this->buffer_was_bound = true; tex_id = *(guint *) this->v_frame.data[plane]; tex_size = QSize(gst_gl_memory_get_texture_width(gl_mem), gst_gl_memory_get_texture_height (gl_mem)); if (gl_mem->tex_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) flags |= QRhiTexture::ExternalOES; rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), tex_size, 1, flags); rhi_tex->createFrom({(guint64) tex_id, 0}); sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_); if (!sync_meta) sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_); gst_gl_sync_meta_set_sync_point (sync_meta, context); gst_gl_sync_meta_wait (sync_meta, qt_context); GST_LOG ("%p binding GL texture %u (%s) for plane %d", this, tex_id, gst_gl_texture_target_to_string(gl_mem->tex_target), plane); out: if (G_UNLIKELY (use_dummy_tex)) { /* Create dummy texture if not already present. * Use the Qt RHI functions instead of the GstGL ones. */ /* Make this a black 64x64 pixel RGBA texture. * This size and format is supported pretty much everywhere, so these * are a safe pick. (64 pixel sidelength must be supported according * to the GLES2 spec, table 6.18.) */ const int tex_sidelength = 64; rhi_tex = rhi->newTexture (video_format_to_rhi_format (v_format, plane), QSize(tex_sidelength, tex_sidelength), 1, {}); g_assert (rhi_tex->create()); int ts = video_format_to_texel_size (v_format, plane); QByteArray dummy_data (tex_sidelength * tex_sidelength * ts, 0); char *data = dummy_data.data(); switch (v_format) { case GST_VIDEO_FORMAT_RGBA: case GST_VIDEO_FORMAT_BGRA: case GST_VIDEO_FORMAT_RGB: for (gsize j = 0; j < tex_sidelength; j++) { for (gsize k = 0; k < tex_sidelength; k++) { data[(j * tex_sidelength + k) * ts + 3] = 0xFF; // opaque } } break; case GST_VIDEO_FORMAT_YV12: if (plane == 1 || plane == 2) { for (gsize j = 0; j < tex_sidelength; j++) { for (gsize k = 0; k < tex_sidelength; k++) { data[(j * tex_sidelength + k) * ts + 0] = 0x7F; } } } break; case GST_VIDEO_FORMAT_NV12: if (plane == 1) { for (gsize j = 0; j < tex_sidelength; j++) { for (gsize k = 0; k < tex_sidelength; k++) { data[(j * tex_sidelength + k) * ts + 0] = 0x7F; data[(j * tex_sidelength + k) * ts + 1] = 0x7F; } } } break; default: g_assert_not_reached (); break; } QRhiTextureSubresourceUploadDescription sub_desc(dummy_data); QRhiTextureUploadEntry entry(0, 0, sub_desc); QRhiTextureUploadDescription desc(entry); res_updates->uploadTexture(rhi_tex, desc); GST_LOG ("%p binding for plane %d fallback dummy Qt texture", this, plane); } ret = new GstQSGTexture(rhi_tex); ret->setFiltering(m_filtering); gst_clear_object (&qt_context); return static_cast(ret); }