From 3c7f9c0fab16263514035b841472438189bfc58f Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Mon, 19 Aug 2024 21:05:43 +1000 Subject: [PATCH] qmlglsink: Add support for external-oes textures Support was added to qml6glsink in MR !7319 Backport similar support to the Qt5 element so it can also support direct DMABuf import from hardware decoders. Part-of: --- .../docs/gst_plugins_cache.json | 2 +- .../gst-plugins-good/ext/qt/gstqsgmaterial.cc | 52 +++++++++++++++---- .../gst-plugins-good/ext/qt/gstqsgmaterial.h | 5 +- .../gst-plugins-good/ext/qt/gstqtsink.cc | 3 +- subprojects/gst-plugins-good/ext/qt/qtitem.cc | 16 +++++- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json index 1bf91722a1..33e7368647 100644 --- a/subprojects/gst-plugins-good/docs/gst_plugins_cache.json +++ b/subprojects/gst-plugins-good/docs/gst_plugins_cache.json @@ -12496,7 +12496,7 @@ "long-name": "Qt Video Sink", "pad-templates": { "sink": { - "caps": "video/x-raw(memory:GLMemory):\n format: { RGB, RGBA, BGRA, YV12, NV12 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n", + "caps": "video/x-raw(memory:GLMemory):\n format: { RGB, RGBA, BGRA, YV12, NV12 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: { (string)2D, (string)external-oes }\n", "direction": "sink", "presence": "always" } diff --git a/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.cc b/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.cc index 59b4eb9b35..67e4dc545a 100644 --- a/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.cc +++ b/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.cc @@ -190,16 +190,20 @@ G_PASTE(GstQSGMaterial_,format)::G_PASTE(GstQSGMaterial_,format)() {} \ G_PASTE(GstQSGMaterial_,format)::~G_PASTE(GstQSGMaterial_,format)() {} DEFINE_MATERIAL(RGBA); +DEFINE_MATERIAL(RGBA_external); DEFINE_MATERIAL(RGBA_SWIZZLE); DEFINE_MATERIAL(YUV_TRIPLANAR); DEFINE_MATERIAL(YUV_BIPLANAR); GstQSGMaterial * -GstQSGMaterial::new_for_format(GstVideoFormat format) +GstQSGMaterial::new_for_format_and_target(GstVideoFormat format, GstGLTextureTarget target) { switch (format) { case GST_VIDEO_FORMAT_RGB: case GST_VIDEO_FORMAT_RGBA: + if (target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) + return static_cast(new GstQSGMaterial_RGBA_external()); + return static_cast(new GstQSGMaterial_RGBA()); case GST_VIDEO_FORMAT_BGRA: return static_cast(new GstQSGMaterial_RGBA_SWIZZLE()); @@ -246,10 +250,12 @@ GstQSGMaterial::~GstQSGMaterial () } bool -GstQSGMaterial::compatibleWith(GstVideoInfo * v_info) +GstQSGMaterial::compatibleWith(GstVideoInfo *v_info, GstGLTextureTarget tex_target) { if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info)) return FALSE; + if (this->tex_target != tex_target) + return FALSE; return TRUE; } @@ -266,11 +272,15 @@ vertexShaderForFormat(GstVideoFormat v_format) #define gles2_precision \ "precision mediump float;\n" +#define external_fragment_header \ + "#extension GL_OES_EGL_image_external : require\n" #define texcoord_input \ "varying vec2 v_texcoord;\n" #define single_texture_input \ "uniform sampler2D " UNIFORM_TEXTURE0_NAME ";\n" +#define single_texture_input_external \ + "uniform samplerExternalOES tex;\n" #define triplanar_texture_input \ "uniform sampler2D " UNIFORM_TRIPLANAR_PLANE0 ";\n" \ "uniform sampler2D " UNIFORM_TRIPLANAR_PLANE1 ";\n" \ @@ -290,7 +300,7 @@ vertexShaderForFormat(GstVideoFormat v_format) "uniform vec3 " UNIFORM_YUV_VCOEFF_NAME ";\n" static char * -fragmentShaderForFormat(GstVideoFormat v_format, GstGLContext * context) +fragmentShaderForFormatAndTarget(GstVideoFormat v_format, GstGLTextureTarget tex_target, GstGLContext * context) { gboolean is_gles2 = (gst_gl_context_get_gl_api (context) & GST_GL_API_GLES2) != 0; @@ -298,11 +308,21 @@ fragmentShaderForFormat(GstVideoFormat v_format, GstGLContext * context) case GST_VIDEO_FORMAT_RGB: case GST_VIDEO_FORMAT_RGBA: { char *swizzle = gst_gl_color_convert_swizzle_shader_string (context); - char *ret = g_strdup_printf ("%s" texcoord_input single_texture_input uniform_opacity - "%s\n" - "void main(void) {\n" - " gl_FragColor = texture2D(tex, v_texcoord) * " UNIFORM_OPACITY_NAME ";\n" - "}\n", is_gles2 ? gles2_precision : "", swizzle); + char *ret = NULL; + + if (tex_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) { + ret = g_strdup_printf (external_fragment_header "%s" texcoord_input single_texture_input_external uniform_opacity + "void main(void) {\n" + " gl_FragColor = texture2D(tex, v_texcoord) * " UNIFORM_OPACITY_NAME ";\n" + "}\n", is_gles2 ? gles2_precision : ""); + } else { + ret = g_strdup_printf ("%s" texcoord_input single_texture_input uniform_opacity + "%s\n" + "void main(void) {\n" + " gl_FragColor = texture2D(tex, v_texcoord) * " UNIFORM_OPACITY_NAME ";\n" + "}\n", is_gles2 ? gles2_precision : "", swizzle); + } + g_clear_pointer (&swizzle, g_free); return ret; } @@ -368,8 +388,10 @@ QSGMaterialShader * GstQSGMaterial::createShader() const { GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info); + GstGLTextureTarget tex_target = this->tex_target; + char *vertex = vertexShaderForFormat(v_format); - char *fragment = fragmentShaderForFormat(v_format, gst_gl_context_get_current ()); + char *fragment = fragmentShaderForFormatAndTarget(v_format, tex_target, gst_gl_context_get_current ()); if (!vertex || !fragment) return nullptr; @@ -410,6 +432,13 @@ GstQSGMaterial::setCaps (GstCaps * caps) GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps); gst_video_info_from_caps (&this->v_info, caps); + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *target_str = gst_structure_get_string (s, "texture-target"); + if (target_str) { + this->tex_target = gst_gl_texture_target_from_string(target_str); + } else { + this->tex_target = GST_GL_TEXTURE_TARGET_2D; + } } /* only called from the streaming thread with scene graph thread blocked */ @@ -449,6 +478,7 @@ GstQSGMaterial::bind(GstQSGMaterialShader *shader, GstVideoFormat v_format) GstGLContext *context, *qt_context; GstGLSyncMeta *sync_meta; GstMemory *mem; + GstGLMemory *gl_mem; gboolean use_dummy_tex = TRUE; if (this->v_frame.buffer) { @@ -480,6 +510,7 @@ GstQSGMaterial::bind(GstQSGMaterialShader *shader, GstVideoFormat v_format) mem = gst_buffer_peek_memory (this->buffer_, 0); g_assert (gst_is_gl_memory (mem)); + gl_mem = (GstGLMemory *)mem; context = ((GstGLBaseMemory *)mem)->context; @@ -504,8 +535,7 @@ GstQSGMaterial::bind(GstQSGMaterialShader *shader, GstVideoFormat v_format) shader->program()->setUniformValue(shader->tex_uniforms[i], i); gl->ActiveTexture (GL_TEXTURE0 + i); GST_LOG ("%p binding for plane %d Qt texture %u", this, i, tex_id); - - gl->BindTexture (GL_TEXTURE_2D, tex_id); + gl->BindTexture (gst_gl_texture_target_to_gl (gl_mem->tex_target), tex_id); } /* Texture was successfully bound, so we do not need diff --git a/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.h b/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.h index b9fa88ae83..09309fb477 100644 --- a/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.h +++ b/subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.h @@ -39,12 +39,12 @@ protected: GstQSGMaterial(); ~GstQSGMaterial(); public: - static GstQSGMaterial *new_for_format (GstVideoFormat format); + static GstQSGMaterial *new_for_format_and_target(GstVideoFormat format, GstGLTextureTarget target); void setCaps (GstCaps * caps); gboolean setBuffer (GstBuffer * buffer); GstBuffer * getBuffer (gboolean * was_bound); - bool compatibleWith(GstVideoInfo *v_info); + bool compatibleWith(GstVideoInfo *v_info, GstGLTextureTarget tex_target); void bind(GstQSGMaterialShader *, GstVideoFormat); @@ -61,6 +61,7 @@ private: GWeakRef qt_context_ref_; GstMemory * mem_; GstVideoInfo v_info; + GstGLTextureTarget tex_target; GstVideoFrame v_frame; float *cms_offset; float *cms_ycoeff; diff --git a/subprojects/gst-plugins-good/ext/qt/gstqtsink.cc b/subprojects/gst-plugins-good/ext/qt/gstqtsink.cc index e7bec02c2f..c0e0cd2573 100644 --- a/subprojects/gst-plugins-good/ext/qt/gstqtsink.cc +++ b/subprojects/gst-plugins-good/ext/qt/gstqtsink.cc @@ -112,7 +112,8 @@ GST_STATIC_PAD_TEMPLATE ("sink", "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " - "texture-target = (string) 2D")); + "texture-target = (string) { " GST_GL_TEXTURE_TARGET_2D_STR ", " + GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR " } ")); #define DEFAULT_FORCE_ASPECT_RATIO TRUE #define DEFAULT_PAR_N 0 diff --git a/subprojects/gst-plugins-good/ext/qt/qtitem.cc b/subprojects/gst-plugins-good/ext/qt/qtitem.cc index 494bb70f02..ecf0f7726d 100644 --- a/subprojects/gst-plugins-good/ext/qt/qtitem.cc +++ b/subprojects/gst-plugins-good/ext/qt/qtitem.cc @@ -73,7 +73,10 @@ struct _QtGLVideoItemPrivate GstCaps *new_caps; GstCaps *caps; GstVideoInfo new_v_info; + GstGLTextureTarget new_tex_target; + GstVideoInfo v_info; + GstGLTextureTarget tex_target; GstVideoRectangle v_rect; gboolean initted; @@ -305,7 +308,7 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode, if (texNode) { geometry = texNode->geometry(); tex = static_cast(texNode->material()); - if (tex && !tex->compatibleWith(&this->priv->v_info)) { + if (tex && !tex->compatibleWith(&this->priv->v_info, this->priv->tex_target)) { delete texNode; texNode = nullptr; } @@ -316,7 +319,7 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode, geometry = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4); texNode->setGeometry(geometry); texNode->setFlag(QSGGeometryNode::Flag::OwnsGeometry); - tex = GstQSGMaterial::new_for_format(GST_VIDEO_INFO_FORMAT (&this->priv->v_info)); + tex = GstQSGMaterial::new_for_format_and_target (GST_VIDEO_INFO_FORMAT (&this->priv->v_info), this->priv->tex_target); texNode->setMaterial(tex); texNode->setFlag(QSGGeometryNode::Flag::OwnsMaterial); } @@ -689,6 +692,7 @@ QtGLVideoItemInterface::setBuffer (GstBuffer * buffer) gst_caps_take (&qt_item->priv->caps, qt_item->priv->new_caps); qt_item->priv->new_caps = NULL; qt_item->priv->v_info = qt_item->priv->new_v_info; + qt_item->priv->tex_target = qt_item->priv->new_tex_target; if (!_calculate_par (qt_item, &qt_item->priv->v_info)) { g_mutex_unlock (&qt_item->priv->lock); @@ -829,6 +833,9 @@ QtGLVideoItemInterface::setCaps (GstCaps * caps) if (!gst_video_info_from_caps (&v_info, caps)) return FALSE; + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *target_str = gst_structure_get_string (s, "texture-target"); + g_mutex_lock (&qt_item->priv->lock); GST_DEBUG ("%p set caps %" GST_PTR_FORMAT, qt_item, caps); @@ -836,6 +843,11 @@ QtGLVideoItemInterface::setCaps (GstCaps * caps) gst_caps_replace (&qt_item->priv->new_caps, caps); qt_item->priv->new_v_info = v_info; + if (target_str) { + qt_item->priv->new_tex_target = gst_gl_texture_target_from_string(target_str); + } else { + qt_item->priv->new_tex_target = GST_GL_TEXTURE_TARGET_2D; + } g_mutex_unlock (&qt_item->priv->lock);