From ac393aa65750400a503dfffc60495025270ec6b4 Mon Sep 17 00:00:00 2001 From: Michael Tretter Date: Mon, 22 Jul 2024 16:38:07 +0200 Subject: [PATCH] qml6glsink: add support for texture-target external-oes In order to use oes-external, the qml6glsink needs a fragment shader that uses the samplerExternalOES. The qsb tool is not able to handle shaders that contain samplerExternalOES since this feature is not supported by all target shading languages. The qsb tool is able to replace a shader in the qsb file to handle this use case. Use it to generate a shader variant that uses samplerExternalOES for OpenGL ES and select that variant if the qml6glsink negotiated texture target oes-external. Part-of: --- .../gst-plugins-good/ext/qt6/RGBA_gles.frag | 31 +++++++++++++++ .../gst-plugins-good/ext/qt6/gstqml6glsink.cc | 3 +- .../ext/qt6/gstqsg6material.cc | 5 ++- .../gst-plugins-good/ext/qt6/meson.build | 24 +++++++++--- .../gst-plugins-good/ext/qt6/qsb-wrapper.py | 38 +++++++++++++++++++ .../gst-plugins-good/ext/qt6/resources.qrc | 1 + 6 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 subprojects/gst-plugins-good/ext/qt6/RGBA_gles.frag create mode 100755 subprojects/gst-plugins-good/ext/qt6/qsb-wrapper.py diff --git a/subprojects/gst-plugins-good/ext/qt6/RGBA_gles.frag b/subprojects/gst-plugins-good/ext/qt6/RGBA_gles.frag new file mode 100644 index 0000000000..50077e6ca3 --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/RGBA_gles.frag @@ -0,0 +1,31 @@ +#version 100 +#extension GL_OES_EGL_image_external : require +precision highp float; +precision highp int; + +struct buf +{ + mat4 qt_Matrix; + ivec4 swizzle; + mat4 color_matrix; + float qt_Opacity; +}; + +uniform buf ubuf; + +uniform samplerExternalOES tex; + +varying vec2 vTexCoord; + +vec4 swizzle(vec4 texel, ivec4 swizzle_1) +{ + return vec4(texel[swizzle_1.x], texel[swizzle_1.y], texel[swizzle_1.z], texel[swizzle_1.w]); +} + +void main() +{ + vec4 param = texture2D(tex, vTexCoord); + ivec4 param_1 = ubuf.swizzle; + vec4 texel = swizzle(param, param_1); + gl_FragData[0] = texel * ubuf.qt_Opacity; +} diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc index f3267bfd56..1f1fd93bfb 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstqml6glsink.cc @@ -115,7 +115,8 @@ GST_STATIC_PAD_TEMPLATE ("sink", "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE ", " "framerate = " GST_VIDEO_FPS_RANGE ", " - "texture-target = (string) 2D")); + "texture-target = (string) { " GST_GL_TEXTURE_TARGET_2D_STR ", " + GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR " } ")); #define DEFAULT_FORCE_ASPECT_RATIO TRUE #define DEFAULT_PAR_N 0 diff --git a/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc index 60ef42edba..722ee1412a 100644 --- a/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc +++ b/subprojects/gst-plugins-good/ext/qt6/gstqsg6material.cc @@ -281,7 +281,10 @@ GstQSGMaterialShader::GstQSGMaterialShader(GstVideoFormat v_format, case GST_VIDEO_FORMAT_RGBA: case GST_VIDEO_FORMAT_BGRA: case GST_VIDEO_FORMAT_RGB: - frag_shader = ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb"; + if (target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) + frag_shader = ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb.external"; + else + frag_shader = ":/org/freedesktop/gstreamer/qml6/RGBA.frag.qsb"; break; case GST_VIDEO_FORMAT_YV12: frag_shader = ":/org/freedesktop/gstreamer/qml6/YUV_TRIPLANAR.frag.qsb"; diff --git a/subprojects/gst-plugins-good/ext/qt6/meson.build b/subprojects/gst-plugins-good/ext/qt6/meson.build index 5e2e87b934..dc7abe0a66 100644 --- a/subprojects/gst-plugins-good/ext/qt6/meson.build +++ b/subprojects/gst-plugins-good/ext/qt6/meson.build @@ -19,10 +19,10 @@ moc_headers = [ ] shader_sources = [ - 'vertex.vert', - 'RGBA.frag', - 'YUV_BIPLANAR.frag', - 'YUV_TRIPLANAR.frag', + { 'glsl': 'vertex.vert' }, + { 'glsl': 'RGBA.frag', '100es': 'RGBA_gles.frag'}, + { 'glsl': 'YUV_BIPLANAR.frag' }, + { 'glsl': 'YUV_TRIPLANAR.frag' }, ] qt6qml_dep = dependency('', required: false) @@ -66,6 +66,7 @@ qsb = find_program('qsb-qt6', 'qsb', dirs: [qt6_bindir], required: qt6_option) if not qsb.found() subdir_done() endif +qsb_wrapper = find_program('qsb-wrapper.py') optional_deps = [] qt_defines = [] @@ -167,7 +168,8 @@ if qt6_option.require(have_qt_windowing, error_message: 'No windowing, enable on moc_files = qt6_mod.preprocess(moc_headers : moc_headers, method: qt6_method) # TODO: dist backup qsb shaders? shaders = [] - foreach shader: shader_sources + foreach shader_source: shader_sources + shader = shader_source['glsl'] qsb_shader = shader + '.qsb' dist_shader = shader + '-dist.qsb' @@ -177,6 +179,18 @@ if qt6_option.require(have_qt_windowing, error_message: 'No windowing, enable on command: [qsb, '--glsl=100 es,120,330', '--batchable', '--output', '@OUTPUT@', '@INPUT@'] ) shaders += [compiled_shader] + + if '100es' in shader_source + qsb_shader_external = shader + '.qsb.external' + + compiled_shader_external = custom_target(qsb_shader_external, + input: [compiled_shader, shader_source['100es']], + output: qsb_shader_external, + command: [qsb_wrapper, qsb, '@OUTPUT@', '@INPUT1@', '@INPUT0@'], + ) + + shaders += [compiled_shader_external] + endif endforeach resource_file = configure_file(input: 'resources.qrc', output: 'resources.qrc', copy: true) qresources = qt6_mod.compile_resources(sources: resource_file, method: qt6_method) diff --git a/subprojects/gst-plugins-good/ext/qt6/qsb-wrapper.py b/subprojects/gst-plugins-good/ext/qt6/qsb-wrapper.py new file mode 100755 index 0000000000..b7b11588cd --- /dev/null +++ b/subprojects/gst-plugins-good/ext/qt6/qsb-wrapper.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +# GStreamer +# Copyright (C) 2024 Michael Tretter +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Library General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Library General Public License for more details. +# +# You should have received a copy of the GNU Library General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. +# + +import shutil +import subprocess +import sys + +assert (len(sys.argv) == 5) + +qsb_tool = sys.argv[1] +qsb_output = sys.argv[2] +gles_shader = sys.argv[3] +qsb_input = sys.argv[4] + +# Copy the qsb file since the qsb tool replaces the shader in place +shutil.copyfile(qsb_input, qsb_output) + +subprocess.run([qsb_tool, + '--silent', + '--replace', 'glsl,100es,{}'.format(gles_shader), + qsb_output]) diff --git a/subprojects/gst-plugins-good/ext/qt6/resources.qrc b/subprojects/gst-plugins-good/ext/qt6/resources.qrc index e20f5f809c..3db66f36e3 100644 --- a/subprojects/gst-plugins-good/ext/qt6/resources.qrc +++ b/subprojects/gst-plugins-good/ext/qt6/resources.qrc @@ -1,6 +1,7 @@ vertex.vert.qsb + RGBA.frag.qsb.external RGBA.frag.qsb YUV_BIPLANAR.frag.qsb YUV_TRIPLANAR.frag.qsb