mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-26 03:31:05 +00:00
Code Issues 1 Projects Releases Wiki Activity

qml: add support for non-RGBA formats as input format

Browse source 
Currently supported are RGBA, BGRA and YV12

Output is still RGBA textures

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5119>
This commit is contained in:
Matthew Waters 2023-07-20 21:12:55 +10:00 committed by GStreamer Marge Bot
parent da3229d0cc
commit 65fc381403
11 changed files with 611 additions and 284 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
subprojects/gst-plugins-good/docs/gst_plugins_cache.json
View file

@ -11999,12 +11999,12 @@
"long-name": "Qt Video Overlay",
"pad-templates": {
"sink": {
"caps": "video/x-raw(ANY):\n format: RGBA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(memory:GLMemory):\n format: RGBA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"caps": "video/x-raw(ANY):\n format: { RGBA, BGRA, YV12 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"direction": "sink",
"presence": "always"
},
"src": {
"caps": "video/x-raw(memory:GLMemory):\n format: RGBA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n\nvideo/x-raw(ANY):\n format: RGBA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"caps": "video/x-raw(memory:GLMemory):\n format: RGBA\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"direction": "src",
"presence": "always"
}
@ -12101,7 +12101,7 @@
"long-name": "Qt Video Sink",
"pad-templates": {
"sink": {
"caps": "video/x-raw(memory:GLMemory):\n format: { RGB, RGBA }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"caps": "video/x-raw(memory:GLMemory):\n format: { RGB, RGBA, BGRA, YV12 }\n width: [ 1, 2147483647 ]\n height: [ 1, 2147483647 ]\n framerate: [ 0/1, 2147483647/1 ]\n texture-target: 2D\n",
"direction": "sink",
"presence": "always"
}

498
subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.cc Normal file
View file

@ -0,0 +1,498 @@
/*
* GStreamer
* Copyright (C) 2023 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <vector>
#include <stdio.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include <gst/gl/gstglfuncs.h>
#include "gstqsgmaterial.h"
#define GST_CAT_DEFAULT gst_qsg_texture_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
#define ATTRIBUTE_POSITION_NAME "a_position"
#define ATTRIBUTE_TEXCOORD_NAME "a_texcoord"
#define UNIFORM_POSITION_MATRIX_NAME "u_transformation"
#define UNIFORM_OPACITY_NAME "opacity"
#define UNIFORM_SWIZZLE_COMPONENTS_NAME "swizzle_components"
#define UNIFORM_TEXTURE0_NAME "tex"
#define UNIFORM_YUV_OFFSET_NAME "yuv_offset"
#define UNIFORM_YUV_YCOEFF_NAME "yuv_ycoeff"
#define UNIFORM_YUV_UCOEFF_NAME "yuv_ucoeff"
#define UNIFORM_YUV_VCOEFF_NAME "yuv_vcoeff"
#define UNIFORM_TRIPLANAR_PLANE0 "Ytex"
#define UNIFORM_TRIPLANAR_PLANE1 "Utex"
#define UNIFORM_TRIPLANAR_PLANE2 "Vtex"
/* matrices from glcolorconvert */
/* FIXME: use the colormatrix support from videoconvert */
/* BT. 601 standard with the following ranges:
* Y = [16..235] (of 255)
* Cb/Cr = [16..240] (of 255)
*/
static const gfloat from_yuv_bt601_offset[] = {-0.0625f, -0.5f, -0.5f};
static const gfloat from_yuv_bt601_rcoeff[] = {1.164f, 0.000f, 1.596f};
static const gfloat from_yuv_bt601_gcoeff[] = {1.164f,-0.391f,-0.813f};
static const gfloat from_yuv_bt601_bcoeff[] = {1.164f, 2.018f, 0.000f};
/* BT. 709 standard with the following ranges:
* Y = [16..235] (of 255)
* Cb/Cr = [16..240] (of 255)
*/
static const gfloat from_yuv_bt709_offset[] = {-0.0625f, -0.5f, -0.5f};
static const gfloat from_yuv_bt709_rcoeff[] = {1.164f, 0.000f, 1.787f};
static const gfloat from_yuv_bt709_gcoeff[] = {1.164f,-0.213f,-0.531f};
static const gfloat from_yuv_bt709_bcoeff[] = {1.164f,2.112f, 0.000f};
class GstQSGMaterialShader : public QSGMaterialShader {
public:
GstQSGMaterialShader(GstVideoFormat v_format, char *vertex, char *fragment);
~GstQSGMaterialShader();
void updateState(const RenderState &state, QSGMaterial *newMaterial, QSGMaterial *oldMaterial) override
{
Q_ASSERT(program()->isLinked());
if (state.isMatrixDirty())
program()->setUniformValue(m_id_matrix, state.combinedMatrix());
if (state.isOpacityDirty())
program()->setUniformValue(m_id_opacity, state.opacity());
GstQSGMaterial *mat = static_cast<GstQSGMaterial *>(newMaterial);
mat->bind(this);
}
char const *const *attributeNames() const override
{
static char const *const names[] = { ATTRIBUTE_POSITION_NAME, ATTRIBUTE_TEXCOORD_NAME, 0 };
return names;
}
void initialize() override
{
const GstVideoFormatInfo *finfo = gst_video_format_get_info (v_format);
QSGMaterialShader::initialize();
m_id_matrix = program()->uniformLocation(UNIFORM_POSITION_MATRIX_NAME);
m_id_opacity = program()->uniformLocation(UNIFORM_OPACITY_NAME);
int swizzle_components = program()->uniformLocation(UNIFORM_SWIZZLE_COMPONENTS_NAME);
int reorder[4];
gst_gl_video_format_swizzle (v_format, reorder);
program()->setUniformValueArray(swizzle_components, reorder, G_N_ELEMENTS (reorder));
const char *tex_names[GST_VIDEO_MAX_PLANES];
switch (v_format) {
case GST_VIDEO_FORMAT_RGB:
case GST_VIDEO_FORMAT_RGBA:
case GST_VIDEO_FORMAT_BGRA:
tex_names[0] = UNIFORM_TEXTURE0_NAME;
break;
case GST_VIDEO_FORMAT_YV12:
tex_names[0] = UNIFORM_TRIPLANAR_PLANE0;
tex_names[1] = UNIFORM_TRIPLANAR_PLANE1;
tex_names[2] = UNIFORM_TRIPLANAR_PLANE2;
break;
default:
g_assert_not_reached ();
break;
}
for (guint i = 0; i < finfo->n_planes; i++) {
this->tex_uniforms[i] = program()->uniformLocation(tex_names[i]);
GST_TRACE ("%p tex uniform %i for tex %s", this, this->tex_uniforms[i], tex_names[i]);
}
this->cms_uniform_offset = program()->uniformLocation(UNIFORM_YUV_OFFSET_NAME);
this->cms_uniform_ycoeff = program()->uniformLocation(UNIFORM_YUV_YCOEFF_NAME);
this->cms_uniform_ucoeff = program()->uniformLocation(UNIFORM_YUV_UCOEFF_NAME);
this->cms_uniform_vcoeff = program()->uniformLocation(UNIFORM_YUV_VCOEFF_NAME);
}
const char *vertexShader() const override;
const char *fragmentShader() const override;
int cms_uniform_offset;
int cms_uniform_ycoeff;
int cms_uniform_ucoeff;
int cms_uniform_vcoeff;
int tex_uniforms[GST_VIDEO_MAX_PLANES];
private:
int m_id_matrix;
int m_id_opacity;
GstVideoFormat v_format;
char *vertex;
char *fragment;
};
GstQSGMaterialShader::GstQSGMaterialShader(GstVideoFormat v_format, char * vertex, char * fragment)
: v_format(v_format),
vertex(vertex),
fragment(fragment)
{
}
GstQSGMaterialShader::~GstQSGMaterialShader()
{
g_clear_pointer (&this->vertex, g_free);
g_clear_pointer (&this->fragment, g_free);
}
const char *
GstQSGMaterialShader::vertexShader() const
{
return vertex;
}
const char *
GstQSGMaterialShader::fragmentShader() const
{
return fragment;
}
GstQSGMaterial::GstQSGMaterial ()
{
static gsize _debug;
if (g_once_init_enter (&_debug)) {
GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsgmaterial", 0,
"Qt Scenegraph Material");
g_once_init_leave (&_debug, 1);
}
g_weak_ref_init (&this->qt_context_ref_, NULL);
gst_video_info_init (&this->v_info);
memset (&this->v_frame, 0, sizeof (this->v_frame));
this->buffer_ = NULL;
this->buffer_was_bound = FALSE;
this->sync_buffer_ = gst_buffer_new ();
}
GstQSGMaterial::~GstQSGMaterial ()
{
g_weak_ref_clear (&this->qt_context_ref_);
gst_buffer_replace (&this->buffer_, NULL);
gst_buffer_replace (&this->sync_buffer_, NULL);
this->buffer_was_bound = FALSE;
if (this->v_frame.buffer) {
gst_video_frame_unmap (&this->v_frame);
memset (&this->v_frame, 0, sizeof (this->v_frame));
}
}
bool
GstQSGMaterial::compatibleWith(GstVideoInfo * v_info)
{
if (GST_VIDEO_INFO_FORMAT (&this->v_info) != GST_VIDEO_INFO_FORMAT (v_info))
return FALSE;
return TRUE;
}
static char *
vertexShaderForFormat(GstVideoFormat v_format)
{
return g_strdup (gst_gl_shader_string_vertex_mat4_vertex_transform);
}
#define qt_inputs \
"attribute vec4 " ATTRIBUTE_POSITION ";\n" \
"attribute vec2 " ATTRIBUTE_TEXCOORD ";\n" \
#define texcoord_input \
"varying vec2 v_texcoord;\n"
#define single_texture_input \
"uniform sampler2D " UNIFORM_TEXTURE0_NAME ";\n"
#define triplanar_texture_input \
"uniform sampler2D " UNIFORM_TRIPLANAR_PLANE0 ";\n" \
"uniform sampler2D " UNIFORM_TRIPLANAR_PLANE1 ";\n" \
"uniform sampler2D " UNIFORM_TRIPLANAR_PLANE2 ";\n"
#define uniform_swizzle \
"uniform int[4] " UNIFORM_SWIZZLE_COMPONENTS_NAME ";\n"
#define uniform_opacity \
"uniform float " UNIFORM_OPACITY_NAME ";\n"
#define uniform_yuv_to_rgb_color_matrix \
"uniform vec3 " UNIFORM_YUV_OFFSET_NAME ";\n" \
"uniform vec3 " UNIFORM_YUV_YCOEFF_NAME ";\n" \
"uniform vec3 " UNIFORM_YUV_UCOEFF_NAME ";\n" \
"uniform vec3 " UNIFORM_YUV_VCOEFF_NAME ";\n"
static char *
fragmentShaderForFormat(GstVideoFormat v_format, GstGLContext * context)
{
switch (v_format) {
case GST_VIDEO_FORMAT_RGB:
case GST_VIDEO_FORMAT_RGBA: {
char *swizzle = gst_gl_color_convert_swizzle_shader_string (context);
char *ret = g_strdup_printf (texcoord_input single_texture_input uniform_opacity
"%s\n"
"void main(void) {\n"
" gl_FragColor = texture2D(tex, v_texcoord) * " UNIFORM_OPACITY_NAME ";\n"
"}\n", swizzle);
g_clear_pointer (&swizzle, g_free);
return ret;
}
case GST_VIDEO_FORMAT_BGRA: {
char *swizzle = gst_gl_color_convert_swizzle_shader_string (context);
char *ret = g_strdup_printf (texcoord_input single_texture_input uniform_swizzle uniform_opacity
"%s\n"
"void main(void) {\n"
" gl_FragColor = swizzle(texture2D(tex, v_texcoord), " UNIFORM_SWIZZLE_COMPONENTS_NAME ") * " UNIFORM_OPACITY_NAME ";\n"
"}\n", swizzle);
g_clear_pointer (&swizzle, g_free);
return ret;
}
case GST_VIDEO_FORMAT_YV12: {
char *yuv_to_rgb = gst_gl_color_convert_yuv_to_rgb_shader_string (context);
char *swizzle = gst_gl_color_convert_swizzle_shader_string (context);
char *ret = g_strdup_printf (texcoord_input triplanar_texture_input uniform_swizzle uniform_yuv_to_rgb_color_matrix uniform_opacity
"%s\n"
"%s\n"
"void main(void) {\n"
" vec4 yuva, rgba;\n"
" yuva.x = texture2D(Ytex, v_texcoord).r;\n"
" yuva.y = texture2D(Utex, v_texcoord).r;\n"
" yuva.z = texture2D(Vtex, v_texcoord).r;\n"
" yuva.a = 1.0;\n"
" yuva = swizzle(yuva, " UNIFORM_SWIZZLE_COMPONENTS_NAME ");\n"
" rgba.rgb = yuv_to_rgb (yuva.xyz, " UNIFORM_YUV_OFFSET_NAME ", " UNIFORM_YUV_YCOEFF_NAME ", " UNIFORM_YUV_UCOEFF_NAME ", " UNIFORM_YUV_VCOEFF_NAME ");\n"
" rgba.a = yuva.a;\n"
" gl_FragColor = rgba * " UNIFORM_OPACITY_NAME ";\n"
//" gl_FragColor = vec4(yuva.x, 0.0, 0.0, 1.0);\n"
"}\n", yuv_to_rgb, swizzle);
g_clear_pointer (&yuv_to_rgb, g_free);
g_clear_pointer (&swizzle, g_free);
return ret;
}
default:
return NULL;
}
}
QSGMaterialShader *
GstQSGMaterial::createShader() const
{
GstVideoFormat v_format = GST_VIDEO_INFO_FORMAT (&this->v_info);
char *vertex = vertexShaderForFormat(v_format);
char *fragment = fragmentShaderForFormat(v_format, NULL);
if (!vertex || !fragment)
return nullptr;
return new GstQSGMaterialShader(v_format, vertex, fragment);
}
/* only called from the streaming thread with scene graph thread blocked */
void
GstQSGMaterial::setCaps (GstCaps * caps)
{
GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps);
gst_video_info_from_caps (&this->v_info, caps);
}
/* only called from the streaming thread with scene graph thread blocked */
gboolean
GstQSGMaterial::setBuffer (GstBuffer * buffer)
{
GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer);
/* FIXME: update more state here */
if (!gst_buffer_replace (&this->buffer_, buffer))
return FALSE;
this->buffer_was_bound = FALSE;
g_weak_ref_set (&this->qt_context_ref_, gst_gl_context_get_current ());
return TRUE;
}
/* only called from the streaming thread with scene graph thread blocked */
GstBuffer *
GstQSGMaterial::getBuffer (gboolean * was_bound)
{
GstBuffer *buffer = NULL;
if (this->buffer_)
buffer = gst_buffer_ref (this->buffer_);
if (was_bound)
*was_bound = this->buffer_was_bound;
return buffer;
}
void
GstQSGMaterial::bind(GstQSGMaterialShader *shader)
{
const GstGLFuncs *gl;
GstGLContext *context, *qt_context;
GstGLSyncMeta *sync_meta;
GstMemory *mem;
gboolean use_dummy_tex = TRUE;
if (this->v_frame.buffer) {
gst_video_frame_unmap (&this->v_frame);
memset (&this->v_frame, 0, sizeof (this->v_frame));
}
qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_));
if (!qt_context)
goto out;
if (!this->buffer_)
goto out;
if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN)
goto out;
this->mem_ = gst_buffer_peek_memory (this->buffer_, 0);
if (!this->mem_)
goto out;
gl = qt_context->gl_vtable;
/* FIXME: should really lock the memory to prevent write access */
if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_,
(GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) {
g_assert_not_reached ();
goto out;
}
mem = gst_buffer_peek_memory (this->buffer_, 0);
g_assert (gst_is_gl_memory (mem));
context = ((GstGLBaseMemory *)mem)->context;
sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_);
if (!sync_meta)
sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_);
gst_gl_sync_meta_set_sync_point (sync_meta, context);
gst_gl_sync_meta_wait (sync_meta, qt_context);
if (this->v_frame.info.finfo->flags & GST_VIDEO_FORMAT_FLAG_YUV) {
if (gst_video_colorimetry_matches (&this->v_frame.info.colorimetry,
GST_VIDEO_COLORIMETRY_BT709)) {
this->cms_offset = (gfloat *) from_yuv_bt709_offset;
this->cms_ycoeff = (gfloat *) from_yuv_bt709_rcoeff;
this->cms_ucoeff = (gfloat *) from_yuv_bt709_gcoeff;
this->cms_vcoeff = (gfloat *) from_yuv_bt709_bcoeff;
} else {
/* defaults/bt601 */
this->cms_offset = (gfloat *) from_yuv_bt601_offset;
this->cms_ycoeff = (gfloat *) from_yuv_bt601_rcoeff;
this->cms_ucoeff = (gfloat *) from_yuv_bt601_gcoeff;
this->cms_vcoeff = (gfloat *) from_yuv_bt601_bcoeff;
}
shader->program()->setUniformValue(shader->cms_uniform_offset, QVector3D(this->cms_offset[0], this->cms_offset[1], this->cms_offset[2]));
shader->program()->setUniformValue(shader->cms_uniform_ycoeff, QVector3D(this->cms_ycoeff[0], this->cms_ycoeff[1], this->cms_ycoeff[2]));
shader->program()->setUniformValue(shader->cms_uniform_ucoeff, QVector3D(this->cms_ucoeff[0], this->cms_ucoeff[1], this->cms_ucoeff[2]));
shader->program()->setUniformValue(shader->cms_uniform_vcoeff, QVector3D(this->cms_vcoeff[0], this->cms_vcoeff[1], this->cms_vcoeff[2]));
} else {
this->cms_offset = this->cms_ycoeff = this->cms_ucoeff = this->cms_vcoeff = NULL;
}
/* reversed iteration order so that glActiveTexture(GL_TEXTURE0) is last which keeps
* us in the default GL state expected by several other qml components
*/
for (int i = GST_VIDEO_FRAME_N_PLANES (&this->v_frame) - 1; i >= 0; i--) {
guint tex_id = *(guint *) this->v_frame.data[i];
shader->program()->setUniformValue(shader->tex_uniforms[i], i);
gl->ActiveTexture (GL_TEXTURE0 + i);
GST_LOG ("%p binding for plane %d Qt texture %u", this, i, tex_id);
gl->BindTexture (GL_TEXTURE_2D, tex_id);
}
/* Texture was successfully bound, so we do not need
* to use the dummy texture */
use_dummy_tex = FALSE;
this->buffer_was_bound = TRUE;
out:
gst_clear_object (&qt_context);
if (G_UNLIKELY (use_dummy_tex)) {
QOpenGLContext *qglcontext = QOpenGLContext::currentContext ();
QOpenGLFunctions *funcs = qglcontext->functions ();
/* Create dummy texture if not already present.
* Use the Qt OpenGL functions instead of the GstGL ones,
* since we are using the Qt OpenGL context here, and we must
* be able to delete the texture in the destructor. */
for (int i = GST_VIDEO_FRAME_N_PLANES (&this->v_frame) - 1; i >= 0; i--) {
shader->program()->setUniformValue(shader->tex_uniforms[i], i);
funcs->glActiveTexture(GL_TEXTURE0 + i);
if (this->dummy_textures[i] == 0) {
/* Make this a black 64x64 pixel RGBA texture.
* This size and format is supported pretty much everywhere, so these
* are a safe pick. (64 pixel sidelength must be supported according
* to the GLES2 spec, table 6.18.)
* Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */
const int tex_sidelength = 64;
std::vector < guint8 > dummy_data (tex_sidelength * tex_sidelength * 4, 0);
switch (GST_VIDEO_FRAME_FORMAT (&this->v_frame)) {
case GST_VIDEO_FORMAT_RGBA:
case GST_VIDEO_FORMAT_BGRA:
case GST_VIDEO_FORMAT_RGB:
break;
case GST_VIDEO_FORMAT_YV12:
if (i == 1 || i == 2) {
guint8 *data = dummy_data.data();
for (gsize j = 0; j < tex_sidelength; j++) {
for (gsize k = 0; k < tex_sidelength; k++) {
data[(j * tex_sidelength + k) * 4 + 0] = 0x7F;
}
}
}
break;
default:
g_assert_not_reached ();
break;
}
funcs->glGenTextures (1, &this->dummy_textures[i]);
funcs->glBindTexture (GL_TEXTURE_2D, this->dummy_textures[i]);
funcs->glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
funcs->glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
funcs->glTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA, tex_sidelength,
tex_sidelength, 0, GL_RGBA, GL_UNSIGNED_BYTE, &dummy_data[0]);
}
g_assert (this->dummy_textures[i] != 0);
funcs->glBindTexture (GL_TEXTURE_2D, this->dummy_textures[i]);
GST_LOG ("%p binding for plane %d fallback dummy Qt texture %u", this, i, this->dummy_textures[i]);
}
}
}

37
subprojects/gst-plugins-good/ext/qt/gstqsgtexture.h → subprojects/gst-plugins-good/ext/qt/gstqsgmaterial.h
View file

@ -18,34 +18,37 @@
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_QSG_TEXTURE_H__
#define __GST_QSG_TEXTURE_H__
#ifndef __GST_QSG_MATERIAL_H__
#define __GST_QSG_MATERIAL_H__
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include "gstqtgl.h"
#include <QtQuick/QSGTexture>
#include <QtQuick/QSGMaterial>
#include <QtQuick/QSGMaterialShader>
#include <QtGui/QOpenGLFunctions>
#include <QtGui/QOpenGLShaderProgram>
class GstQSGTexture : public QSGTexture, protected QOpenGLFunctions
class GstQSGMaterialShader;
class GstQSGMaterial : public QSGMaterial
{
Q_OBJECT
public:
GstQSGTexture ();
~GstQSGTexture ();
GstQSGMaterial ();
~GstQSGMaterial ();
void setCaps (GstCaps * caps);
gboolean setBuffer (GstBuffer * buffer);
GstBuffer * getBuffer (gboolean * was_bound);
bool compatibleWith(GstVideoInfo *v_info);
/* QSGTexture */
void bind ();
int textureId () const;
QSize textureSize () const;
bool hasAlphaChannel () const;
bool hasMipmaps () const;
void bind(GstQSGMaterialShader *);
/* QSGMaterial */
QSGMaterialType *type() const override { static QSGMaterialType type; return &type; };
QSGMaterialShader *createShader() const override;
private:
GstBuffer * buffer_;
@ -53,9 +56,13 @@ private:
GstBuffer * sync_buffer_;
GWeakRef qt_context_ref_;
GstMemory * mem_;
GLuint dummy_tex_id_;
GstVideoInfo v_info;
GstVideoFrame v_frame;
float *cms_offset;
float *cms_ycoeff;
float *cms_ucoeff;
float *cms_vcoeff;
guint dummy_textures[GST_VIDEO_MAX_PLANES];
};
#endif /* __GST_QSG_TEXTURE_H__ */
#endif /* __GST_QSG_MATERIAL_H__ */

248
subprojects/gst-plugins-good/ext/qt/gstqsgtexture.cc
View file

@ -1,248 +0,0 @@
/*
* GStreamer
* Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <vector>
#include <stdio.h>
#include <gst/video/video.h>
#include <gst/gl/gl.h>
#include <gst/gl/gstglfuncs.h>
#include "gstqsgtexture.h"
#define GST_CAT_DEFAULT gst_qsg_texture_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
GstQSGTexture::GstQSGTexture ()
{
static gsize _debug;
initializeOpenGLFunctions();
if (g_once_init_enter (&_debug)) {
GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "qtqsgtexture", 0,
"Qt Scenegraph Texture");
g_once_init_leave (&_debug, 1);
}
g_weak_ref_init (&this->qt_context_ref_, NULL);
gst_video_info_init (&this->v_info);
this->buffer_ = NULL;
this->buffer_was_bound = FALSE;
this->sync_buffer_ = gst_buffer_new ();
this->dummy_tex_id_ = 0;
}
GstQSGTexture::~GstQSGTexture ()
{
g_weak_ref_clear (&this->qt_context_ref_);
gst_buffer_replace (&this->buffer_, NULL);
gst_buffer_replace (&this->sync_buffer_, NULL);
this->buffer_was_bound = FALSE;
if (this->dummy_tex_id_ && QOpenGLContext::currentContext ()) {
QOpenGLContext::currentContext ()->functions ()->glDeleteTextures (1,
&this->dummy_tex_id_);
}
}
/* only called from the streaming thread with scene graph thread blocked */
void
GstQSGTexture::setCaps (GstCaps * caps)
{
GST_LOG ("%p setCaps %" GST_PTR_FORMAT, this, caps);
gst_video_info_from_caps (&this->v_info, caps);
}
/* only called from the streaming thread with scene graph thread blocked */
gboolean
GstQSGTexture::setBuffer (GstBuffer * buffer)
{
GST_LOG ("%p setBuffer %" GST_PTR_FORMAT, this, buffer);
/* FIXME: update more state here */
if (!gst_buffer_replace (&this->buffer_, buffer))
return FALSE;
this->buffer_was_bound = FALSE;
g_weak_ref_set (&this->qt_context_ref_, gst_gl_context_get_current ());
return TRUE;
}
/* only called from the streaming thread with scene graph thread blocked */
GstBuffer *
GstQSGTexture::getBuffer (gboolean * was_bound)
{
GstBuffer *buffer = NULL;
if (this->buffer_)
buffer = gst_buffer_ref (this->buffer_);
if (was_bound)
*was_bound = this->buffer_was_bound;
return buffer;
}
/* only called from qt's scene graph render thread */
void
GstQSGTexture::bind ()
{
const GstGLFuncs *gl;
GstGLContext *context, *qt_context;
GstGLSyncMeta *sync_meta;
GstMemory *mem;
guint tex_id;
gboolean use_dummy_tex = TRUE;
qt_context = GST_GL_CONTEXT (g_weak_ref_get (&this->qt_context_ref_));
if (!qt_context)
goto out;
if (!this->buffer_)
goto out;
if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN)
goto out;
this->mem_ = gst_buffer_peek_memory (this->buffer_, 0);
if (!this->mem_)
goto out;
gl = qt_context->gl_vtable;
/* FIXME: should really lock the memory to prevent write access */
if (!gst_video_frame_map (&this->v_frame, &this->v_info, this->buffer_,
(GstMapFlags) (GST_MAP_READ | GST_MAP_GL))) {
g_assert_not_reached ();
goto out;
}
mem = gst_buffer_peek_memory (this->buffer_, 0);
g_assert (gst_is_gl_memory (mem));
context = ((GstGLBaseMemory *)mem)->context;
sync_meta = gst_buffer_get_gl_sync_meta (this->sync_buffer_);
if (!sync_meta)
sync_meta = gst_buffer_add_gl_sync_meta (context, this->sync_buffer_);
gst_gl_sync_meta_set_sync_point (sync_meta, context);
gst_gl_sync_meta_wait (sync_meta, qt_context);
tex_id = *(guint *) this->v_frame.data[0];
GST_LOG ("%p binding Qt texture %u", this, tex_id);
gl->BindTexture (GL_TEXTURE_2D, tex_id);
gst_video_frame_unmap (&this->v_frame);
/* Texture was successfully bound, so we do not need
* to use the dummy texture */
use_dummy_tex = FALSE;
this->buffer_was_bound = TRUE;
out:
gst_clear_object (&qt_context);
if (G_UNLIKELY (use_dummy_tex)) {
QOpenGLContext *qglcontext = QOpenGLContext::currentContext ();
QOpenGLFunctions *funcs = qglcontext->functions ();
/* Create dummy texture if not already present.
* Use the Qt OpenGL functions instead of the GstGL ones,
* since we are using the Qt OpenGL context here, and we must
* be able to delete the texture in the destructor. */
if (this->dummy_tex_id_ == 0) {
/* Make this a black 64x64 pixel RGBA texture.
* This size and format is supported pretty much everywhere, so these
* are a safe pick. (64 pixel sidelength must be supported according
* to the GLES2 spec, table 6.18.)
* Set min/mag filters to GL_LINEAR to make sure no mipmapping is used. */
const int tex_sidelength = 64;
std::vector < guint8 > dummy_data (tex_sidelength * tex_sidelength * 4, 0);
funcs->glGenTextures (1, &this->dummy_tex_id_);
funcs->glBindTexture (GL_TEXTURE_2D, this->dummy_tex_id_);
funcs->glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
funcs->glTexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
funcs->glTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA, tex_sidelength,
tex_sidelength, 0, GL_RGBA, GL_UNSIGNED_BYTE, &dummy_data[0]);
}
g_assert (this->dummy_tex_id_ != 0);
funcs->glBindTexture (GL_TEXTURE_2D, this->dummy_tex_id_);
GST_LOG ("%p binding fallback dummy Qt texture %u", this, this->dummy_tex_id_);
}
}
/* can be called from any thread */
int
GstQSGTexture::textureId () const
{
int tex_id = 0;
if (this->buffer_) {
GstMemory *mem = gst_buffer_peek_memory (this->buffer_, 0);
tex_id = ((GstGLMemory *) mem)->tex_id;
}
GST_LOG ("%p get texture id %u", this, tex_id);
return tex_id;
}
/* can be called from any thread */
QSize
GstQSGTexture::textureSize () const
{
if (GST_VIDEO_INFO_FORMAT (&this->v_info) == GST_VIDEO_FORMAT_UNKNOWN)
return QSize (0, 0);
GST_TRACE ("%p get texture size %ux%u", this, this->v_info.width,
this->v_info.height);
return QSize (this->v_info.width, this->v_info.height);
}
/* can be called from any thread */
bool
GstQSGTexture::hasAlphaChannel () const
{
const bool has_alpha = GST_VIDEO_FORMAT_INFO_HAS_ALPHA(this->v_info.finfo);
GST_LOG ("%p get has alpha channel %u", this, has_alpha);
return has_alpha;
}
/* can be called from any thread */
bool
GstQSGTexture::hasMipmaps () const
{
return false;
}

54
subprojects/gst-plugins-good/ext/qt/gstqtoverlay.cc
View file

@ -91,12 +91,41 @@
#define GST_CAT_DEFAULT gst_debug_qt_gl_overlay
GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
/* *INDENT-OFF* */
static GstStaticPadTemplate qt_overlay_src_pad_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
"format = (string) RGBA, "
"width = " GST_VIDEO_SIZE_RANGE ", "
"height = " GST_VIDEO_SIZE_RANGE ", "
"framerate = " GST_VIDEO_FPS_RANGE ","
"texture-target = (string) 2D"
));
static GstStaticPadTemplate qt_overlay_sink_pad_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("video/x-raw(ANY), "
"format = (string) { RGBA, BGRA, YV12 }, "
"width = " GST_VIDEO_SIZE_RANGE ", "
"height = " GST_VIDEO_SIZE_RANGE ", "
"framerate = " GST_VIDEO_FPS_RANGE ","
"texture-target = (string) 2D"
));
/* *INDENT-ON* */
static void gst_qt_overlay_finalize (GObject * object);
static void gst_qt_overlay_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * param_spec);
static void gst_qt_overlay_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * param_spec);
static GstCaps * gst_qt_overlay_transform_internal_caps (GstGLFilter * filter,
GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps);
static gboolean gst_qt_overlay_gl_start (GstGLBaseFilter * bfilter);
static void gst_qt_overlay_gl_stop (GstGLBaseFilter * bfilter);
static gboolean gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter,
@ -193,7 +222,10 @@ gst_qt_overlay_class_init (GstQtOverlayClass * klass)
g_signal_new ("qml-scene-destroyed", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);
gst_gl_filter_add_rgba_pad_templates (glfilter_class);
gst_element_class_add_static_pad_template (element_class,
&qt_overlay_src_pad_template);
gst_element_class_add_static_pad_template (element_class,
&qt_overlay_sink_pad_template);
btrans_class->prepare_output_buffer = gst_qt_overlay_prepare_output_buffer;
btrans_class->transform = gst_qt_overlay_transform;
@ -202,6 +234,8 @@ gst_qt_overlay_class_init (GstQtOverlayClass * klass)
glbasefilter_class->gl_stop = gst_qt_overlay_gl_stop;
glbasefilter_class->gl_set_caps = gst_qt_overlay_gl_set_caps;
glfilter_class->transform_internal_caps = gst_qt_overlay_transform_internal_caps;
element_class->change_state = gst_qt_overlay_change_state;
}
@ -398,6 +432,24 @@ gst_qt_overlay_gl_set_caps (GstGLBaseFilter * bfilter, GstCaps * in_caps,
return TRUE;
}
static GstCaps *
gst_qt_overlay_transform_internal_caps (GstGLFilter * filter,
GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps)
{
GstCaps *tmp = GST_GL_FILTER_CLASS (parent_class)->transform_internal_caps (filter, direction, caps, filter_caps);
int i, n;
n = gst_caps_get_size (tmp);
for (i = 0; i < n; i++) {
GstStructure *s = gst_caps_get_structure (tmp, i);
gst_structure_remove_fields (s, "format", "colorimetry", "chroma-site",
"texture-target", NULL);
}
return tmp;
}
static GstFlowReturn
gst_qt_overlay_prepare_output_buffer (GstBaseTransform * btrans,
GstBuffer * buffer, GstBuffer ** outbuf)

2
subprojects/gst-plugins-good/ext/qt/gstqtsink.cc
View file

@ -108,7 +108,7 @@ GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
"format = (string) { RGB, RGBA }, "
"format = (string) { RGB, RGBA, BGRA, YV12 }, "
"width = " GST_VIDEO_SIZE_RANGE ", "
"height = " GST_VIDEO_SIZE_RANGE ", "
"framerate = " GST_VIDEO_FPS_RANGE ", "

3
subprojects/gst-plugins-good/ext/qt/meson.build
View file

@ -1,7 +1,7 @@
sources = [
'gstplugin.cc',
'gstqtelement.cc',
'gstqsgtexture.cc',
'gstqsgmaterial.cc',
'gstqtglutility.cc',
'gstqtoverlay.cc',
'gstqtsink.cc',
@ -14,7 +14,6 @@ sources = [
moc_headers = [
'qtitem.h',
'qtwindow.h',
'gstqsgtexture.h',
'qtglrenderer.h',
]

30
subprojects/gst-plugins-good/ext/qt/qtitem.cc
View file

@ -26,7 +26,7 @@
#include <gst/video/video.h>
#include "qtitem.h"
#include "gstqsgtexture.h"
#include "gstqsgmaterial.h"
#include "gstqtglutility.h"
#include <QtCore/QMutexLocker>
@ -283,9 +283,10 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
if (!this->priv->initted)
return oldNode;
QSGSimpleTextureNode *texNode = static_cast<QSGSimpleTextureNode *> (oldNode);
QSGGeometryNode *texNode = static_cast<QSGGeometryNode *> (oldNode);
GstVideoRectangle src, dst, result;
GstQSGTexture *tex;
GstQSGMaterial *tex = nullptr;
QSGGeometry *geometry = nullptr;
g_mutex_lock (&this->priv->lock);
@ -300,13 +301,22 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
if (gst_gl_context_get_current() == NULL)
gst_gl_context_activate (this->priv->other_context, TRUE);
if (!texNode) {
texNode = new QSGSimpleTextureNode ();
texNode->setOwnsTexture (true);
texNode->setTexture (new GstQSGTexture ());
if (texNode) {
geometry = texNode->geometry();
tex = static_cast<GstQSGMaterial *>(texNode->material());
if (tex && !tex->compatibleWith(&this->priv->v_info)) {
delete texNode;
texNode = nullptr;
}
}
tex = static_cast<GstQSGTexture *> (texNode->texture());
if (!texNode) {
texNode = new QSGGeometryNode();
geometry = new QSGGeometry(QSGGeometry::defaultAttributes_TexturedPoint2D(), 4);
texNode->setGeometry(geometry);
tex = new GstQSGMaterial();
texNode->setMaterial(tex);
}
if ((old_buffer = tex->getBuffer(&was_bound))) {
if (old_buffer == this->priv->buffer) {
@ -360,7 +370,9 @@ QtGLVideoItem::updatePaintNode(QSGNode * oldNode,
result.h = boundingRect().height();
}
texNode->setRect (QRectF (result.x, result.y, result.w, result.h));
QRectF rect(result.x, result.y, result.w, result.h);
QRectF sourceRect(0, 0, 1, 1);
QSGGeometry::updateTexturedRectGeometry(geometry, rect, sourceRect);
g_mutex_unlock (&this->priv->lock);

1
subprojects/gst-plugins-good/ext/qt/qtwindow.cc
View file

@ -27,7 +27,6 @@
#include <gst/video/video.h>
#include <gst/gl/gstglfuncs.h>
#include "qtwindow.h"
#include "gstqsgtexture.h"
#include "gstqtglutility.h"
#include <QtCore/QDateTime>

8
subprojects/gst-plugins-good/tests/examples/qt/qmloverlay/main.cpp
View file

@ -57,6 +57,10 @@ int main(int argc, char *argv[])
GstElement *pipeline = gst_pipeline_new (NULL);
GstElement *src = gst_element_factory_make ("videotestsrc", NULL);
GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL);
GstCaps *caps = gst_caps_from_string ("video/x-raw,format=RGBA");
g_object_set (capsfilter, "caps", caps, NULL);
gst_caps_unref (caps);
GstElement *glupload = gst_element_factory_make ("glupload", NULL);
/* the plugin must be loaded before loading the qml file to register the
* GstGLVideoItem qml item */
@ -66,8 +70,8 @@ int main(int argc, char *argv[])
g_assert (src && glupload && overlay && sink);
gst_bin_add_many (GST_BIN (pipeline), src, glupload, overlay, overlay2, sink, NULL);
gst_element_link_many (src, glupload, overlay, overlay2, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), src, capsfilter, glupload, overlay, overlay2, sink, NULL);
gst_element_link_many (src, capsfilter, glupload, overlay, overlay2, sink, NULL);
/* load qmlglsink output */
QQmlApplicationEngine engine;

8
subprojects/gst-plugins-good/tests/examples/qt/qmlsink/main.cpp
View file

@ -46,6 +46,10 @@ int main(int argc, char *argv[])
GstElement *pipeline = gst_pipeline_new (NULL);
GstElement *src = gst_element_factory_make ("videotestsrc", NULL);
GstElement *capsfilter = gst_element_factory_make ("capsfilter", NULL);
GstCaps *caps = gst_caps_from_string ("video/x-raw,format=YV12");
g_object_set (capsfilter, "caps", caps, NULL);
gst_caps_unref (caps);
GstElement *glupload = gst_element_factory_make ("glupload", NULL);
/* the plugin must be loaded before loading the qml file to register the
* GstGLVideoItem qml item */
@ -53,8 +57,8 @@ int main(int argc, char *argv[])
g_assert (src && glupload && sink);
gst_bin_add_many (GST_BIN (pipeline), src, glupload, sink, NULL);
gst_element_link_many (src, glupload, sink, NULL);
gst_bin_add_many (GST_BIN (pipeline), src, capsfilter, glupload, sink, NULL);
gst_element_link_many (src, capsfilter, glupload, sink, NULL);
QQmlApplicationEngine engine;
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));