/* * GStreamer * Copyright (C) 2009 Julien Isorce * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public * License as published by the Free Software Foundation; either * version 2 of the License, or (at your option) any later version. * * This library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with this library; if not, write to the * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ /** * SECTION:element-glmosaic * @title: glmosaic * * glmixer sub element. N gl sink pads to 1 source pad. * N + 1 OpenGL contexts shared together. * N <= 6 because the rendering is more a like a cube than a mosaic * Each opengl input stream is rendered on a cube face * * ## Examples * |[ * gst-launch-1.0 videotestsrc ! video/x-raw, format=YUY2 ! glupload ! glcolorconvert ! queue ! glmosaic name=m ! glimagesink \ * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! glupload ! glcolorconvert ! queue ! m. \ * videotestsrc ! video/x-raw, framerate=15/1, width=1500, height=1500 ! glupload ! gleffects effect=3 ! queue ! m. \ * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \ * videotestsrc ! glupload ! glfiltercube ! queue ! m. \ * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. * ]| */ #ifdef HAVE_CONFIG_H #include "config.h" #endif #include "gstglmosaic.h" #include "gstglutils.h" #define GST_CAT_DEFAULT gst_gl_mosaic_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); enum { PROP_0, }; static void gst_gl_mosaic_child_proxy_init (gpointer g_iface, gpointer iface_data); #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_mosaic_debug, "glmosaic", 0, "glmosaic element"); G_DEFINE_TYPE_WITH_CODE (GstGLMosaic, gst_gl_mosaic, GST_TYPE_GL_MIXER, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, gst_gl_mosaic_child_proxy_init); DEBUG_INIT); static GstPad *gst_gl_mosaic_request_new_pad (GstElement * element, GstPadTemplate * temp, const gchar * req_name, const GstCaps * caps); static void gst_gl_mosaic_release_pad (GstElement * element, GstPad * pad); static void gst_gl_mosaic_reset (GstGLMixer * mixer); static gboolean gst_gl_mosaic_set_caps (GstGLMixer * mixer, GstCaps * outcaps); static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mixer, GstGLMemory * out_tex); static gboolean gst_gl_mosaic_callback (gpointer stuff); /* vertex source */ static const gchar *mosaic_v_src = "uniform mat4 u_matrix; \n" "uniform float xrot_degree, yrot_degree, zrot_degree; \n" "attribute vec4 a_position; \n" "attribute vec2 a_texCoord; \n" "varying vec2 v_texCoord; \n" "void main() \n" "{ \n" " float PI = 3.14159265; \n" " float xrot = xrot_degree*2.0*PI/360.0; \n" " float yrot = yrot_degree*2.0*PI/360.0; \n" " float zrot = zrot_degree*2.0*PI/360.0; \n" " mat4 matX = mat4 ( \n" " 1.0, 0.0, 0.0, 0.0, \n" " 0.0, cos(xrot), sin(xrot), 0.0, \n" " 0.0, -sin(xrot), cos(xrot), 0.0, \n" " 0.0, 0.0, 0.0, 1.0 ); \n" " mat4 matY = mat4 ( \n" " cos(yrot), 0.0, -sin(yrot), 0.0, \n" " 0.0, 1.0, 0.0, 0.0, \n" " sin(yrot), 0.0, cos(yrot), 0.0, \n" " 0.0, 0.0, 0.0, 1.0 ); \n" " mat4 matZ = mat4 ( \n" " cos(zrot), sin(zrot), 0.0, 0.0, \n" " -sin(zrot), cos(zrot), 0.0, 0.0, \n" " 0.0, 0.0, 1.0, 0.0, \n" " 0.0, 0.0, 0.0, 1.0 ); \n" " gl_Position = u_matrix * matZ * matY * matX * a_position; \n" " v_texCoord = a_texCoord; \n" "} \n"; /* fragment source */ static const gchar *mosaic_f_src = "uniform sampler2D s_texture; \n" "varying vec2 v_texCoord; \n" "void main() \n" "{ \n" " gl_FragColor = texture2D( s_texture, v_texCoord );\n" "} \n"; static void gst_gl_mosaic_class_init (GstGLMosaicClass * klass) { GstElementClass *element_class; element_class = GST_ELEMENT_CLASS (klass); element_class->request_new_pad = GST_DEBUG_FUNCPTR (gst_gl_mosaic_request_new_pad); element_class->release_pad = GST_DEBUG_FUNCPTR (gst_gl_mosaic_release_pad); gst_element_class_set_metadata (element_class, "OpenGL mosaic", "Filter/Effect/Video", "OpenGL mosaic", "Julien Isorce "); GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_mosaic_set_caps; GST_GL_MIXER_CLASS (klass)->reset = gst_gl_mosaic_reset; GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_mosaic_process_textures; } static void gst_gl_mosaic_init (GstGLMosaic * mosaic) { mosaic->shader = NULL; mosaic->attr_position_loc = -1; mosaic->attr_texture_loc = -1; } static GstPad * gst_gl_mosaic_request_new_pad (GstElement * element, GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps) { GstPad *newpad; newpad = (GstPad *) GST_ELEMENT_CLASS (gst_gl_mosaic_parent_class)->request_new_pad (element, templ, req_name, caps); if (newpad == NULL) goto could_not_create; gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad), GST_OBJECT_NAME (newpad)); return newpad; could_not_create: { GST_DEBUG_OBJECT (element, "could not create/add pad"); return NULL; } } static void gst_gl_mosaic_release_pad (GstElement * element, GstPad * pad) { GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (element); GST_DEBUG_OBJECT (gl_mosaic, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad)); gst_child_proxy_child_removed (GST_CHILD_PROXY (gl_mosaic), G_OBJECT (pad), GST_OBJECT_NAME (pad)); GST_ELEMENT_CLASS (gst_gl_mosaic_parent_class)->release_pad (element, pad); } static void gst_gl_mosaic_reset (GstGLMixer * mixer) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); if (mosaic->shader) gst_object_unref (mosaic->shader); mosaic->shader = NULL; mosaic->attr_position_loc = -1; mosaic->attr_texture_loc = -1; mosaic->xrot = 0.0; mosaic->yrot = 0.0; mosaic->zrot = 0.0; } static gboolean gst_gl_mosaic_set_caps (GstGLMixer * mixer, GstCaps * outcaps) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); g_clear_object (&mosaic->shader); return TRUE; } static void _mosaic_render (GstGLContext * context, GstGLMosaic * mosaic) { GstGLMixer *mixer = GST_GL_MIXER (mosaic); if (!mosaic->shader) { gchar *frag_str = g_strdup_printf ("%s%s", gst_gl_shader_string_get_highest_precision (GST_GL_BASE_MIXER (mixer)->context, GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY), mosaic_f_src); gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, mosaic_v_src, frag_str, &mosaic->shader); g_free (frag_str); } gst_gl_framebuffer_draw_to_texture (mixer->fbo, mosaic->out_tex, gst_gl_mosaic_callback, mosaic); } static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mix, GstGLMemory * out_tex) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mix); GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; mosaic->out_tex = out_tex; gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _mosaic_render, mosaic); return TRUE; } static void _bind_buffer (GstGLMosaic * mosaic) { GstGLContext *context = GST_GL_BASE_MIXER (mosaic)->context; const GstGLFuncs *gl = context->gl_vtable; gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, mosaic->vbo_indices); gl->BindBuffer (GL_ARRAY_BUFFER, mosaic->vertex_buffer); /* Load the vertex position */ gl->VertexAttribPointer (mosaic->attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), (void *) 0); /* Load the texture coordinate */ gl->VertexAttribPointer (mosaic->attr_texture_loc, 2, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); gl->EnableVertexAttribArray (mosaic->attr_position_loc); gl->EnableVertexAttribArray (mosaic->attr_texture_loc); } static void _unbind_buffer (GstGLMosaic * mosaic) { GstGLContext *context = GST_GL_BASE_MIXER (mosaic)->context; const GstGLFuncs *gl = context->gl_vtable; gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); gl->BindBuffer (GL_ARRAY_BUFFER, 0); gl->DisableVertexAttribArray (mosaic->attr_position_loc); gl->DisableVertexAttribArray (mosaic->attr_texture_loc); } /* opengl scene, params: input texture (not the output mixer->texture) */ static gboolean gst_gl_mosaic_callback (gpointer stuff) { GstGLMosaic *mosaic = GST_GL_MOSAIC (stuff); GstGLMixer *mixer = GST_GL_MIXER (mosaic); GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; GList *walk; const GLfloat matrix[] = { 0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 1.0f }; guint count = 0; gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); gl->Enable (GL_DEPTH_TEST); gl->ClearColor (0.0, 0.0, 0.0, 0.0); gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); gst_gl_shader_use (mosaic->shader); if (mosaic->attr_position_loc == -1) { mosaic->attr_position_loc = gst_gl_shader_get_attribute_location (mosaic->shader, "a_position"); } if (mosaic->attr_texture_loc == -1) { mosaic->attr_texture_loc = gst_gl_shader_get_attribute_location (mosaic->shader, "a_texCoord"); } gst_gl_shader_set_uniform_1i (mosaic->shader, "s_texture", 0); gst_gl_shader_set_uniform_1f (mosaic->shader, "xrot_degree", mosaic->xrot); gst_gl_shader_set_uniform_1f (mosaic->shader, "yrot_degree", mosaic->yrot); gst_gl_shader_set_uniform_1f (mosaic->shader, "zrot_degree", mosaic->zrot); gst_gl_shader_set_uniform_matrix_4fv (mosaic->shader, "u_matrix", 1, GL_FALSE, matrix); if (!mosaic->vertex_buffer) { /* *INDENT-OFF* */ gfloat vertices[] = { /* front face */ 1.0f, 1.0f,-1.0f, 1.0f, 0.0f, 1.0f,-1.0f,-1.0f, 1.0f, 1.0f, -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, -1.0f, 1.0f,-1.0f, 0.0f, 0.0f, /* right face */ 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f,-1.0f, 1.0f, 0.0f, 0.0f, 1.0f,-1.0f,-1.0f, 0.0f, 1.0f, 1.0f, 1.0f,-1.0f, 1.0f, 1.0f, /* left face */ -1.0f, 1.0f, 1.0f, 1.0f, 0.0f, -1.0f, 1.0f,-1.0f, 1.0f, 1.0f, -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, -1.0f,-1.0f, 1.0f, 0.0f, 0.0f, /* top face */ 1.0f,-1.0f, 1.0f, 1.0f, 0.0f, -1.0f,-1.0f, 1.0f, 0.0f, 0.0f, -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, 1.0f,-1.0f,-1.0f, 1.0f, 1.0f, /* bottom face */ 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, 1.0f, 1.0f,-1.0f, 1.0f, 1.0f, -1.0f, 1.0f,-1.0f, 0.0f, 1.0f, -1.0f, 1.0f, 1.0f, 0.0f, 0.0f, /* back face */ 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, -1.0f, 1.0f, 1.0f, 0.0f, 0.0f, -1.0f,-1.0f, 1.0f, 0.0f, 1.0f, 1.0f,-1.0f, 1.0f, 1.0f, 1.0f }; const GLushort indices[] = { 0, 1, 2, 0, 2, 3, 4, 5, 6, 4, 6, 7, 8, 9, 10, 8, 10, 11, 12, 13, 14, 12, 14, 15, 16, 17, 18, 16, 18, 19, 20, 21, 22, 20, 22, 23, }; /* *INDENT-ON* */ if (gl->GenVertexArrays) { gl->GenVertexArrays (1, &mosaic->vao); gl->BindVertexArray (mosaic->vao); } gl->GenBuffers (1, &mosaic->vertex_buffer); gl->BindBuffer (GL_ARRAY_BUFFER, mosaic->vertex_buffer); gl->BufferData (GL_ARRAY_BUFFER, sizeof (vertices), vertices, GL_STATIC_DRAW); gl->GenBuffers (1, &mosaic->vbo_indices); gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, mosaic->vbo_indices); gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, GL_STATIC_DRAW); } if (gl->GenVertexArrays) gl->BindVertexArray (mosaic->vao); _bind_buffer (mosaic); GST_OBJECT_LOCK (mosaic); walk = GST_ELEMENT (mosaic)->sinkpads; while (walk) { GstGLMixerPad *pad = walk->data; guint in_tex; guint width, height; in_tex = pad->current_texture; width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); if (count >= 6) { GST_FIXME_OBJECT (mosaic, "Skipping 7th pad (and all subsequent pads)"); break; } if (!in_tex || width <= 0 || height <= 0) { GST_DEBUG ("skipping texture:%u pad:%p width:%u height %u", in_tex, pad, width, height); count++; walk = g_list_next (walk); continue; } GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, width, height); gl->ActiveTexture (GL_TEXTURE0); gl->BindTexture (GL_TEXTURE_2D, in_tex); gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, (void *) (6 * sizeof (GLushort) * count)); ++count; walk = g_list_next (walk); } GST_OBJECT_UNLOCK (mosaic); if (gl->GenVertexArrays) gl->BindVertexArray (0); else _unbind_buffer (mosaic); gl->BindTexture (GL_TEXTURE_2D, 0); gl->Disable (GL_DEPTH_TEST); gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); mosaic->xrot += 0.6f; mosaic->yrot += 0.4f; mosaic->zrot += 0.8f; return TRUE; } /* GstChildProxy implementation */ static GObject * gst_gl_mosaic_child_proxy_get_child_by_index (GstChildProxy * child_proxy, guint index) { GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (child_proxy); GObject *obj = NULL; GST_OBJECT_LOCK (gl_mosaic); obj = g_list_nth_data (GST_ELEMENT_CAST (gl_mosaic)->sinkpads, index); if (obj) gst_object_ref (obj); GST_OBJECT_UNLOCK (gl_mosaic); return obj; } static guint gst_gl_mosaic_child_proxy_get_children_count (GstChildProxy * child_proxy) { guint count = 0; GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (child_proxy); GST_OBJECT_LOCK (gl_mosaic); count = GST_ELEMENT_CAST (gl_mosaic)->numsinkpads; GST_OBJECT_UNLOCK (gl_mosaic); GST_INFO_OBJECT (gl_mosaic, "Children Count: %d", count); return count; } static void gst_gl_mosaic_child_proxy_init (gpointer g_iface, gpointer iface_data) { GstChildProxyInterface *iface = g_iface; iface->get_child_by_index = gst_gl_mosaic_child_proxy_get_child_by_index; iface->get_children_count = gst_gl_mosaic_child_proxy_get_children_count; }