From 279c2b07b422348c8310b84fb54c6f3e013a9a9b Mon Sep 17 00:00:00 2001 From: Lubosz Sarnecki Date: Sat, 13 Jun 2015 15:35:47 +0200 Subject: [PATCH] glcompositionoverlay: Add class for managing GstVideoOverlayCompositionOverlay buffers Add a class to store and manage the OpenGL texture, vertex buffer and GstVideoOverlayRectangle. Transforms overlay coordinate space to vertex buffer space with aspect ratios in mind. = Example Pipelines = Simple pipeline gst-launch-1.0 videotestsrc ! \ textoverlay text="Hello World" font-desc="sans bold 30" ! \ glimagesink Display 3 static overlays at different positions gst-launch-1.0 videotestsrc ! \ textoverlay text="text1" valignment="top" font-desc="sans bold 30" ! \ textoverlay text="text2" halignment="right" font-desc="sans bold 30" ! \ textoverlay text="text3" halignment="left" font-desc="sans bold 30" ! \ glimagesink Display subtitle file over testsrc gst-launch-1.0 videotestsrc ! \ textoverlay name=foo filesrc location=foo.srt ! subparse ! queue ! foo. foo. ! \ glimagesink https://bugzilla.gnome.org/show_bug.cgi?id=745107 --- gst-libs/gst/gl/Makefile.am | 4 +- gst-libs/gst/gl/gstgl_fwd.h | 3 + gst-libs/gst/gl/gstglcompositionoverlay.c | 328 ++++++++++++++++++++++ gst-libs/gst/gl/gstglcompositionoverlay.h | 83 ++++++ 4 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 gst-libs/gst/gl/gstglcompositionoverlay.c create mode 100644 gst-libs/gst/gl/gstglcompositionoverlay.h diff --git a/gst-libs/gst/gl/Makefile.am b/gst-libs/gst/gl/Makefile.am index 023f468e551..942638f5067 100644 --- a/gst-libs/gst/gl/Makefile.am +++ b/gst-libs/gst/gl/Makefile.am @@ -28,7 +28,8 @@ libgstgl_@GST_API_VERSION@_la_SOURCES = \ gstglutils.c \ gstglframebuffer.c \ gstglsyncmeta.c \ - gstglviewconvert.c + gstglviewconvert.c \ + gstglcompositionoverlay.c libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl libgstgl_@GST_API_VERSION@include_HEADERS = \ @@ -52,6 +53,7 @@ libgstgl_@GST_API_VERSION@include_HEADERS = \ gstglframebuffer.h \ gstglsyncmeta.h \ gstglviewconvert.h \ + gstglcompositionoverlay.h \ gstgl_fwd.h \ gl.h diff --git a/gst-libs/gst/gl/gstgl_fwd.h b/gst-libs/gst/gl/gstgl_fwd.h index cbf20e133ce..7c2bd34c5fc 100644 --- a/gst-libs/gst/gl/gstgl_fwd.h +++ b/gst-libs/gst/gl/gstgl_fwd.h @@ -78,6 +78,9 @@ typedef struct _GstGLViewConvert GstGLViewConvert; typedef struct _GstGLViewConvertClass GstGLViewConvertClass; typedef struct _GstGLViewConvertPrivate GstGLViewConvertPrivate; +typedef struct _GstGLCompositionOverlay GstGLCompositionOverlay; +typedef struct _GstGLCompositionOverlayClass GstGLCompositionOverlayClass; + G_END_DECLS #endif /* __GST_GL_FWD_H__ */ diff --git a/gst-libs/gst/gl/gstglcompositionoverlay.c b/gst-libs/gst/gl/gstglcompositionoverlay.c new file mode 100644 index 00000000000..e5f7b9b2133 --- /dev/null +++ b/gst-libs/gst/gl/gstglcompositionoverlay.c @@ -0,0 +1,328 @@ +/* + * GStreamer + * Copyright (C) 2015 Lubosz Sarnecki + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gl.h" +#include "gstglcompositionoverlay.h" + + +GST_DEBUG_CATEGORY_STATIC (gst_gl_composition_overlay_debug); +#define GST_CAT_DEFAULT gst_gl_composition_overlay_debug + + +#define DEBUG_INIT \ + GST_DEBUG_CATEGORY_INIT (gst_gl_composition_overlay_debug, \ + "glcompositionoverlay", 0, "compositionoverlay"); + +G_DEFINE_TYPE_WITH_CODE (GstGLCompositionOverlay, gst_gl_composition_overlay, + GST_TYPE_OBJECT, DEBUG_INIT); + +static void gst_gl_composition_overlay_finalize (GObject * object); +static void +gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay * + overlay, GstBuffer * video_buffer, guint window_width, guint window_height); + +static void +gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass) +{ + G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize; +} + +static void +gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay) +{ +} + +GstGLCompositionOverlay * +gst_gl_composition_overlay_new (GstGLContext * context, + GstVideoOverlayRectangle * rectangle, + GLint position_attrib, GLint texcoord_attrib) +{ + GstGLCompositionOverlay *overlay = + g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL); + + overlay->gl_memory = NULL; + overlay->texture_id = -1; + overlay->rectangle = rectangle; + overlay->context = gst_object_ref (context); + overlay->vao = 0; + overlay->position_attrib = position_attrib; + overlay->texcoord_attrib = texcoord_attrib; + + GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay"); + + return overlay; +} + +static void +gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context, + gpointer overlay_pointer) +{ + const GstGLFuncs *gl = context->gl_vtable; + GstGLCompositionOverlay *overlay = + (GstGLCompositionOverlay *) overlay_pointer; + if (overlay->vao) { + gl->DeleteVertexArrays (1, &overlay->vao); + overlay->vao = 0; + } + + if (overlay->position_buffer) { + gl->DeleteBuffers (1, &overlay->position_buffer); + overlay->position_buffer = 0; + } + + if (overlay->texcoord_buffer) { + gl->DeleteBuffers (1, &overlay->position_buffer); + overlay->position_buffer = 0; + } + + if (overlay->index_buffer) { + gl->DeleteBuffers (1, &overlay->index_buffer); + overlay->index_buffer = 0; + } +} + +static void +gst_gl_composition_overlay_finalize (GObject * object) +{ + GstGLCompositionOverlay *overlay; + + overlay = GST_GL_COMPOSITION_OVERLAY (object); + + if (overlay->gl_memory) + gst_memory_unref ((GstMemory *) overlay->gl_memory); + overlay->gl_memory = NULL; + if (overlay->context) { + gst_gl_context_thread_add (overlay->context, + gst_gl_composition_overlay_free_vertex_buffer, overlay); + gst_object_unref (overlay->context); + } + + G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object); +} + +static void +gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context, + gpointer overlay_pointer) +{ + const GstGLFuncs *gl = context->gl_vtable; + GstGLCompositionOverlay *overlay = + (GstGLCompositionOverlay *) overlay_pointer; + + /* *INDENT-OFF* */ + static const GLfloat texcoords[] = { + 1.0f, 0.0f, + 0.0f, 0.0f, + 0.0f, 1.0f, + 1.0f, 1.0f + }; + + static const GLushort indices[] = { + 0, 1, 2, 0, 2, 3 + }; + /* *INDENT-ON* */ + + gl->GenVertexArrays (1, &overlay->vao); + gl->BindVertexArray (overlay->vao); + + gl->GenBuffers (1, &overlay->position_buffer); + gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer); + gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions, + GL_STATIC_DRAW); + + /* Load the vertex position */ + gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE, + 4 * sizeof (GLfloat), NULL); + + gl->GenBuffers (1, &overlay->texcoord_buffer); + gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer); + gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords, + GL_STATIC_DRAW); + + /* Load the texture coordinate */ + gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE, + 2 * sizeof (GLfloat), NULL); + + gl->GenBuffers (1, &overlay->index_buffer); + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer); + gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, + GL_STATIC_DRAW); + + gl->EnableVertexAttribArray (overlay->position_attrib); + gl->EnableVertexAttribArray (overlay->texcoord_attrib); + + gl->BindVertexArray (0); + + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); + gl->BindBuffer (GL_ARRAY_BUFFER, 0); +} + +static void +gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay * + overlay, GstBuffer * video_buffer, guint window_width, guint window_height) +{ + gint comp_x, comp_y; + guint comp_width, comp_height; + GstVideoMeta *meta; + guint width, height; + + float rel_x, rel_y, rel_w, rel_h; + + gfloat window_aspect = 1.0; + gfloat video_aspect = 1.0; + + meta = gst_buffer_get_video_meta (video_buffer); + + gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle, + &comp_x, &comp_y, &comp_width, &comp_height); + + width = window_width; + height = window_height; + + window_aspect = (float) width / (float) height; + video_aspect = (float) meta->width / (float) meta->height; + + /* calculate relative position */ + rel_x = (float) comp_x / (float) width; + rel_y = (float) comp_y / (float) height; + + rel_w = (float) comp_width / (float) width; + rel_h = (float) comp_height / (float) height; + + /* transform to window aspect ratio */ + if (window_aspect <= video_aspect) { + rel_y *= video_aspect / window_aspect; + rel_h *= video_aspect / window_aspect; + } else { + rel_x *= window_aspect / video_aspect; + rel_w *= window_aspect / video_aspect; + } + + /* transform from [0,1] to [-1,1], invert y axis */ + rel_x = rel_x * 2.0 - 1.0; + rel_y = (1.0 - rel_y) * 2.0 - 1.0; + rel_w = rel_w * 2.0; + rel_h = rel_h * 2.0; + + /* initialize position array */ + overlay->positions[0] = rel_x + rel_w; + overlay->positions[1] = rel_y; + overlay->positions[2] = 0.0; + overlay->positions[3] = 1.0; + overlay->positions[4] = rel_x; + overlay->positions[5] = rel_y; + overlay->positions[6] = 0.0; + overlay->positions[7] = 1.0; + overlay->positions[8] = rel_x; + overlay->positions[9] = rel_y - rel_h; + overlay->positions[10] = 0.0; + overlay->positions[11] = 1.0; + overlay->positions[12] = rel_x + rel_w; + overlay->positions[13] = rel_y - rel_h; + overlay->positions[14] = 0.0; + overlay->positions[15] = 1.0; + + gst_gl_context_thread_add (overlay->context, + gst_gl_composition_overlay_free_vertex_buffer, overlay); + + gst_gl_context_thread_add (overlay->context, + gst_gl_composition_overlay_init_vertex_buffer, overlay); + + GST_DEBUG + ("overlay position: (%d,%d) size: %dx%d video size: %dx%d, sink window %dx%d", + comp_x, comp_y, comp_width, comp_height, meta->width, meta->height, + window_width, window_height); +} + +void +gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay, + GstBuffer * buf, guint window_width, guint window_height) +{ + GstMapInfo info; + GstVideoMeta *vmeta; + GstGLMemory *comp_gl_memory = NULL; + gint stride; + GstBuffer *comp_buffer = NULL; + GstVideoMeta *meta; + GstVideoInfo text_info; + gpointer raw_overlay_data; + GstBuffer *overlay_buffer = NULL; + GstVideoFrame gl_frame; + + comp_buffer = + gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle, + GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA); + + vmeta = gst_buffer_get_video_meta (comp_buffer); + + if (gst_video_meta_map (vmeta, 0, &info, &raw_overlay_data, &stride, + GST_MAP_READ)) { + g_assert (raw_overlay_data); + + meta = gst_buffer_get_video_meta (comp_buffer); + + gst_gl_composition_overlay_add_transformation (overlay, buf, window_width, + window_height); + + gst_video_info_init (&text_info); + gst_video_info_set_format (&text_info, meta->format, meta->width, + meta->height); + + comp_gl_memory = + gst_gl_memory_wrapped (overlay->context, &text_info, 0, NULL, + raw_overlay_data, NULL, NULL); + + overlay_buffer = gst_buffer_new (); + gst_buffer_append_memory (overlay_buffer, + gst_memory_ref ((GstMemory *) comp_gl_memory)); + + if (!gst_video_frame_map (&gl_frame, &text_info, overlay_buffer, + GST_MAP_READ | GST_MAP_GL)) { + gst_buffer_unref (overlay_buffer); + gst_video_meta_unmap (vmeta, 0, &info); + GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture"); + return; + } + + gst_buffer_unref (overlay_buffer); + overlay->texture_id = comp_gl_memory->tex_id; + GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id); + gst_video_frame_unmap (&gl_frame); + overlay->gl_memory = comp_gl_memory; + + gst_video_meta_unmap (vmeta, 0, &info); + } +} + +void +gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay, + GstGLShader * shader) +{ + const GstGLFuncs *gl = overlay->context->gl_vtable; + gl->BindVertexArray (overlay->vao); + if (overlay->texture_id != -1) + gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id); + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); +} diff --git a/gst-libs/gst/gl/gstglcompositionoverlay.h b/gst-libs/gst/gl/gstglcompositionoverlay.h new file mode 100644 index 00000000000..a7a7ccb10f1 --- /dev/null +++ b/gst-libs/gst/gl/gstglcompositionoverlay.h @@ -0,0 +1,83 @@ +/* + * GStreamer + * Copyright (C) 2015 Lubosz Sarnecki + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_GL_COMPOSITION_OVERLAY_H__ +#define __GST_GL_COMPOSITION_OVERLAY_H__ + +#include +#include + +#define GST_TYPE_GL_COMPOSITION_OVERLAY (gst_gl_composition_overlay_get_type()) +#define GST_GL_COMPOSITION_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_COMPOSITION_OVERLAY,GstGLCompositionOverlay)) +#define GST_GL_COMPOSITION_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_COMPOSITION_OVERLAY,GstGLCompositionOverlayClass)) +#define GST_IS_GL_COMPOSITION_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_COMPOSITION_OVERLAY)) +#define GST_IS_GL_COMPOSITION_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_COMPOSITION_OVERLAY)) +#define GST_GL_COMPOSITION_OVERLAY_CAST(obj) ((GstGLCompositionOverlay*)(obj)) + +G_BEGIN_DECLS + +GType gst_gl_composition_overlay_get_type (void); + +/** + * GstGLCompositionOverlay + * + * Opaque #GstGLCompositionOverlay object + */ +struct _GstGLCompositionOverlay +{ + /*< private >*/ + GstObject parent; + GstGLContext *context; + + GLuint vao; + GLuint index_buffer; + GLuint position_buffer; + GLuint texcoord_buffer; + GLint position_attrib; + GLint texcoord_attrib; + + GLfloat positions[16]; + + GLuint texture_id; + GstGLMemory *gl_memory; + GstVideoOverlayRectangle *rectangle; +}; + +/** + * GstGLCompositionOverlayClass: + * + */ +struct _GstGLCompositionOverlayClass +{ + GstObjectClass object_class; +}; + +GstGLCompositionOverlay *gst_gl_composition_overlay_new (GstGLContext * context, + GstVideoOverlayRectangle * rectangle, GLint position_attrib, + GLint texcoord_attrib); + +void gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay, + GstBuffer * buf, guint window_width, guint window_height); + +void gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay, + GstGLShader * shader); + +G_END_DECLS +#endif /* __GST_GL_COMPOSITION_OVERLAY_H__ */