glcompositionoverlay: Add class for managing GstVideoOverlayCompositionOverlay buffers

Add a class to store and manage the OpenGL texture,
vertex buffer and GstVideoOverlayRectangle.
Transforms overlay coordinate space to vertex buffer space with aspect ratios in mind.

= Example Pipelines =

Simple pipeline

gst-launch-1.0 videotestsrc ! \
  textoverlay text="Hello World" font-desc="sans bold 30" ! \
  glimagesink

Display 3 static overlays at different positions

gst-launch-1.0 videotestsrc ! \
  textoverlay text="text1" valignment="top" font-desc="sans bold 30" ! \
  textoverlay text="text2" halignment="right" font-desc="sans bold 30" ! \
  textoverlay text="text3" halignment="left" font-desc="sans bold 30" ! \
  glimagesink

Display subtitle file over testsrc

gst-launch-1.0 videotestsrc ! \
  textoverlay name=foo filesrc location=foo.srt ! subparse ! queue ! foo. foo. ! \
  glimagesink

https://bugzilla.gnome.org/show_bug.cgi?id=745107

gst-libs/gst/gl/Makefile.am

@@ -28,7 +28,8 @@ libgstgl_@GST_API_VERSION@_la_SOURCES = \
         gstglutils.c \
         gstglframebuffer.c \
         gstglsyncmeta.c \
-        gstglviewconvert.c
+        gstglviewconvert.c \
+        gstglcompositionoverlay.c
 
 libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl
 libgstgl_@GST_API_VERSION@include_HEADERS = \
@@ -52,6 +53,7 @@ libgstgl_@GST_API_VERSION@include_HEADERS = \
 	gstglframebuffer.h \
 	gstglsyncmeta.h \
 	gstglviewconvert.h \
+	gstglcompositionoverlay.h \
 	gstgl_fwd.h \
 	gl.h
 

gst-libs/gst/gl/gstgl_fwd.h

@@ -78,6 +78,9 @@ typedef struct _GstGLViewConvert GstGLViewConvert;
 typedef struct _GstGLViewConvertClass GstGLViewConvertClass;
 typedef struct _GstGLViewConvertPrivate GstGLViewConvertPrivate;
 
+typedef struct _GstGLCompositionOverlay GstGLCompositionOverlay;
+typedef struct _GstGLCompositionOverlayClass GstGLCompositionOverlayClass;
+
 G_END_DECLS
 
 #endif /* __GST_GL_FWD_H__ */

gst-libs/gst/gl/gstglcompositionoverlay.c

@@ -0,0 +1,328 @@
+/*
+ * GStreamer
+ * Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+
+#include "gl.h"
+#include "gstglcompositionoverlay.h"
+
+
+GST_DEBUG_CATEGORY_STATIC (gst_gl_composition_overlay_debug);
+#define GST_CAT_DEFAULT gst_gl_composition_overlay_debug
+
+
+#define DEBUG_INIT \
+  GST_DEBUG_CATEGORY_INIT (gst_gl_composition_overlay_debug, \
+      "glcompositionoverlay", 0, "compositionoverlay");
+
+G_DEFINE_TYPE_WITH_CODE (GstGLCompositionOverlay, gst_gl_composition_overlay,
+    GST_TYPE_OBJECT, DEBUG_INIT);
+
+static void gst_gl_composition_overlay_finalize (GObject * object);
+static void
+gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
+    overlay, GstBuffer * video_buffer, guint window_width, guint window_height);
+
+static void
+gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass)
+{
+  G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize;
+}
+
+static void
+gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay)
+{
+}
+
+GstGLCompositionOverlay *
+gst_gl_composition_overlay_new (GstGLContext * context,
+    GstVideoOverlayRectangle * rectangle,
+    GLint position_attrib, GLint texcoord_attrib)
+{
+  GstGLCompositionOverlay *overlay =
+      g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL);
+
+  overlay->gl_memory = NULL;
+  overlay->texture_id = -1;
+  overlay->rectangle = rectangle;
+  overlay->context = gst_object_ref (context);
+  overlay->vao = 0;
+  overlay->position_attrib = position_attrib;
+  overlay->texcoord_attrib = texcoord_attrib;
+
+  GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay");
+
+  return overlay;
+}
+
+static void
+gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context,
+    gpointer overlay_pointer)
+{
+  const GstGLFuncs *gl = context->gl_vtable;
+  GstGLCompositionOverlay *overlay =
+      (GstGLCompositionOverlay *) overlay_pointer;
+  if (overlay->vao) {
+    gl->DeleteVertexArrays (1, &overlay->vao);
+    overlay->vao = 0;
+  }
+
+  if (overlay->position_buffer) {
+    gl->DeleteBuffers (1, &overlay->position_buffer);
+    overlay->position_buffer = 0;
+  }
+
+  if (overlay->texcoord_buffer) {
+    gl->DeleteBuffers (1, &overlay->position_buffer);
+    overlay->position_buffer = 0;
+  }
+
+  if (overlay->index_buffer) {
+    gl->DeleteBuffers (1, &overlay->index_buffer);
+    overlay->index_buffer = 0;
+  }
+}
+
+static void
+gst_gl_composition_overlay_finalize (GObject * object)
+{
+  GstGLCompositionOverlay *overlay;
+
+  overlay = GST_GL_COMPOSITION_OVERLAY (object);
+
+  if (overlay->gl_memory)
+    gst_memory_unref ((GstMemory *) overlay->gl_memory);
+  overlay->gl_memory = NULL;
+  if (overlay->context) {
+    gst_gl_context_thread_add (overlay->context,
+        gst_gl_composition_overlay_free_vertex_buffer, overlay);
+    gst_object_unref (overlay->context);
+  }
+
+  G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object);
+}
+
+static void
+gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context,
+    gpointer overlay_pointer)
+{
+  const GstGLFuncs *gl = context->gl_vtable;
+  GstGLCompositionOverlay *overlay =
+      (GstGLCompositionOverlay *) overlay_pointer;
+
+  /* *INDENT-OFF* */
+  static const GLfloat texcoords[] = {
+      1.0f, 0.0f,
+      0.0f, 0.0f,
+      0.0f, 1.0f,
+      1.0f, 1.0f
+  };
+
+  static const GLushort indices[] = {
+    0, 1, 2, 0, 2, 3
+  };
+  /* *INDENT-ON* */
+
+  gl->GenVertexArrays (1, &overlay->vao);
+  gl->BindVertexArray (overlay->vao);
+
+  gl->GenBuffers (1, &overlay->position_buffer);
+  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
+  gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions,
+      GL_STATIC_DRAW);
+
+  /* Load the vertex position */
+  gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
+      4 * sizeof (GLfloat), NULL);
+
+  gl->GenBuffers (1, &overlay->texcoord_buffer);
+  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
+  gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords,
+      GL_STATIC_DRAW);
+
+  /* Load the texture coordinate */
+  gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
+      2 * sizeof (GLfloat), NULL);
+
+  gl->GenBuffers (1, &overlay->index_buffer);
+  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
+  gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
+      GL_STATIC_DRAW);
+
+  gl->EnableVertexAttribArray (overlay->position_attrib);
+  gl->EnableVertexAttribArray (overlay->texcoord_attrib);
+
+  gl->BindVertexArray (0);
+
+  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
+  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
+}
+
+static void
+gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
+    overlay, GstBuffer * video_buffer, guint window_width, guint window_height)
+{
+  gint comp_x, comp_y;
+  guint comp_width, comp_height;
+  GstVideoMeta *meta;
+  guint width, height;
+
+  float rel_x, rel_y, rel_w, rel_h;
+
+  gfloat window_aspect = 1.0;
+  gfloat video_aspect = 1.0;
+
+  meta = gst_buffer_get_video_meta (video_buffer);
+
+  gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
+      &comp_x, &comp_y, &comp_width, &comp_height);
+
+  width = window_width;
+  height = window_height;
+
+  window_aspect = (float) width / (float) height;
+  video_aspect = (float) meta->width / (float) meta->height;
+
+  /* calculate relative position */
+  rel_x = (float) comp_x / (float) width;
+  rel_y = (float) comp_y / (float) height;
+
+  rel_w = (float) comp_width / (float) width;
+  rel_h = (float) comp_height / (float) height;
+
+  /* transform to window aspect ratio */
+  if (window_aspect <= video_aspect) {
+    rel_y *= video_aspect / window_aspect;
+    rel_h *= video_aspect / window_aspect;
+  } else {
+    rel_x *= window_aspect / video_aspect;
+    rel_w *= window_aspect / video_aspect;
+  }
+
+  /* transform from [0,1] to [-1,1], invert y axis */
+  rel_x = rel_x * 2.0 - 1.0;
+  rel_y = (1.0 - rel_y) * 2.0 - 1.0;
+  rel_w = rel_w * 2.0;
+  rel_h = rel_h * 2.0;
+
+  /* initialize position array */
+  overlay->positions[0] = rel_x + rel_w;
+  overlay->positions[1] = rel_y;
+  overlay->positions[2] = 0.0;
+  overlay->positions[3] = 1.0;
+  overlay->positions[4] = rel_x;
+  overlay->positions[5] = rel_y;
+  overlay->positions[6] = 0.0;
+  overlay->positions[7] = 1.0;
+  overlay->positions[8] = rel_x;
+  overlay->positions[9] = rel_y - rel_h;
+  overlay->positions[10] = 0.0;
+  overlay->positions[11] = 1.0;
+  overlay->positions[12] = rel_x + rel_w;
+  overlay->positions[13] = rel_y - rel_h;
+  overlay->positions[14] = 0.0;
+  overlay->positions[15] = 1.0;
+
+  gst_gl_context_thread_add (overlay->context,
+      gst_gl_composition_overlay_free_vertex_buffer, overlay);
+
+  gst_gl_context_thread_add (overlay->context,
+      gst_gl_composition_overlay_init_vertex_buffer, overlay);
+
+  GST_DEBUG
+      ("overlay position: (%d,%d) size: %dx%d video size: %dx%d, sink window %dx%d",
+      comp_x, comp_y, comp_width, comp_height, meta->width, meta->height,
+      window_width, window_height);
+}
+
+void
+gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay,
+    GstBuffer * buf, guint window_width, guint window_height)
+{
+  GstMapInfo info;
+  GstVideoMeta *vmeta;
+  GstGLMemory *comp_gl_memory = NULL;
+  gint stride;
+  GstBuffer *comp_buffer = NULL;
+  GstVideoMeta *meta;
+  GstVideoInfo text_info;
+  gpointer raw_overlay_data;
+  GstBuffer *overlay_buffer = NULL;
+  GstVideoFrame gl_frame;
+
+  comp_buffer =
+      gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle,
+      GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
+
+  vmeta = gst_buffer_get_video_meta (comp_buffer);
+
+  if (gst_video_meta_map (vmeta, 0, &info, &raw_overlay_data, &stride,
+          GST_MAP_READ)) {
+    g_assert (raw_overlay_data);
+
+    meta = gst_buffer_get_video_meta (comp_buffer);
+
+    gst_gl_composition_overlay_add_transformation (overlay, buf, window_width,
+        window_height);
+
+    gst_video_info_init (&text_info);
+    gst_video_info_set_format (&text_info, meta->format, meta->width,
+        meta->height);
+
+    comp_gl_memory =
+        gst_gl_memory_wrapped (overlay->context, &text_info, 0, NULL,
+        raw_overlay_data, NULL, NULL);
+
+    overlay_buffer = gst_buffer_new ();
+    gst_buffer_append_memory (overlay_buffer,
+        gst_memory_ref ((GstMemory *) comp_gl_memory));
+
+    if (!gst_video_frame_map (&gl_frame, &text_info, overlay_buffer,
+            GST_MAP_READ | GST_MAP_GL)) {
+      gst_buffer_unref (overlay_buffer);
+      gst_video_meta_unmap (vmeta, 0, &info);
+      GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture");
+      return;
+    }
+
+    gst_buffer_unref (overlay_buffer);
+    overlay->texture_id = comp_gl_memory->tex_id;
+    GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id);
+    gst_video_frame_unmap (&gl_frame);
+    overlay->gl_memory = comp_gl_memory;
+
+    gst_video_meta_unmap (vmeta, 0, &info);
+  }
+}
+
+void
+gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay,
+    GstGLShader * shader)
+{
+  const GstGLFuncs *gl = overlay->context->gl_vtable;
+  gl->BindVertexArray (overlay->vao);
+  if (overlay->texture_id != -1)
+    gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id);
+  gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
+}

gst-libs/gst/gl/gstglcompositionoverlay.h

@@ -0,0 +1,83 @@
+/*
+ * GStreamer
+ * Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_GL_COMPOSITION_OVERLAY_H__
+#define __GST_GL_COMPOSITION_OVERLAY_H__
+
+#include <gst/video/video.h>
+#include <gst/gl/gstgl_fwd.h>
+
+#define GST_TYPE_GL_COMPOSITION_OVERLAY (gst_gl_composition_overlay_get_type())
+#define GST_GL_COMPOSITION_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_COMPOSITION_OVERLAY,GstGLCompositionOverlay))
+#define GST_GL_COMPOSITION_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_COMPOSITION_OVERLAY,GstGLCompositionOverlayClass))
+#define GST_IS_GL_COMPOSITION_OVERLAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_COMPOSITION_OVERLAY))
+#define GST_IS_GL_COMPOSITION_OVERLAY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_COMPOSITION_OVERLAY))
+#define GST_GL_COMPOSITION_OVERLAY_CAST(obj) ((GstGLCompositionOverlay*)(obj))
+
+G_BEGIN_DECLS
+
+GType gst_gl_composition_overlay_get_type (void);
+
+/**
+ * GstGLCompositionOverlay
+ *
+ * Opaque #GstGLCompositionOverlay object
+ */
+struct _GstGLCompositionOverlay
+{
+  /*< private >*/
+  GstObject parent;
+  GstGLContext *context;
+
+  GLuint vao;
+  GLuint index_buffer;
+  GLuint position_buffer;
+  GLuint texcoord_buffer;
+  GLint  position_attrib;
+  GLint  texcoord_attrib;
+
+  GLfloat positions[16];
+
+  GLuint texture_id;
+  GstGLMemory *gl_memory;
+  GstVideoOverlayRectangle *rectangle;
+};
+
+/**
+ * GstGLCompositionOverlayClass:
+ *
+ */
+struct _GstGLCompositionOverlayClass
+{
+  GstObjectClass object_class;
+};
+
+GstGLCompositionOverlay *gst_gl_composition_overlay_new (GstGLContext * context,
+    GstVideoOverlayRectangle * rectangle, GLint position_attrib,
+    GLint texcoord_attrib);
+
+void gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay,
+    GstBuffer * buf, guint window_width, guint window_height);
+
+void gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay,
+    GstGLShader * shader);
+
+G_END_DECLS
+#endif /* __GST_GL_COMPOSITION_OVERLAY_H__ */