gstreamer/gst-libs/gst/gl/gstgloverlaycompositor.c

641 lines
19 KiB
C
Raw Normal View History

/*
* GStreamer
* Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:gstgloverlaycompositor
* @short_description: Composite multiple overlays using OpenGL
* @see_also: #GstGLMemory, #GstGLContext
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdio.h>
#include "gl.h"
#include "gstgloverlaycompositor.h"
GST_DEBUG_CATEGORY_STATIC (gst_gl_overlay_compositor_debug);
#define GST_CAT_DEFAULT gst_gl_overlay_compositor_debug
/*****************************************************************************
* GstGLCompositionOverlay object is internally used by GstGLOverlayCompositor
*****************************************************************************/
#define GST_TYPE_GL_COMPOSITION_OVERLAY (gst_gl_composition_overlay_get_type())
#define GST_GL_COMPOSITION_OVERLAY(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_COMPOSITION_OVERLAY,\
GstGLCompositionOverlay))
typedef struct _GstGLCompositionOverlay GstGLCompositionOverlay;
typedef struct _GstGLCompositionOverlayClass GstGLCompositionOverlayClass;
static GType gst_gl_composition_overlay_get_type (void);
/* *INDENT-OFF* */
const gchar *fragment_shader =
"#ifdef GL_ES\n"
"precision mediump float;\n"
"#endif\n"
"varying vec2 v_texcoord;\n"
"uniform sampler2D tex;\n"
"void main(void)\n"
"{\n"
" vec4 t = texture2D(tex, v_texcoord);\n"
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
" gl_FragColor = t.bgra;\n"
#else
" gl_FragColor = t.gbar;\n"
#endif
"}";
/* *INDENT-ON* */
struct _GstGLCompositionOverlay
{
GstObject parent;
GstGLContext *context;
GLuint vao;
GLuint index_buffer;
GLuint position_buffer;
GLuint texcoord_buffer;
GLint position_attrib;
GLint texcoord_attrib;
GLfloat positions[16];
GLuint texture_id;
GstGLMemory *gl_memory;
GstVideoOverlayRectangle *rectangle;
};
struct _GstGLCompositionOverlayClass
{
GstObjectClass object_class;
};
G_DEFINE_TYPE (GstGLCompositionOverlay, gst_gl_composition_overlay,
GST_TYPE_OBJECT);
static void
gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context,
gpointer overlay_pointer)
{
const GstGLFuncs *gl = context->gl_vtable;
GstGLCompositionOverlay *overlay =
(GstGLCompositionOverlay *) overlay_pointer;
/* *INDENT-OFF* */
static const GLfloat texcoords[] = {
1.0f, 0.0f,
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f
};
static const GLushort indices[] = {
0, 1, 2, 0, 2, 3
};
/* *INDENT-ON* */
if (gl->GenVertexArrays) {
gl->GenVertexArrays (1, &overlay->vao);
gl->BindVertexArray (overlay->vao);
}
gl->GenBuffers (1, &overlay->position_buffer);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions,
GL_STATIC_DRAW);
/* Load the vertex position */
gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
4 * sizeof (GLfloat), NULL);
gl->GenBuffers (1, &overlay->texcoord_buffer);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords,
GL_STATIC_DRAW);
/* Load the texture coordinate */
gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
2 * sizeof (GLfloat), NULL);
gl->GenBuffers (1, &overlay->index_buffer);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
GL_STATIC_DRAW);
gl->EnableVertexAttribArray (overlay->position_attrib);
gl->EnableVertexAttribArray (overlay->texcoord_attrib);
if (gl->GenVertexArrays) {
gl->BindVertexArray (0);
}
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
}
static void
gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context,
gpointer overlay_pointer)
{
const GstGLFuncs *gl = context->gl_vtable;
GstGLCompositionOverlay *overlay =
(GstGLCompositionOverlay *) overlay_pointer;
if (overlay->vao) {
gl->DeleteVertexArrays (1, &overlay->vao);
overlay->vao = 0;
}
if (overlay->position_buffer) {
gl->DeleteBuffers (1, &overlay->position_buffer);
overlay->position_buffer = 0;
}
if (overlay->texcoord_buffer) {
gl->DeleteBuffers (1, &overlay->position_buffer);
overlay->position_buffer = 0;
}
if (overlay->index_buffer) {
gl->DeleteBuffers (1, &overlay->index_buffer);
overlay->index_buffer = 0;
}
}
static void
gst_gl_composition_overlay_bind_vertex_buffer (GstGLCompositionOverlay *
overlay)
{
const GstGLFuncs *gl = overlay->context->gl_vtable;
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
4 * sizeof (GLfloat), NULL);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
2 * sizeof (GLfloat), NULL);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
gl->EnableVertexAttribArray (overlay->position_attrib);
gl->EnableVertexAttribArray (overlay->texcoord_attrib);
}
static void
gst_gl_composition_overlay_finalize (GObject * object)
{
GstGLCompositionOverlay *overlay;
overlay = GST_GL_COMPOSITION_OVERLAY (object);
if (overlay->gl_memory)
gst_memory_unref ((GstMemory *) overlay->gl_memory);
if (overlay->context) {
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_free_vertex_buffer, overlay);
gst_object_unref (overlay->context);
}
G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object);
}
static void
gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass)
{
G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize;
}
static void
gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay)
{
}
static void
gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
overlay, GstBuffer * video_buffer)
{
gint comp_x, comp_y;
guint comp_width, comp_height;
GstVideoMeta *meta;
guint width, height;
float rel_x, rel_y, rel_w, rel_h;
meta = gst_buffer_get_video_meta (video_buffer);
gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
&comp_x, &comp_y, &comp_width, &comp_height);
width = meta->width;
height = meta->height;
/* calculate relative position */
rel_x = (float) comp_x / (float) width;
rel_y = (float) comp_y / (float) height;
rel_w = (float) comp_width / (float) width;
rel_h = (float) comp_height / (float) height;
/* transform from [0,1] to [-1,1], invert y axis */
rel_x = rel_x * 2.0 - 1.0;
rel_y = (1.0 - rel_y) * 2.0 - 1.0;
rel_w = rel_w * 2.0;
rel_h = rel_h * 2.0;
/* initialize position array */
overlay->positions[0] = rel_x + rel_w;
overlay->positions[1] = rel_y;
overlay->positions[2] = 0.0;
overlay->positions[3] = 1.0;
overlay->positions[4] = rel_x;
overlay->positions[5] = rel_y;
overlay->positions[6] = 0.0;
overlay->positions[7] = 1.0;
overlay->positions[8] = rel_x;
overlay->positions[9] = rel_y - rel_h;
overlay->positions[10] = 0.0;
overlay->positions[11] = 1.0;
overlay->positions[12] = rel_x + rel_w;
overlay->positions[13] = rel_y - rel_h;
overlay->positions[14] = 0.0;
overlay->positions[15] = 1.0;
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_free_vertex_buffer, overlay);
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_init_vertex_buffer, overlay);
GST_DEBUG
("overlay position: (%d,%d) size: %dx%d video size: %dx%d",
comp_x, comp_y, comp_width, comp_height, meta->width, meta->height);
}
/* helper object API functions */
static GstGLCompositionOverlay *
gst_gl_composition_overlay_new (GstGLContext * context,
GstVideoOverlayRectangle * rectangle,
GLint position_attrib, GLint texcoord_attrib)
{
GstGLCompositionOverlay *overlay =
g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL);
overlay->gl_memory = NULL;
overlay->texture_id = -1;
overlay->rectangle = rectangle;
overlay->context = gst_object_ref (context);
overlay->vao = 0;
overlay->position_attrib = position_attrib;
overlay->texcoord_attrib = texcoord_attrib;
GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay");
return overlay;
}
static void
_video_frame_unmap_and_free (gpointer user_data)
{
GstVideoFrame *frame = user_data;
gst_video_frame_unmap (frame);
g_slice_free (GstVideoFrame, frame);
}
static void
gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay,
GstBuffer * buf)
{
GstGLMemory *comp_gl_memory = NULL;
GstBuffer *comp_buffer = NULL;
GstBuffer *overlay_buffer = NULL;
GstVideoInfo vinfo;
GstVideoMeta *vmeta;
GstVideoFrame *comp_frame;
GstVideoFrame gl_frame;
comp_buffer =
gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle,
GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
comp_frame = g_slice_new (GstVideoFrame);
vmeta = gst_buffer_get_video_meta (comp_buffer);
gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width,
vmeta->height);
vinfo.stride[0] = vmeta->stride[0];
if (gst_video_frame_map (comp_frame, &vinfo, comp_buffer, GST_MAP_READ)) {
GstGLVideoAllocationParams *params;
GstGLBaseMemoryAllocator *mem_allocator;
GstAllocator *allocator;
allocator =
GST_ALLOCATOR (gst_gl_memory_allocator_get_default (overlay->context));
mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
gst_gl_composition_overlay_add_transformation (overlay, buf);
params = gst_gl_video_allocation_params_new_wrapped_data (overlay->context,
NULL, &comp_frame->info, 0, NULL, GST_GL_TEXTURE_TARGET_2D,
GST_GL_RGBA, comp_frame->data[0], comp_frame,
_video_frame_unmap_and_free);
comp_gl_memory =
(GstGLMemory *) gst_gl_base_memory_alloc (mem_allocator,
(GstGLAllocationParams *) params);
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
gst_object_unref (allocator);
overlay_buffer = gst_buffer_new ();
gst_buffer_append_memory (overlay_buffer, (GstMemory *) comp_gl_memory);
if (!gst_video_frame_map (&gl_frame, &comp_frame->info, overlay_buffer,
GST_MAP_READ | GST_MAP_GL)) {
gst_buffer_unref (overlay_buffer);
_video_frame_unmap_and_free (comp_frame);
GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture");
return;
}
gst_memory_ref ((GstMemory *) comp_gl_memory);
overlay->gl_memory = comp_gl_memory;
overlay->texture_id = comp_gl_memory->tex_id;
gst_buffer_unref (overlay_buffer);
gst_video_frame_unmap (&gl_frame);
GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id);
} else {
g_slice_free (GstVideoFrame, comp_frame);
}
}
static void
gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay,
GstGLShader * shader)
{
const GstGLFuncs *gl = overlay->context->gl_vtable;
if (gl->GenVertexArrays)
gl->BindVertexArray (overlay->vao);
else
gst_gl_composition_overlay_bind_vertex_buffer (overlay);
if (overlay->texture_id != -1)
gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id);
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
}
/********************************************************************
* GstGLOverlayCompositor object, the public helper object to render
* GstVideoCompositionOverlayMeta
********************************************************************/
#define DEBUG_INIT \
GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_debug, \
"gloverlaycompositor", 0, "overlaycompositor");
G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositor, gst_gl_overlay_compositor,
GST_TYPE_OBJECT, DEBUG_INIT);
static void gst_gl_overlay_compositor_finalize (GObject * object);
static gboolean _is_rectangle_in_overlays (GList * overlays,
GstVideoOverlayRectangle * rectangle);
static gboolean _is_overlay_in_rectangles (GstVideoOverlayComposition *
composition, GstGLCompositionOverlay * overlay);
static void
gst_gl_overlay_compositor_class_init (GstGLOverlayCompositorClass * klass)
{
G_OBJECT_CLASS (klass)->finalize = gst_gl_overlay_compositor_finalize;
}
static void
gst_gl_overlay_compositor_init (GstGLOverlayCompositor * compositor)
{
}
static void
gst_gl_overlay_compositor_init_gl (GstGLContext * context,
gpointer compositor_pointer)
{
GstGLOverlayCompositor *compositor =
(GstGLOverlayCompositor *) compositor_pointer;
GError *error = NULL;
if (!(compositor->shader =
gst_gl_shader_new_link_with_stages (context, &error,
gst_glsl_stage_new_default_vertex (context),
gst_glsl_stage_new_with_string (context, GL_FRAGMENT_SHADER,
GST_GLSL_VERSION_NONE,
GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY,
fragment_shader), NULL))) {
GST_ERROR_OBJECT (compositor, "could not initialize shader: %s",
error->message);
return;
}
compositor->position_attrib =
gst_gl_shader_get_attribute_location (compositor->shader, "a_position");
compositor->texcoord_attrib =
gst_gl_shader_get_attribute_location (compositor->shader, "a_texcoord");
}
GstGLOverlayCompositor *
gst_gl_overlay_compositor_new (GstGLContext * context)
{
GstGLOverlayCompositor *compositor =
g_object_new (GST_TYPE_GL_OVERLAY_COMPOSITOR, NULL);
compositor->context = gst_object_ref (context);
gst_gl_context_thread_add (compositor->context,
gst_gl_overlay_compositor_init_gl, compositor);
GST_DEBUG_OBJECT (compositor, "Created new GstGLOverlayCompositor");
return compositor;
}
static void
gst_gl_overlay_compositor_finalize (GObject * object)
{
GstGLOverlayCompositor *compositor;
compositor = GST_GL_OVERLAY_COMPOSITOR (object);
gst_gl_overlay_compositor_free_overlays (compositor);
if (compositor->context)
gst_object_unref (compositor->context);
if (compositor->shader) {
gst_object_unref (compositor->shader);
compositor->shader = NULL;
}
G_OBJECT_CLASS (gst_gl_overlay_compositor_parent_class)->finalize (object);
}
static gboolean
_is_rectangle_in_overlays (GList * overlays,
GstVideoOverlayRectangle * rectangle)
{
GList *l;
for (l = overlays; l != NULL; l = l->next) {
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
if (overlay->rectangle == rectangle)
return TRUE;
}
return FALSE;
}
static gboolean
_is_overlay_in_rectangles (GstVideoOverlayComposition * composition,
GstGLCompositionOverlay * overlay)
{
guint i;
for (i = 0; i < gst_video_overlay_composition_n_rectangles (composition); i++) {
GstVideoOverlayRectangle *rectangle =
gst_video_overlay_composition_get_rectangle (composition, i);
if (overlay->rectangle == rectangle)
return TRUE;
}
return FALSE;
}
void
gst_gl_overlay_compositor_free_overlays (GstGLOverlayCompositor * compositor)
{
GList *l = compositor->overlays;
while (l != NULL) {
GList *next = l->next;
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
compositor->overlays = g_list_delete_link (compositor->overlays, l);
gst_object_unref (overlay);
l = next;
}
g_list_free (compositor->overlays);
compositor->overlays = NULL;
}
void
gst_gl_overlay_compositor_upload_overlays (GstGLOverlayCompositor * compositor,
GstBuffer * buf)
{
GstVideoOverlayCompositionMeta *composition_meta;
composition_meta = gst_buffer_get_video_overlay_composition_meta (buf);
if (composition_meta) {
GstVideoOverlayComposition *composition = NULL;
guint num_overlays, i;
GList *l = compositor->overlays;
GST_DEBUG ("GstVideoOverlayCompositionMeta found.");
composition = composition_meta->overlay;
num_overlays = gst_video_overlay_composition_n_rectangles (composition);
/* add new overlays to list */
for (i = 0; i < num_overlays; i++) {
GstVideoOverlayRectangle *rectangle =
gst_video_overlay_composition_get_rectangle (composition, i);
if (!_is_rectangle_in_overlays (compositor->overlays, rectangle)) {
GstGLCompositionOverlay *overlay =
gst_gl_composition_overlay_new (compositor->context, rectangle,
compositor->position_attrib, compositor->texcoord_attrib);
gst_gl_composition_overlay_upload (overlay, buf);
compositor->overlays = g_list_append (compositor->overlays, overlay);
}
}
/* remove old overlays from list */
while (l != NULL) {
GList *next = l->next;
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
if (!_is_overlay_in_rectangles (composition, overlay)) {
compositor->overlays = g_list_delete_link (compositor->overlays, l);
gst_object_unref (overlay);
}
l = next;
}
} else {
gst_gl_overlay_compositor_free_overlays (compositor);
}
}
void
gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor)
{
const GstGLFuncs *gl = compositor->context->gl_vtable;
if (compositor->overlays != NULL) {
GList *l;
gl->Enable (GL_BLEND);
gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
gst_gl_shader_use (compositor->shader);
gl->ActiveTexture (GL_TEXTURE0);
gst_gl_shader_set_uniform_1i (compositor->shader, "tex", 0);
for (l = compositor->overlays; l != NULL; l = l->next) {
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
gst_gl_composition_overlay_draw (overlay, compositor->shader);
}
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->Disable (GL_BLEND);
}
}
GstCaps *
gst_gl_overlay_compositor_add_caps (GstCaps * caps)
{
GstCaps *composition_caps;
int i;
composition_caps = gst_caps_copy (caps);
for (i = 0; i < gst_caps_get_size (composition_caps); i++) {
GstCapsFeatures *f = gst_caps_get_features (composition_caps, i);
gst_caps_features_add (f,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
}
caps = gst_caps_merge (composition_caps, caps);
return caps;
}