gstreamer/gst-libs/gst/gl/gstgloverlaycompositor.c
Alessandro Decina 28dbe4fffc glmemory: add gst_gl_memory_allocator_get_default
Add gst_gl_memory_allocator_get_default to get the default allocator based on
the opengl version. Allows us to stop hardcoding the PBO allocator which isn't
supported on gles2.

Fixes GL upload on iOS9 among other things.
2017-12-09 19:32:13 +00:00

633 lines
18 KiB
C

/*
* GStreamer
* Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <stdio.h>
#include "gl.h"
#include "gstgloverlaycompositor.h"
GST_DEBUG_CATEGORY_STATIC (gst_gl_overlay_compositor_debug);
#define GST_CAT_DEFAULT gst_gl_overlay_compositor_debug
/*****************************************************************************
* GstGLCompositionOverlay object is internally used by GstGLOverlayCompositor
*****************************************************************************/
#define GST_TYPE_GL_COMPOSITION_OVERLAY (gst_gl_composition_overlay_get_type())
#define GST_GL_COMPOSITION_OVERLAY(obj) \
(G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_COMPOSITION_OVERLAY,\
GstGLCompositionOverlay))
typedef struct _GstGLCompositionOverlay GstGLCompositionOverlay;
typedef struct _GstGLCompositionOverlayClass GstGLCompositionOverlayClass;
static GType gst_gl_composition_overlay_get_type (void);
/* *INDENT-OFF* */
const gchar *fragment_shader =
"#ifdef GL_ES\n"
"precision mediump float;\n"
"#endif\n"
"varying vec2 v_texcoord;\n"
"uniform sampler2D tex;\n"
"void main(void)\n"
"{\n"
" vec4 t = texture2D(tex, v_texcoord);\n"
#if G_BYTE_ORDER == G_LITTLE_ENDIAN
" gl_FragColor = t.bgra;\n"
#else
" gl_FragColor = t.gbar;\n"
#endif
"}";
/* *INDENT-ON* */
struct _GstGLCompositionOverlay
{
GstObject parent;
GstGLContext *context;
GLuint vao;
GLuint index_buffer;
GLuint position_buffer;
GLuint texcoord_buffer;
GLint position_attrib;
GLint texcoord_attrib;
GLfloat positions[16];
GLuint texture_id;
GstGLMemory *gl_memory;
GstVideoOverlayRectangle *rectangle;
};
struct _GstGLCompositionOverlayClass
{
GstObjectClass object_class;
};
G_DEFINE_TYPE (GstGLCompositionOverlay, gst_gl_composition_overlay,
GST_TYPE_OBJECT);
static void
gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context,
gpointer overlay_pointer)
{
const GstGLFuncs *gl = context->gl_vtable;
GstGLCompositionOverlay *overlay =
(GstGLCompositionOverlay *) overlay_pointer;
/* *INDENT-OFF* */
static const GLfloat texcoords[] = {
1.0f, 0.0f,
0.0f, 0.0f,
0.0f, 1.0f,
1.0f, 1.0f
};
static const GLushort indices[] = {
0, 1, 2, 0, 2, 3
};
/* *INDENT-ON* */
if (gl->GenVertexArrays) {
gl->GenVertexArrays (1, &overlay->vao);
gl->BindVertexArray (overlay->vao);
}
gl->GenBuffers (1, &overlay->position_buffer);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions,
GL_STATIC_DRAW);
/* Load the vertex position */
gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
4 * sizeof (GLfloat), NULL);
gl->GenBuffers (1, &overlay->texcoord_buffer);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords,
GL_STATIC_DRAW);
/* Load the texture coordinate */
gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
2 * sizeof (GLfloat), NULL);
gl->GenBuffers (1, &overlay->index_buffer);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
GL_STATIC_DRAW);
gl->EnableVertexAttribArray (overlay->position_attrib);
gl->EnableVertexAttribArray (overlay->texcoord_attrib);
if (gl->GenVertexArrays) {
gl->BindVertexArray (0);
}
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
}
static void
gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context,
gpointer overlay_pointer)
{
const GstGLFuncs *gl = context->gl_vtable;
GstGLCompositionOverlay *overlay =
(GstGLCompositionOverlay *) overlay_pointer;
if (overlay->vao) {
gl->DeleteVertexArrays (1, &overlay->vao);
overlay->vao = 0;
}
if (overlay->position_buffer) {
gl->DeleteBuffers (1, &overlay->position_buffer);
overlay->position_buffer = 0;
}
if (overlay->texcoord_buffer) {
gl->DeleteBuffers (1, &overlay->position_buffer);
overlay->position_buffer = 0;
}
if (overlay->index_buffer) {
gl->DeleteBuffers (1, &overlay->index_buffer);
overlay->index_buffer = 0;
}
}
static void
gst_gl_composition_overlay_bind_vertex_buffer (GstGLCompositionOverlay *
overlay)
{
const GstGLFuncs *gl = overlay->context->gl_vtable;
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
4 * sizeof (GLfloat), NULL);
gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
2 * sizeof (GLfloat), NULL);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
gl->EnableVertexAttribArray (overlay->position_attrib);
gl->EnableVertexAttribArray (overlay->texcoord_attrib);
}
static void
gst_gl_composition_overlay_finalize (GObject * object)
{
GstGLCompositionOverlay *overlay;
overlay = GST_GL_COMPOSITION_OVERLAY (object);
if (overlay->gl_memory)
gst_memory_unref ((GstMemory *) overlay->gl_memory);
if (overlay->context) {
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_free_vertex_buffer, overlay);
gst_object_unref (overlay->context);
}
G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object);
}
static void
gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass)
{
G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize;
}
static void
gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay)
{
}
static void
gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
overlay, GstBuffer * video_buffer)
{
gint comp_x, comp_y;
guint comp_width, comp_height;
GstVideoMeta *meta;
guint width, height;
float rel_x, rel_y, rel_w, rel_h;
meta = gst_buffer_get_video_meta (video_buffer);
gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
&comp_x, &comp_y, &comp_width, &comp_height);
width = meta->width;
height = meta->height;
/* calculate relative position */
rel_x = (float) comp_x / (float) width;
rel_y = (float) comp_y / (float) height;
rel_w = (float) comp_width / (float) width;
rel_h = (float) comp_height / (float) height;
/* transform from [0,1] to [-1,1], invert y axis */
rel_x = rel_x * 2.0 - 1.0;
rel_y = (1.0 - rel_y) * 2.0 - 1.0;
rel_w = rel_w * 2.0;
rel_h = rel_h * 2.0;
/* initialize position array */
overlay->positions[0] = rel_x + rel_w;
overlay->positions[1] = rel_y;
overlay->positions[2] = 0.0;
overlay->positions[3] = 1.0;
overlay->positions[4] = rel_x;
overlay->positions[5] = rel_y;
overlay->positions[6] = 0.0;
overlay->positions[7] = 1.0;
overlay->positions[8] = rel_x;
overlay->positions[9] = rel_y - rel_h;
overlay->positions[10] = 0.0;
overlay->positions[11] = 1.0;
overlay->positions[12] = rel_x + rel_w;
overlay->positions[13] = rel_y - rel_h;
overlay->positions[14] = 0.0;
overlay->positions[15] = 1.0;
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_free_vertex_buffer, overlay);
gst_gl_context_thread_add (overlay->context,
gst_gl_composition_overlay_init_vertex_buffer, overlay);
GST_DEBUG
("overlay position: (%d,%d) size: %dx%d video size: %dx%d",
comp_x, comp_y, comp_width, comp_height, meta->width, meta->height);
}
/* helper object API functions */
static GstGLCompositionOverlay *
gst_gl_composition_overlay_new (GstGLContext * context,
GstVideoOverlayRectangle * rectangle,
GLint position_attrib, GLint texcoord_attrib)
{
GstGLCompositionOverlay *overlay =
g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL);
overlay->gl_memory = NULL;
overlay->texture_id = -1;
overlay->rectangle = rectangle;
overlay->context = gst_object_ref (context);
overlay->vao = 0;
overlay->position_attrib = position_attrib;
overlay->texcoord_attrib = texcoord_attrib;
GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay");
return overlay;
}
static void
_video_frame_unmap_and_free (gpointer user_data)
{
GstVideoFrame *frame = user_data;
gst_video_frame_unmap (frame);
g_slice_free (GstVideoFrame, frame);
}
static void
gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay,
GstBuffer * buf)
{
GstGLMemory *comp_gl_memory = NULL;
GstBuffer *comp_buffer = NULL;
GstBuffer *overlay_buffer = NULL;
GstVideoInfo vinfo;
GstVideoMeta *vmeta;
GstVideoFrame *comp_frame;
GstVideoFrame gl_frame;
comp_buffer =
gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle,
GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);
comp_frame = g_slice_new (GstVideoFrame);
vmeta = gst_buffer_get_video_meta (comp_buffer);
gst_video_info_set_format (&vinfo, vmeta->format, vmeta->width,
vmeta->height);
vinfo.stride[0] = vmeta->stride[0];
if (gst_video_frame_map (comp_frame, &vinfo, comp_buffer, GST_MAP_READ)) {
GstGLVideoAllocationParams *params;
GstGLBaseMemoryAllocator *mem_allocator;
GstAllocator *allocator;
allocator =
GST_ALLOCATOR (gst_gl_memory_allocator_get_default (overlay->context));
mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
gst_gl_composition_overlay_add_transformation (overlay, buf);
params = gst_gl_video_allocation_params_new_wrapped_data (overlay->context,
NULL, &comp_frame->info, 0, NULL, GST_GL_TEXTURE_TARGET_2D,
comp_frame->data[0], comp_frame, _video_frame_unmap_and_free);
comp_gl_memory =
(GstGLMemory *) gst_gl_base_memory_alloc (mem_allocator,
(GstGLAllocationParams *) params);
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
gst_object_unref (allocator);
overlay_buffer = gst_buffer_new ();
gst_buffer_append_memory (overlay_buffer, (GstMemory *) comp_gl_memory);
if (!gst_video_frame_map (&gl_frame, &comp_frame->info, overlay_buffer,
GST_MAP_READ | GST_MAP_GL)) {
gst_buffer_unref (overlay_buffer);
_video_frame_unmap_and_free (comp_frame);
GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture");
return;
}
gst_memory_ref ((GstMemory *) comp_gl_memory);
overlay->gl_memory = comp_gl_memory;
overlay->texture_id = comp_gl_memory->tex_id;
gst_buffer_unref (overlay_buffer);
gst_video_frame_unmap (&gl_frame);
GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id);
} else {
g_slice_free (GstVideoFrame, comp_frame);
}
}
static void
gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay,
GstGLShader * shader)
{
const GstGLFuncs *gl = overlay->context->gl_vtable;
if (gl->GenVertexArrays)
gl->BindVertexArray (overlay->vao);
else
gst_gl_composition_overlay_bind_vertex_buffer (overlay);
if (overlay->texture_id != -1)
gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id);
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
}
/********************************************************************
* GstGLOverlayCompositor object, the public helper object to render
* GstVideoCompositionOverlayMeta
********************************************************************/
#define DEBUG_INIT \
GST_DEBUG_CATEGORY_INIT (gst_gl_overlay_compositor_debug, \
"gloverlaycompositor", 0, "overlaycompositor");
G_DEFINE_TYPE_WITH_CODE (GstGLOverlayCompositor, gst_gl_overlay_compositor,
GST_TYPE_OBJECT, DEBUG_INIT);
static void gst_gl_overlay_compositor_finalize (GObject * object);
static gboolean _is_rectangle_in_overlays (GList * overlays,
GstVideoOverlayRectangle * rectangle);
static gboolean _is_overlay_in_rectangles (GstVideoOverlayComposition *
composition, GstGLCompositionOverlay * overlay);
static void
gst_gl_overlay_compositor_class_init (GstGLOverlayCompositorClass * klass)
{
G_OBJECT_CLASS (klass)->finalize = gst_gl_overlay_compositor_finalize;
}
static void
gst_gl_overlay_compositor_init (GstGLOverlayCompositor * compositor)
{
}
static void
gst_gl_overlay_compositor_init_gl (GstGLContext * context,
gpointer compositor_pointer)
{
GstGLOverlayCompositor *compositor =
(GstGLOverlayCompositor *) compositor_pointer;
GError *error = NULL;
if (!(compositor->shader =
gst_gl_shader_new_link_with_stages (context, &error,
gst_glsl_stage_new_default_vertex (context),
gst_glsl_stage_new_with_string (context, GL_FRAGMENT_SHADER,
GST_GLSL_VERSION_NONE,
GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY,
fragment_shader), NULL))) {
GST_ERROR_OBJECT (compositor, "could not initialize shader: %s",
error->message);
return;
}
compositor->position_attrib =
gst_gl_shader_get_attribute_location (compositor->shader, "a_position");
compositor->texcoord_attrib =
gst_gl_shader_get_attribute_location (compositor->shader, "a_texcoord");
}
GstGLOverlayCompositor *
gst_gl_overlay_compositor_new (GstGLContext * context)
{
GstGLOverlayCompositor *compositor =
g_object_new (GST_TYPE_GL_OVERLAY_COMPOSITOR, NULL);
compositor->context = gst_object_ref (context);
gst_gl_context_thread_add (compositor->context,
gst_gl_overlay_compositor_init_gl, compositor);
GST_DEBUG_OBJECT (compositor, "Created new GstGLOverlayCompositor");
return compositor;
}
static void
gst_gl_overlay_compositor_finalize (GObject * object)
{
GstGLOverlayCompositor *compositor;
compositor = GST_GL_OVERLAY_COMPOSITOR (object);
gst_gl_overlay_compositor_free_overlays (compositor);
if (compositor->context)
gst_object_unref (compositor->context);
if (compositor->shader) {
gst_object_unref (compositor->shader);
compositor->shader = NULL;
}
G_OBJECT_CLASS (gst_gl_overlay_compositor_parent_class)->finalize (object);
}
static gboolean
_is_rectangle_in_overlays (GList * overlays,
GstVideoOverlayRectangle * rectangle)
{
GList *l;
for (l = overlays; l != NULL; l = l->next) {
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
if (overlay->rectangle == rectangle)
return TRUE;
}
return FALSE;
}
static gboolean
_is_overlay_in_rectangles (GstVideoOverlayComposition * composition,
GstGLCompositionOverlay * overlay)
{
guint i;
for (i = 0; i < gst_video_overlay_composition_n_rectangles (composition); i++) {
GstVideoOverlayRectangle *rectangle =
gst_video_overlay_composition_get_rectangle (composition, i);
if (overlay->rectangle == rectangle)
return TRUE;
}
return FALSE;
}
void
gst_gl_overlay_compositor_free_overlays (GstGLOverlayCompositor * compositor)
{
GList *l = compositor->overlays;
while (l != NULL) {
GList *next = l->next;
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
compositor->overlays = g_list_delete_link (compositor->overlays, l);
gst_object_unref (overlay);
l = next;
}
g_list_free (compositor->overlays);
compositor->overlays = NULL;
}
void
gst_gl_overlay_compositor_upload_overlays (GstGLOverlayCompositor * compositor,
GstBuffer * buf)
{
GstVideoOverlayCompositionMeta *composition_meta;
composition_meta = gst_buffer_get_video_overlay_composition_meta (buf);
if (composition_meta) {
GstVideoOverlayComposition *composition = NULL;
guint num_overlays, i;
GList *l = compositor->overlays;
GST_DEBUG ("GstVideoOverlayCompositionMeta found.");
composition = composition_meta->overlay;
num_overlays = gst_video_overlay_composition_n_rectangles (composition);
/* add new overlays to list */
for (i = 0; i < num_overlays; i++) {
GstVideoOverlayRectangle *rectangle =
gst_video_overlay_composition_get_rectangle (composition, i);
if (!_is_rectangle_in_overlays (compositor->overlays, rectangle)) {
GstGLCompositionOverlay *overlay =
gst_gl_composition_overlay_new (compositor->context, rectangle,
compositor->position_attrib, compositor->texcoord_attrib);
gst_gl_composition_overlay_upload (overlay, buf);
compositor->overlays = g_list_append (compositor->overlays, overlay);
}
}
/* remove old overlays from list */
while (l != NULL) {
GList *next = l->next;
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
if (!_is_overlay_in_rectangles (composition, overlay)) {
compositor->overlays = g_list_delete_link (compositor->overlays, l);
gst_object_unref (overlay);
}
l = next;
}
} else {
gst_gl_overlay_compositor_free_overlays (compositor);
}
}
void
gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor)
{
const GstGLFuncs *gl = compositor->context->gl_vtable;
if (compositor->overlays != NULL) {
GList *l;
gl->Enable (GL_BLEND);
gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
gst_gl_shader_use (compositor->shader);
gl->ActiveTexture (GL_TEXTURE0);
gst_gl_shader_set_uniform_1i (compositor->shader, "tex", 0);
for (l = compositor->overlays; l != NULL; l = l->next) {
GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
gst_gl_composition_overlay_draw (overlay, compositor->shader);
}
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->Disable (GL_BLEND);
}
}
GstCaps *
gst_gl_overlay_compositor_add_caps (GstCaps * caps)
{
GstCaps *composition_caps;
int i;
composition_caps = gst_caps_copy (caps);
for (i = 0; i < gst_caps_get_size (composition_caps); i++) {
GstCapsFeatures *f = gst_caps_get_features (composition_caps, i);
gst_caps_features_add (f,
GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
}
caps = gst_caps_merge (composition_caps, caps);
return caps;
}