gstreamer/gst-libs/gst/gl/gstglcompositionoverlay.c

/*
 * GStreamer
 * Copyright (C) 2015 Lubosz Sarnecki <lubosz.sarnecki@collabora.co.uk>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <stdio.h>

#include "gl.h"
#include "gstglcompositionoverlay.h"


GST_DEBUG_CATEGORY_STATIC (gst_gl_composition_overlay_debug);
#define GST_CAT_DEFAULT gst_gl_composition_overlay_debug


#define DEBUG_INIT \
  GST_DEBUG_CATEGORY_INIT (gst_gl_composition_overlay_debug, \
      "glcompositionoverlay", 0, "compositionoverlay");

G_DEFINE_TYPE_WITH_CODE (GstGLCompositionOverlay, gst_gl_composition_overlay,
    GST_TYPE_OBJECT, DEBUG_INIT);

static void gst_gl_composition_overlay_finalize (GObject * object);
static void
gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
    overlay, GstBuffer * video_buffer, guint window_width, guint window_height);

static void
gst_gl_composition_overlay_class_init (GstGLCompositionOverlayClass * klass)
{
  G_OBJECT_CLASS (klass)->finalize = gst_gl_composition_overlay_finalize;
}

static void
gst_gl_composition_overlay_init (GstGLCompositionOverlay * overlay)
{
}

GstGLCompositionOverlay *
gst_gl_composition_overlay_new (GstGLContext * context,
    GstVideoOverlayRectangle * rectangle,
    GLint position_attrib, GLint texcoord_attrib)
{
  GstGLCompositionOverlay *overlay =
      g_object_new (GST_TYPE_GL_COMPOSITION_OVERLAY, NULL);

  overlay->gl_memory = NULL;
  overlay->texture_id = -1;
  overlay->rectangle = rectangle;
  overlay->context = gst_object_ref (context);
  overlay->vao = 0;
  overlay->position_attrib = position_attrib;
  overlay->texcoord_attrib = texcoord_attrib;

  GST_DEBUG_OBJECT (overlay, "Created new GstGLCompositionOverlay");

  return overlay;
}

static void
gst_gl_composition_overlay_free_vertex_buffer (GstGLContext * context,
    gpointer overlay_pointer)
{
  const GstGLFuncs *gl = context->gl_vtable;
  GstGLCompositionOverlay *overlay =
      (GstGLCompositionOverlay *) overlay_pointer;
  if (overlay->vao) {
    gl->DeleteVertexArrays (1, &overlay->vao);
    overlay->vao = 0;
  }

  if (overlay->position_buffer) {
    gl->DeleteBuffers (1, &overlay->position_buffer);
    overlay->position_buffer = 0;
  }

  if (overlay->texcoord_buffer) {
    gl->DeleteBuffers (1, &overlay->position_buffer);
    overlay->position_buffer = 0;
  }

  if (overlay->index_buffer) {
    gl->DeleteBuffers (1, &overlay->index_buffer);
    overlay->index_buffer = 0;
  }
}

static void
gst_gl_composition_overlay_finalize (GObject * object)
{
  GstGLCompositionOverlay *overlay;

  overlay = GST_GL_COMPOSITION_OVERLAY (object);

  if (overlay->gl_memory)
    gst_memory_unref ((GstMemory *) overlay->gl_memory);
  overlay->gl_memory = NULL;
  if (overlay->context) {
    gst_gl_context_thread_add (overlay->context,
        gst_gl_composition_overlay_free_vertex_buffer, overlay);
    gst_object_unref (overlay->context);
  }

  G_OBJECT_CLASS (gst_gl_composition_overlay_parent_class)->finalize (object);
}

static void
gst_gl_composition_overlay_bind_vertex_buffer (GstGLCompositionOverlay *
    overlay)
{
  const GstGLFuncs *gl = overlay->context->gl_vtable;
  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
  gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
      4 * sizeof (GLfloat), NULL);

  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
  gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
      2 * sizeof (GLfloat), NULL);

  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);

  gl->EnableVertexAttribArray (overlay->position_attrib);
  gl->EnableVertexAttribArray (overlay->texcoord_attrib);
}

static void
gst_gl_composition_overlay_init_vertex_buffer (GstGLContext * context,
    gpointer overlay_pointer)
{
  const GstGLFuncs *gl = context->gl_vtable;
  GstGLCompositionOverlay *overlay =
      (GstGLCompositionOverlay *) overlay_pointer;

  /* *INDENT-OFF* */
  static const GLfloat texcoords[] = {
      1.0f, 0.0f,
      0.0f, 0.0f,
      0.0f, 1.0f,
      1.0f, 1.0f
  };

  static const GLushort indices[] = {
    0, 1, 2, 0, 2, 3
  };
  /* *INDENT-ON* */

  if (gl->GenVertexArrays) {
    gl->GenVertexArrays (1, &overlay->vao);
    gl->BindVertexArray (overlay->vao);
  }

  gl->GenBuffers (1, &overlay->position_buffer);
  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->position_buffer);
  gl->BufferData (GL_ARRAY_BUFFER, 4 * 4 * sizeof (GLfloat), overlay->positions,
      GL_STATIC_DRAW);

  /* Load the vertex position */
  gl->VertexAttribPointer (overlay->position_attrib, 4, GL_FLOAT, GL_FALSE,
      4 * sizeof (GLfloat), NULL);

  gl->GenBuffers (1, &overlay->texcoord_buffer);
  gl->BindBuffer (GL_ARRAY_BUFFER, overlay->texcoord_buffer);
  gl->BufferData (GL_ARRAY_BUFFER, 4 * 2 * sizeof (GLfloat), texcoords,
      GL_STATIC_DRAW);

  /* Load the texture coordinate */
  gl->VertexAttribPointer (overlay->texcoord_attrib, 2, GL_FLOAT, GL_FALSE,
      2 * sizeof (GLfloat), NULL);

  gl->GenBuffers (1, &overlay->index_buffer);
  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, overlay->index_buffer);
  gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
      GL_STATIC_DRAW);

  gl->EnableVertexAttribArray (overlay->position_attrib);
  gl->EnableVertexAttribArray (overlay->texcoord_attrib);

  if (gl->GenVertexArrays) {
    gl->BindVertexArray (0);
  }

  gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
  gl->BindBuffer (GL_ARRAY_BUFFER, 0);
}

static void
gst_gl_composition_overlay_add_transformation (GstGLCompositionOverlay *
    overlay, GstBuffer * video_buffer, guint window_width, guint window_height)
{
  gint comp_x, comp_y;
  guint comp_width, comp_height;
  GstVideoMeta *meta;
  guint width, height;

  float rel_x, rel_y, rel_w, rel_h;

  gfloat window_aspect = 1.0;
  gfloat video_aspect = 1.0;

  meta = gst_buffer_get_video_meta (video_buffer);

  gst_video_overlay_rectangle_get_render_rectangle (overlay->rectangle,
      &comp_x, &comp_y, &comp_width, &comp_height);

  width = window_width;
  height = window_height;

  window_aspect = (float) width / (float) height;
  video_aspect = (float) meta->width / (float) meta->height;

  /* calculate relative position */
  rel_x = (float) comp_x / (float) width;
  rel_y = (float) comp_y / (float) height;

  rel_w = (float) comp_width / (float) width;
  rel_h = (float) comp_height / (float) height;

  /* transform to window aspect ratio */
  if (window_aspect <= video_aspect) {
    rel_y *= video_aspect / window_aspect;
    rel_h *= video_aspect / window_aspect;
  } else {
    rel_x *= window_aspect / video_aspect;
    rel_w *= window_aspect / video_aspect;
  }

  /* transform from [0,1] to [-1,1], invert y axis */
  rel_x = rel_x * 2.0 - 1.0;
  rel_y = (1.0 - rel_y) * 2.0 - 1.0;
  rel_w = rel_w * 2.0;
  rel_h = rel_h * 2.0;

  /* initialize position array */
  overlay->positions[0] = rel_x + rel_w;
  overlay->positions[1] = rel_y;
  overlay->positions[2] = 0.0;
  overlay->positions[3] = 1.0;
  overlay->positions[4] = rel_x;
  overlay->positions[5] = rel_y;
  overlay->positions[6] = 0.0;
  overlay->positions[7] = 1.0;
  overlay->positions[8] = rel_x;
  overlay->positions[9] = rel_y - rel_h;
  overlay->positions[10] = 0.0;
  overlay->positions[11] = 1.0;
  overlay->positions[12] = rel_x + rel_w;
  overlay->positions[13] = rel_y - rel_h;
  overlay->positions[14] = 0.0;
  overlay->positions[15] = 1.0;

  gst_gl_context_thread_add (overlay->context,
      gst_gl_composition_overlay_free_vertex_buffer, overlay);

  gst_gl_context_thread_add (overlay->context,
      gst_gl_composition_overlay_init_vertex_buffer, overlay);

  GST_DEBUG
      ("overlay position: (%d,%d) size: %dx%d video size: %dx%d, sink window %dx%d",
      comp_x, comp_y, comp_width, comp_height, meta->width, meta->height,
      window_width, window_height);
}

void
gst_gl_composition_overlay_upload (GstGLCompositionOverlay * overlay,
    GstBuffer * buf, guint window_width, guint window_height)
{
  GstMapInfo info;
  GstVideoMeta *vmeta;
  GstGLMemory *comp_gl_memory = NULL;
  gint stride;
  GstBuffer *comp_buffer = NULL;
  GstVideoMeta *meta;
  GstVideoInfo text_info;
  gpointer raw_overlay_data;
  GstBuffer *overlay_buffer = NULL;
  GstVideoFrame gl_frame;

  comp_buffer =
      gst_video_overlay_rectangle_get_pixels_unscaled_argb (overlay->rectangle,
      GST_VIDEO_OVERLAY_FORMAT_FLAG_PREMULTIPLIED_ALPHA);

  vmeta = gst_buffer_get_video_meta (comp_buffer);

  if (gst_video_meta_map (vmeta, 0, &info, &raw_overlay_data, &stride,
          GST_MAP_READ)) {
    g_assert (raw_overlay_data);

    meta = gst_buffer_get_video_meta (comp_buffer);

    gst_gl_composition_overlay_add_transformation (overlay, buf, window_width,
        window_height);

    gst_video_info_init (&text_info);
    gst_video_info_set_format (&text_info, meta->format, meta->width,
        meta->height);

    comp_gl_memory =
        gst_gl_memory_wrapped (overlay->context, &text_info, 0, NULL,
        raw_overlay_data, NULL, NULL);

    overlay_buffer = gst_buffer_new ();
    gst_buffer_append_memory (overlay_buffer,
        gst_memory_ref ((GstMemory *) comp_gl_memory));

    if (!gst_video_frame_map (&gl_frame, &text_info, overlay_buffer,
            GST_MAP_READ | GST_MAP_GL)) {
      gst_buffer_unref (overlay_buffer);
      gst_video_meta_unmap (vmeta, 0, &info);
      GST_WARNING_OBJECT (overlay, "Cannot upload overlay texture");
      return;
    }

    gst_buffer_unref (overlay_buffer);
    overlay->texture_id = comp_gl_memory->tex_id;
    GST_DEBUG ("uploaded overlay texture %d", overlay->texture_id);
    gst_video_frame_unmap (&gl_frame);
    overlay->gl_memory = comp_gl_memory;

    gst_video_meta_unmap (vmeta, 0, &info);
  }
}

void
gst_gl_composition_overlay_draw (GstGLCompositionOverlay * overlay,
    GstGLShader * shader)
{
  const GstGLFuncs *gl = overlay->context->gl_vtable;
  if (gl->GenVertexArrays)
    gl->BindVertexArray (overlay->vao);
  else
    gst_gl_composition_overlay_bind_vertex_buffer (overlay);

  if (overlay->texture_id != -1)
    gl->BindTexture (GL_TEXTURE_2D, overlay->texture_id);
  gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
}