gstreamer/gst/gl/gstglbumper.c

/*
 * GStreamer
 * Copyright (C) 2008 Cyril Comparon <cyril.comparon@gmail.com>
 * Copyright (C) 2008 Julien Isorce <julien.isorce@gmail.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-glbumper
 *
 * Bump mapping using the normal method.
 *
 * <refsect2>
 * <title>Examples</title>
 * |[
 * gst-launch -v videotestsrc ! glupload ! glbumper location=normalmap.bmp ! glimagesink
 * ]| A pipeline to test normal mapping.
 * FBO (Frame Buffer Object) and GLSL (OpenGL Shading Language) are required.
 * </refsect2>
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <stdlib.h>
#include <png.h>
#include "gstglbumper.h"

#if PNG_LIBPNG_VER >= 10400
#define int_p_NULL         NULL
#define png_infopp_NULL    NULL
#endif

#define GST_CAT_DEFAULT gst_gl_bumper_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);

enum
{
  PROP_0,
  PROP_LOCATION
};

#define DEBUG_INIT \
  GST_DEBUG_CATEGORY_INIT (gst_gl_bumper_debug, "glbumper", 0, "glbumper element");

G_DEFINE_TYPE_WITH_CODE (GstGLBumper, gst_gl_bumper, GST_TYPE_GL_FILTER,
    DEBUG_INIT);

static void gst_gl_bumper_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_gl_bumper_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

static void gst_gl_bumper_reset (GstGLFilter * filter);
static gboolean gst_gl_bumper_init_shader (GstGLFilter * filter);
static gboolean gst_gl_bumper_filter_texture (GstGLFilter * filter,
    guint in_tex, guint out_tex);
static void gst_gl_bumper_callback (gint width, gint height, guint texture,
    gpointer stuff);

//vertex source
static const gchar *bumper_v_src =
    "attribute vec3 aTangent;\n"
    "\n"
    "varying vec3 vNormal;\n"
    "varying vec3 vTangent;\n"
    "varying vec3 vVertexToLight0;\n"
    "varying vec3 vVertexToLight1;\n"
    "\n"
    "void main()\n"
    "{\n"
    "  // transform the vertex\n"
    "  gl_Position = gl_ProjectionMatrix * gl_ModelViewMatrix * gl_Vertex;\n"
    "\n"
    "  // transform the normal and the tangent to scene coords\n"
    "  vNormal = normalize(gl_NormalMatrix * gl_Normal);\n"
    "  vTangent = normalize(gl_NormalMatrix * aTangent);\n"
    "\n"
    "  // transforming the vertex position to modelview-space\n"
    "  //const vec4 vertexInSceneCoords = gl_ModelViewMatrix * gl_Vertex;\n"
    "\n"
    "  // calculate the vector from the vertex position to the light position\n"
    "  vVertexToLight0 = normalize(gl_LightSource[0].position).xyz;\n"
    "  vVertexToLight1 = normalize(gl_LightSource[1].position).xyz;\n"
    "\n"
    "  // transit vertex color\n"
    "  gl_FrontColor = gl_BackColor = gl_Color;\n"
    "\n"
    "  // use the two first sets of texture coordinates in the fragment shader\n"
    "  gl_TexCoord[0] = gl_MultiTexCoord0;\n"
    "  gl_TexCoord[1] = gl_MultiTexCoord1;\n" "}\n";

//fragment source
static const gchar *bumper_f_src =
    "#extension GL_ARB_texture_rectangle : enable\n"
    "uniform sampler2DRect texture0;\n"
    "uniform sampler2DRect texture1;\n"
    "\n"
    "varying vec3 vNormal;\n"
    "varying vec3 vTangent;\n"
    "varying vec3 vVertexToLight0;\n"
    "varying vec3 vVertexToLight1;\n"
    "\n"
    "void main()\n"
    "{\n"
    "  // get the color of the textures\n"
    "  vec4 textureColor = texture2DRect(texture0, gl_TexCoord[0].st);\n"
    "  vec3 normalmapItem = texture2DRect(texture1, gl_TexCoord[1].st).xyz * 2.0 - 1.0;\n"
    "\n"
    "  // calculate matrix that transform from tangent space to normalmap space (contrary of intuition)\n"
    "  vec3 binormal = cross(vNormal, vTangent);\n"
    "  mat3 tangentSpace2normalmapSpaceMat = mat3(vTangent, binormal, vNormal);\n"
    "\n"
    "  // disturb the normal\n"
    "  vec3 disturbedNormal = tangentSpace2normalmapSpaceMat * normalmapItem;\n"
    "\n"
    "  // calculate the diffuse term and clamping it to [0;1]\n"
    "  float diffuseTerm0 = clamp(dot(disturbedNormal, vVertexToLight0), 0.0, 1.0);\n"
    "  float diffuseTerm1 = clamp(dot(disturbedNormal, vVertexToLight1), 0.0, 1.0);\n"
    "\n"
    "  vec3 irradiance = (diffuseTerm0 * gl_LightSource[0].diffuse.rgb + diffuseTerm1 * gl_LightSource[1].diffuse.rgb);\n"
    "\n"
    "  // calculate the final color\n"
    "  gl_FragColor = vec4(irradiance * textureColor.rgb, textureColor.w);\n"
    "}\n";

#define LOAD_ERROR(display, msg) { gst_gl_display_set_error (display, "unable to load %s: %s", bumper->location, msg); return; }

//png reading error handler
static void
user_warning_fn (png_structp png_ptr, png_const_charp warning_msg)
{
  g_warning ("%s\n", warning_msg);
}

//Called in the gl thread
static void
gst_gl_bumper_init_resources (GstGLFilter * filter)
{
  GstGLBumper *bumper = GST_GL_BUMPER (filter);
  GstGLDisplay *display = filter->display;

  png_structp png_ptr;
  png_infop info_ptr;
  png_uint_32 width = 0;
  png_uint_32 height = 0;
  gint bit_depth = 0;
  gint color_type = 0;
  gint interlace_type = 0;
  png_FILE_p fp = NULL;
  guint y = 0;
  guchar *raw_data = NULL;
  guchar **rows = NULL;
  png_byte magic[8];
  gint n_read;

  if (!display)
    return;

  if (!bumper->location) {
    gst_gl_display_set_error (display, "A filename is required");
    return;
  }

  /* BEGIN load png image file */

  if ((fp = fopen (bumper->location, "rb")) == NULL)
    LOAD_ERROR (display, "file not found");

  /* Read magic number */
  n_read = fread (magic, 1, sizeof (magic), fp);
  if (n_read != sizeof (magic)) {
    fclose (fp);
    LOAD_ERROR (display, "can't read PNG magic number");
  }

  /* Check for valid magic number */
  if (png_sig_cmp (magic, 0, sizeof (magic))) {
    fclose (fp);
    LOAD_ERROR (display, "not a valid PNG image");
  }

  png_ptr = png_create_read_struct (PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);

  if (png_ptr == NULL) {
    fclose (fp);
    LOAD_ERROR (display, "failed to initialize the png_struct");
  }

  png_set_error_fn (png_ptr, NULL, NULL, user_warning_fn);

  info_ptr = png_create_info_struct (png_ptr);
  if (info_ptr == NULL) {
    fclose (fp);
    png_destroy_read_struct (&png_ptr, png_infopp_NULL, png_infopp_NULL);
    LOAD_ERROR (display,
        "failed to initialize the memory for image information");
  }

  png_init_io (png_ptr, fp);

  png_set_sig_bytes (png_ptr, sizeof (magic));

  png_read_info (png_ptr, info_ptr);

  png_get_IHDR (png_ptr, info_ptr, &width, &height, &bit_depth, &color_type,
      &interlace_type, int_p_NULL, int_p_NULL);

  if (color_type != PNG_COLOR_TYPE_RGB) {
    fclose (fp);
    png_destroy_read_struct (&png_ptr, png_infopp_NULL, png_infopp_NULL);
    LOAD_ERROR (display, "color type is not rgb");
  }

  raw_data = (guchar *) malloc (sizeof (guchar) * width * height * 3);

  rows = (guchar **) malloc (sizeof (guchar *) * height);

  for (y = 0; y < height; ++y)
    rows[y] = (guchar *) (raw_data + y * width * 3);

  png_read_image (png_ptr, rows);

  free (rows);

  png_read_end (png_ptr, info_ptr);
  png_destroy_read_struct (&png_ptr, &info_ptr, png_infopp_NULL);
  fclose (fp);

  /* END load png image file */

  bumper->bumpmap_width = width;
  bumper->bumpmap_height = height;

  glGenTextures (1, &bumper->bumpmap);
  glBindTexture (GL_TEXTURE_RECTANGLE_ARB, bumper->bumpmap);
  glTexImage2D (GL_TEXTURE_RECTANGLE_ARB, 0, GL_RGBA,
      bumper->bumpmap_width, bumper->bumpmap_height, 0,
      GL_RGB, GL_UNSIGNED_BYTE, raw_data);

  free (raw_data);
}

//Called in the gl thread
static void
gst_gl_bumper_reset_resources (GstGLFilter * filter)
{
  GstGLBumper *bumper = GST_GL_BUMPER (filter);

  if (bumper->bumpmap) {
    glDeleteTextures (1, &bumper->bumpmap);
    bumper->bumpmap = 0;
  }
}

static void
gst_gl_bumper_class_init (GstGLBumperClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;

  gobject_class = (GObjectClass *) klass;
  element_class = GST_ELEMENT_CLASS (klass);
  gobject_class->set_property = gst_gl_bumper_set_property;
  gobject_class->get_property = gst_gl_bumper_get_property;

  GST_GL_FILTER_CLASS (klass)->filter_texture = gst_gl_bumper_filter_texture;
  GST_GL_FILTER_CLASS (klass)->display_init_cb = gst_gl_bumper_init_resources;
  GST_GL_FILTER_CLASS (klass)->display_reset_cb = gst_gl_bumper_reset_resources;
  GST_GL_FILTER_CLASS (klass)->onInitFBO = gst_gl_bumper_init_shader;
  GST_GL_FILTER_CLASS (klass)->onReset = gst_gl_bumper_reset;

  g_object_class_install_property (gobject_class,
      PROP_LOCATION, g_param_spec_string ("location",
          "Normal map location",
          "Normal map location", NULL,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  gst_element_class_set_metadata (element_class, "OpenGL bumper filter",
      "Filter/Effect", "Bump mapping filter",
      "Cyril Comparon <cyril.comparon@gmail.com>, "
      "Julien Isorce <julien.isorce@gmail.com>");
}

static void
gst_gl_bumper_init (GstGLBumper * bumper)
{
  bumper->shader = NULL;
  bumper->bumpmap = 0;
  bumper->bumpmap_width = 0;
  bumper->bumpmap_height = 0;
  bumper->location = NULL;
}

static void
gst_gl_bumper_reset (GstGLFilter * filter)
{
  GstGLBumper *bumper_filter = GST_GL_BUMPER (filter);

  //blocking call, wait the opengl thread has destroyed the shader
  gst_gl_display_del_shader (filter->display, bumper_filter->shader);
}

static void
gst_gl_bumper_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstGLBumper *bumper = GST_GL_BUMPER (object);

  switch (prop_id) {
    case PROP_LOCATION:
      if (bumper->location != NULL)
        g_free (bumper->location);
      bumper->location = g_value_dup_string (value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_bumper_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstGLBumper *bumper = GST_GL_BUMPER (object);

  switch (prop_id) {
    case PROP_LOCATION:
      g_value_set_string (value, bumper->location);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
gst_gl_bumper_init_shader (GstGLFilter * filter)
{
  GstGLBumper *bumper = GST_GL_BUMPER (filter);

  //blocking call, wait the opengl thread has compiled the shader
  return gst_gl_display_gen_shader (filter->display, bumper_v_src, bumper_f_src,
      &bumper->shader);
}

static gboolean
gst_gl_bumper_filter_texture (GstGLFilter * filter, guint in_tex, guint out_tex)
{
  gpointer bumper_filter = GST_GL_BUMPER (filter);

  //blocking call, use a FBO
  gst_gl_display_use_fbo (filter->display,
      GST_VIDEO_INFO_WIDTH (&filter->out_info),
      GST_VIDEO_INFO_HEIGHT (&filter->out_info),
      filter->fbo, filter->depthbuffer, out_tex, gst_gl_bumper_callback,
      GST_VIDEO_INFO_WIDTH (&filter->in_info),
      GST_VIDEO_INFO_HEIGHT (&filter->in_info),
      in_tex, 45,
      (gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
      (gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info), 0.1, 50,
      GST_GL_DISPLAY_PROJECTION_PERSPECTIVE, bumper_filter);

  return TRUE;
}

typedef struct _MeshData
{
  float x, y, z;                /* Vertex */
  float nx, ny, nz;             /* Normal */
  float s0, t0;                 /* TexCoord0 */
  float s1, t1;                 /* TexCoord1 */
  float va0, vb0, vc0;          /* VertexAttrib */
} MeshData;

//opengl scene, params: input texture (not the output filter->texture)
static void
gst_gl_bumper_callback (gint width, gint height, guint texture, gpointer stuff)
{
  static GLfloat xrot = 0;
  static GLfloat yrot = 0;
  static GLfloat zrot = 0;

  GstGLFuncs *gl;
  GstGLBumper *bumper = GST_GL_BUMPER (stuff);
  GstGLDisplay *display = GST_GL_FILTER (bumper)->display;
  GLint locTangent = 0;

  //choose the lights
  GLfloat light_direction0[] = { 1.0, 0.0, -1.0, 0.0 }; // light goes along -x
  GLfloat light_direction1[] = { -1.0, 0.0, -1.0, 0.0 };        // light goes along x
  GLfloat light_diffuse0[] = { 1.0, 1.0, 1.0, 1.0 };
  GLfloat light_diffuse1[] = { 1.0, 1.0, 1.0, 1.0 };
  GLfloat mat_diffuse[] = { 1.0, 1.0, 1.0, 1.0 };

  /* image size */
  gfloat i_w = (gfloat) width;
  gfloat i_h = (gfloat) height;
  /* bumpmap size */
  gfloat b_w = (gfloat) bumper->bumpmap_width;
  gfloat b_h = (gfloat) bumper->bumpmap_height;

/* *INDENT-OFF* */
  MeshData mesh[] = {
   /* |     Vertex      |     Normal      |TexCoord0|TexCoord1|  VertexAttrib  | */
/*F*/ { 1.0,  1.0, -1.0,  0.0,  0.0, -1.0, 0.0, 0.0, 0.0, 0.0,  0.0,  1.0,  0.0},
/*r*/ { 1.0, -1.0, -1.0,  0.0,  0.0, -1.0, i_w, 0.0, b_w, 0.0,  0.0,  1.0,  0.0},
/*o*/ {-1.0, -1.0, -1.0,  0.0,  0.0, -1.0, i_w, i_h, b_w, b_h,  0.0,  1.0,  0.0},
      {-1.0,  1.0, -1.0,  0.0,  0.0, -1.0, 0.0, i_h, 0.0, b_h,  0.0,  1.0,  0.0},
/*R*/ {-1.0,  1.0, -1.0, -1.0,  0.0,  0.0, 0.0, 0.0, 0.0, 0.0,  0.0,  1.0,  0.0},
/*i*/ {-1.0, -1.0, -1.0, -1.0,  0.0,  0.0, i_w, 0.0, b_w, 0.0,  0.0,  1.0,  0.0},
/*g*/ {-1.0, -1.0,  1.0, -1.0,  0.0,  0.0, i_w, i_h, b_w, b_h,  0.0,  1.0,  0.0},
      {-1.0,  1.0,  1.0, -1.0,  0.0,  0.0, 0.0, i_h, 0.0, b_h,  0.0,  1.0,  0.0},
/*B*/ {-1.0,  1.0,  1.0,  0.0,  0.0,  1.0, 0.0, 0.0, 0.0, 0.0,  0.0,  1.0,  0.0},
/*a*/ {-1.0, -1.0,  1.0,  0.0,  0.0,  1.0, i_w, 0.0, b_w, 0.0,  0.0,  1.0,  0.0},
/*c*/ { 1.0, -1.0,  1.0,  0.0,  0.0,  1.0, i_w, i_h, b_w, b_h,  0.0,  1.0,  0.0},
      { 1.0,  1.0,  1.0,  0.0,  0.0,  1.0, 0.0, i_h, 0.0, b_h,  0.0,  1.0,  0.0},
/*L*/ { 1.0,  1.0,  1.0,  1.0,  0.0,  0.0, 0.0, 0.0, 0.0, 0.0,  0.0,  1.0,  0.0},
/*e*/ { 1.0, -1.0,  1.0,  1.0,  0.0,  0.0, i_w, 0.0, b_w, 0.0,  0.0,  1.0,  0.0},
/*f*/ { 1.0, -1.0, -1.0,  1.0,  0.0,  0.0, i_w, i_h, b_w, b_h,  0.0,  1.0,  0.0},
      { 1.0,  1.0, -1.0,  1.0,  0.0,  0.0, 0.0, i_h, 0.0, b_h,  0.0,  1.0,  0.0},
/*T*/ { 1.0,  1.0,  1.0,  0.0,  1.0,  0.0, 0.0, 0.0, 0.0, 0.0,  0.0,  0.0,  1.0},
/*o*/ { 1.0,  1.0, -1.0,  0.0,  1.0,  0.0, i_w, 0.0, b_w, 0.0,  0.0,  0.0,  1.0},
/*p*/ {-1.0,  1.0, -1.0,  0.0,  1.0,  0.0, i_w, i_h, b_w, b_h,  0.0,  0.0,  1.0},
      {-1.0,  1.0,  1.0,  0.0,  1.0,  0.0, 0.0, i_h, 0.0, b_h,  0.0,  0.0,  1.0},
/*B*/ { 1.0, -1.0, -1.0,  0.0, -1.0,  0.0, 0.0, 0.0, 0.0, 0.0,  0.0,  0.0, -1.0},
/*o*/ { 1.0, -1.0,  1.0,  0.0, -1.0,  0.0, i_w, 0.0, b_w, 0.0,  0.0,  0.0, -1.0},
/*t*/ {-1.0, -1.0,  1.0,  0.0, -1.0,  0.0, i_w, i_h, b_w, b_h,  0.0,  0.0, -1.0},
      {-1.0, -1.0, -1.0,  0.0, -1.0,  0.0, 0.0, i_h, 0.0, b_h,  0.0,  0.0, -1.0},
  };

  GLushort indices[] = {
    0, 1, 2,
    0, 2, 3,
    4, 5, 6,
    4, 6, 7,
    8, 9, 10,
    8, 10, 11,
    12, 13, 14,
    12, 14, 15,
    16, 17, 18,
    16, 18, 19,
    20, 21, 22,
    20, 22, 23
  };

/* *INDENT-ON* */

  gl = GST_GL_FILTER (bumper)->display->gl_vtable;

  //eye point
  gl->MatrixMode (GL_PROJECTION);
  gluLookAt (0.0, 0.0, -6.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0);
  gl->MatrixMode (GL_MODELVIEW);

  //scene conf
  gl->Enable (GL_DEPTH_TEST);
  gl->DepthFunc (GL_LEQUAL);
  gl->Hint (GL_PERSPECTIVE_CORRECTION_HINT, GL_NICEST);

  gl->ShadeModel (GL_SMOOTH);

  //set the lights
  gl->Lightfv (GL_LIGHT0, GL_POSITION, light_direction0);
  gl->Lightfv (GL_LIGHT0, GL_DIFFUSE, light_diffuse0);
  gl->Lightfv (GL_LIGHT1, GL_POSITION, light_direction1);
  gl->Lightfv (GL_LIGHT1, GL_DIFFUSE, light_diffuse1);
  gl->Materialfv (GL_FRONT, GL_DIFFUSE, mat_diffuse);
  gl->ColorMaterial (GL_FRONT_AND_BACK, GL_DIFFUSE);
  gl->Enable (GL_COLOR_MATERIAL);
  gl->Enable (GL_LIGHTING);
  gl->Enable (GL_LIGHT0);
  gl->Enable (GL_LIGHT1);
  //configure shader
  gst_gl_shader_use (bumper->shader);
  locTangent =
      gst_gl_shader_get_attribute_location (bumper->shader, "aTangent");

  //set the normal map
  gl->ActiveTexture (GL_TEXTURE1);
  gst_gl_shader_set_uniform_1i (bumper->shader, "texture1", 1);
  gl->BindTexture (GL_TEXTURE_RECTANGLE_ARB, bumper->bumpmap);

  //set the video texture
  gl->ActiveTexture (GL_TEXTURE0);
  gst_gl_shader_set_uniform_1i (bumper->shader, "texture0", 0);
  gl->BindTexture (GL_TEXTURE_RECTANGLE_ARB, texture);

  gl->Rotatef (xrot, 1.0f, 0.0f, 0.0f);
  gl->Rotatef (yrot, 0.0f, 1.0f, 0.0f);
  gl->Rotatef (zrot, 0.0f, 0.0f, 1.0f);

  gl->EnableVertexAttribArray (locTangent);

  gl->ClientActiveTexture (GL_TEXTURE0);
  gl->EnableClientState (GL_TEXTURE_COORD_ARRAY);
  gl->EnableClientState (GL_VERTEX_ARRAY);
  gl->EnableClientState (GL_NORMAL_ARRAY);

  gl->VertexAttribPointer (locTangent, 3, GL_FLOAT, 0, sizeof (MeshData),
      &mesh[0].va0);
  gl->VertexPointer (3, GL_FLOAT, sizeof (MeshData), &mesh[0].x);
  gl->NormalPointer (GL_FLOAT, sizeof (MeshData), &mesh[0].nx);
  gl->TexCoordPointer (2, GL_FLOAT, sizeof (MeshData), &mesh[0].s0);

  gl->ClientActiveTexture (GL_TEXTURE1);
  gl->EnableClientState (GL_TEXTURE_COORD_ARRAY);
  gl->TexCoordPointer (2, GL_FLOAT, sizeof (MeshData), &mesh[0].s1);

  gl->DrawElements (GL_TRIANGLES, 36, GL_UNSIGNED_SHORT, indices);

  gl->DisableClientState (GL_VERTEX_ARRAY);
  gl->DisableClientState (GL_TEXTURE_COORD_ARRAY);
  gl->DisableClientState (GL_NORMAL_ARRAY);

  gl->ClientActiveTexture (GL_TEXTURE0);
  gl->DisableClientState (GL_TEXTURE_COORD_ARRAY);

  gl->DisableVertexAttribArray (locTangent);

  gst_gl_display_clear_shader (display);

  gl->Disable (GL_LIGHT0);
  gl->Disable (GL_LIGHT1);
  gl->Disable (GL_LIGHTING);
  gl->Disable (GL_COLOR_MATERIAL);

  xrot += 1.0f;
  yrot += 0.9f;
  zrot += 0.6f;
}