mirror of
https://gitlab.freedesktop.org/gstreamer/gstreamer.git
synced 2024-09-09 05:38:58 +00:00
With NEON instructions enabled, graphene expects the memory passed to it 16-byte-aligned. Otherwise unaligned memory access faults occur causing SIGBUS signals. graphene has alloc functions for its structures that take care of this, so use them. See also: https://github.com/ebassi/graphene/issues/215#issuecomment-794744829 Suggested-by: Sebastian Dröge <sebastian@centricular.com> Signed-off-by: Bastian Krause <bst@pengutronix.de> Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1321>
1018 lines
34 KiB
C
1018 lines
34 KiB
C
/*
|
|
* GStreamer
|
|
* Copyright (C) 2014 Lubosz Sarnecki <lubosz@gmail.com>
|
|
* Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Library General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Library General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Library General Public
|
|
* License along with this library; if not, write to the
|
|
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
|
|
* Boston, MA 02110-1301, USA.
|
|
*/
|
|
|
|
/**
|
|
* SECTION:element-gltransformation
|
|
* @title: gltransformation
|
|
*
|
|
* Transforms video on the GPU.
|
|
*
|
|
* ## Examples
|
|
* |[
|
|
* gst-launch-1.0 gltestsrc ! gltransformation rotation-z=45 ! glimagesink
|
|
* ]| A pipeline to rotate by 45 degrees
|
|
* |[
|
|
* gst-launch-1.0 gltestsrc ! gltransformation translation-x=0.5 ! glimagesink
|
|
* ]| Translate the video by 0.5
|
|
* |[
|
|
* gst-launch-1.0 gltestsrc ! gltransformation scale-y=0.5 scale-x=0.5 ! glimagesink
|
|
* ]| Resize the video by 0.5
|
|
* |[
|
|
* gst-launch-1.0 gltestsrc ! gltransformation rotation-x=-45 ortho=True ! glimagesink
|
|
* ]| Rotate the video around the X-Axis by -45° with an orthographic projection
|
|
*
|
|
*/
|
|
|
|
#ifdef HAVE_CONFIG_H
|
|
#include "config.h"
|
|
#endif
|
|
|
|
#include "gstglelements.h"
|
|
#include "gstgltransformation.h"
|
|
|
|
#include <gst/gl/gstglapi.h>
|
|
#include <graphene-gobject.h>
|
|
#include "gstglutils.h"
|
|
|
|
#define GST_CAT_DEFAULT gst_gl_transformation_debug
|
|
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
|
|
|
|
#define gst_gl_transformation_parent_class parent_class
|
|
|
|
#define VEC4_FORMAT "%f,%f,%f,%f"
|
|
#define VEC4_ARGS(v) graphene_vec4_get_x (v), graphene_vec4_get_y (v), graphene_vec4_get_z (v), graphene_vec4_get_w (v)
|
|
#define VEC3_FORMAT "%f,%f,%f"
|
|
#define VEC3_ARGS(v) graphene_vec3_get_x (v), graphene_vec3_get_y (v), graphene_vec3_get_z (v)
|
|
#define VEC2_FORMAT "%f,%f"
|
|
#define VEC2_ARGS(v) graphene_vec2_get_x (v), graphene_vec2_get_y (v)
|
|
#define POINT3D_FORMAT "%f,%f,%f"
|
|
#define POINT3D_ARGS(p) (p)->x, (p)->y, (p)->z
|
|
|
|
enum
|
|
{
|
|
PROP_0,
|
|
PROP_FOV,
|
|
PROP_ORTHO,
|
|
PROP_TRANSLATION_X,
|
|
PROP_TRANSLATION_Y,
|
|
PROP_TRANSLATION_Z,
|
|
PROP_ROTATION_X,
|
|
PROP_ROTATION_Y,
|
|
PROP_ROTATION_Z,
|
|
PROP_SCALE_X,
|
|
PROP_SCALE_Y,
|
|
PROP_MVP,
|
|
PROP_PIVOT_X,
|
|
PROP_PIVOT_Y,
|
|
PROP_PIVOT_Z,
|
|
};
|
|
|
|
#define DEBUG_INIT \
|
|
GST_DEBUG_CATEGORY_INIT (gst_gl_transformation_debug, "gltransformation", 0, "gltransformation element");
|
|
|
|
G_DEFINE_TYPE_WITH_CODE (GstGLTransformation, gst_gl_transformation,
|
|
GST_TYPE_GL_FILTER, DEBUG_INIT);
|
|
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gltransformation, "gltransformation",
|
|
GST_RANK_NONE, GST_TYPE_GL_TRANSFORMATION, gl_element_init (plugin));
|
|
|
|
static void gst_gl_transformation_finalize (GObject * object);
|
|
static void gst_gl_transformation_set_property (GObject * object, guint prop_id,
|
|
const GValue * value, GParamSpec * pspec);
|
|
static void gst_gl_transformation_get_property (GObject * object, guint prop_id,
|
|
GValue * value, GParamSpec * pspec);
|
|
|
|
static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
|
|
GstCaps * incaps, GstCaps * outcaps);
|
|
static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
|
|
GstEvent * event);
|
|
static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
|
|
GstQuery * query, GType api, const GstStructure * params);
|
|
static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
|
|
trans, GstQuery * query);
|
|
|
|
static void gst_gl_transformation_gl_stop (GstGLBaseFilter * filter);
|
|
static gboolean gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter);
|
|
static gboolean gst_gl_transformation_callback (gpointer stuff);
|
|
static void gst_gl_transformation_build_mvp (GstGLTransformation *
|
|
transformation);
|
|
|
|
static GstFlowReturn
|
|
gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
|
|
GstBuffer * inbuf, GstBuffer ** outbuf);
|
|
static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
|
|
GstBuffer * inbuf, GstBuffer * outbuf);
|
|
static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
|
|
GstGLMemory * in_tex, GstGLMemory * out_tex);
|
|
|
|
static void
|
|
gst_gl_transformation_class_init (GstGLTransformationClass * klass)
|
|
{
|
|
GObjectClass *gobject_class;
|
|
GstElementClass *element_class;
|
|
GstBaseTransformClass *base_transform_class;
|
|
|
|
gobject_class = (GObjectClass *) klass;
|
|
element_class = GST_ELEMENT_CLASS (klass);
|
|
base_transform_class = GST_BASE_TRANSFORM_CLASS (klass);
|
|
|
|
gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
|
|
|
|
gobject_class->set_property = gst_gl_transformation_set_property;
|
|
gobject_class->get_property = gst_gl_transformation_get_property;
|
|
|
|
base_transform_class->src_event = gst_gl_transformation_src_event;
|
|
base_transform_class->decide_allocation =
|
|
gst_gl_transformation_decide_allocation;
|
|
base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
|
|
|
|
GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_transformation_gl_start;
|
|
GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_transformation_gl_stop;
|
|
|
|
GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
|
|
GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
|
|
GST_GL_FILTER_CLASS (klass)->filter_texture =
|
|
gst_gl_transformation_filter_texture;
|
|
GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
|
|
gst_gl_transformation_prepare_output_buffer;
|
|
|
|
g_object_class_install_property (gobject_class, PROP_FOV,
|
|
g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
|
|
0.0, G_MAXFLOAT, 90.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_ORTHO,
|
|
g_param_spec_boolean ("ortho", "Orthographic",
|
|
"Use orthographic projection", FALSE,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
/* Rotation */
|
|
g_object_class_install_property (gobject_class, PROP_ROTATION_X,
|
|
g_param_spec_float ("rotation-x", "X Rotation",
|
|
"Rotates the video around the X-Axis in degrees.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
|
|
g_param_spec_float ("rotation-y", "Y Rotation",
|
|
"Rotates the video around the Y-Axis in degrees.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
|
|
g_param_spec_float ("rotation-z", "Z Rotation",
|
|
"Rotates the video around the Z-Axis in degrees.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
/* Translation */
|
|
g_object_class_install_property (gobject_class, PROP_TRANSLATION_X,
|
|
g_param_spec_float ("translation-x", "X Translation",
|
|
"Translates the video at the X-Axis, in universal [0-1] coordinate.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_TRANSLATION_Y,
|
|
g_param_spec_float ("translation-y", "Y Translation",
|
|
"Translates the video at the Y-Axis, in universal [0-1] coordinate.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_TRANSLATION_Z,
|
|
g_param_spec_float ("translation-z", "Z Translation",
|
|
"Translates the video at the Z-Axis, in universal [0-1] coordinate.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
/* Scale */
|
|
g_object_class_install_property (gobject_class, PROP_SCALE_X,
|
|
g_param_spec_float ("scale-x", "X Scale",
|
|
"Scale multiplier for the X-Axis.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 1.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_SCALE_Y,
|
|
g_param_spec_float ("scale-y", "Y Scale",
|
|
"Scale multiplier for the Y-Axis.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 1.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
/* Pivot */
|
|
g_object_class_install_property (gobject_class, PROP_PIVOT_X,
|
|
g_param_spec_float ("pivot-x", "X Pivot",
|
|
"Rotation pivot point X coordinate, where 0 is the center,"
|
|
" -1 the left border, +1 the right border and <-1, >1 outside.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_PIVOT_Y,
|
|
g_param_spec_float ("pivot-y", "Y Pivot",
|
|
"Rotation pivot point X coordinate, where 0 is the center,"
|
|
" -1 the left border, +1 the right border and <-1, >1 outside.",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
g_object_class_install_property (gobject_class, PROP_PIVOT_Z,
|
|
g_param_spec_float ("pivot-z", "Z Pivot",
|
|
"Relevant for rotation in 3D space. You look into the negative Z axis direction",
|
|
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
|
|
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
/* MVP */
|
|
g_object_class_install_property (gobject_class, PROP_MVP,
|
|
g_param_spec_boxed ("mvp-matrix",
|
|
"Modelview Projection Matrix",
|
|
"The final Graphene 4x4 Matrix for transformation",
|
|
GRAPHENE_TYPE_MATRIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
|
|
|
|
gst_element_class_set_metadata (element_class, "OpenGL transformation filter",
|
|
"Filter/Effect/Video", "Transform video on the GPU",
|
|
"Lubosz Sarnecki <lubosz@gmail.com>\n"
|
|
"Matthew Waters <matthew@centricular.com>");
|
|
|
|
GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
|
|
GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
|
|
|
|
gobject_class->finalize = gst_gl_transformation_finalize;
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_init (GstGLTransformation * filter)
|
|
{
|
|
filter->shader = NULL;
|
|
filter->fov = 90;
|
|
filter->aspect = 1.0;
|
|
filter->znear = 0.1f;
|
|
filter->zfar = 100.0;
|
|
|
|
filter->xscale = 1.0;
|
|
filter->yscale = 1.0;
|
|
|
|
filter->in_tex = 0;
|
|
|
|
/* let graphene alloc its structures with correct memory alignment, otherwise
|
|
* unaligned memory access faults can occur */
|
|
filter->model_matrix = graphene_matrix_alloc ();
|
|
filter->view_matrix = graphene_matrix_alloc ();
|
|
filter->projection_matrix = graphene_matrix_alloc ();
|
|
filter->inv_model_matrix = graphene_matrix_alloc ();
|
|
filter->inv_view_matrix = graphene_matrix_alloc ();
|
|
filter->inv_projection_matrix = graphene_matrix_alloc ();
|
|
filter->mvp_matrix = graphene_matrix_alloc ();
|
|
|
|
filter->camera_position = graphene_vec3_alloc ();
|
|
|
|
gst_gl_transformation_build_mvp (filter);
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_finalize (GObject * object)
|
|
{
|
|
GstGLTransformation *transformation;
|
|
|
|
g_return_if_fail (GST_IS_GL_TRANSFORMATION (object));
|
|
|
|
transformation = GST_GL_TRANSFORMATION (object);
|
|
|
|
graphene_matrix_free (transformation->model_matrix);
|
|
graphene_matrix_free (transformation->view_matrix);
|
|
graphene_matrix_free (transformation->projection_matrix);
|
|
graphene_matrix_free (transformation->inv_model_matrix);
|
|
graphene_matrix_free (transformation->inv_view_matrix);
|
|
graphene_matrix_free (transformation->inv_projection_matrix);
|
|
graphene_matrix_free (transformation->mvp_matrix);
|
|
|
|
graphene_vec3_free (transformation->camera_position);
|
|
|
|
G_OBJECT_CLASS (parent_class)->finalize (object);
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_build_mvp (GstGLTransformation * transformation)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (transformation);
|
|
graphene_matrix_t modelview_matrix;
|
|
|
|
if (!filter->out_info.finfo) {
|
|
graphene_matrix_init_identity (transformation->model_matrix);
|
|
graphene_matrix_init_identity (transformation->view_matrix);
|
|
graphene_matrix_init_identity (transformation->projection_matrix);
|
|
} else {
|
|
graphene_point3d_t translation_vector =
|
|
GRAPHENE_POINT3D_INIT (transformation->xtranslation * 2.0 *
|
|
transformation->aspect,
|
|
transformation->ytranslation * 2.0,
|
|
transformation->ztranslation * 2.0);
|
|
|
|
graphene_point3d_t pivot_vector =
|
|
GRAPHENE_POINT3D_INIT (-transformation->xpivot * transformation->aspect,
|
|
transformation->ypivot,
|
|
-transformation->zpivot);
|
|
|
|
graphene_point3d_t negative_pivot_vector;
|
|
|
|
graphene_vec3_t center;
|
|
graphene_vec3_t up;
|
|
|
|
gboolean current_passthrough;
|
|
gboolean passthrough;
|
|
|
|
graphene_vec3_init (transformation->camera_position, 0.f, 0.f, 1.f);
|
|
graphene_vec3_init (¢er, 0.f, 0.f, 0.f);
|
|
graphene_vec3_init (&up, 0.f, 1.f, 0.f);
|
|
|
|
/* Translate into pivot origin */
|
|
graphene_matrix_init_translate (transformation->model_matrix,
|
|
&pivot_vector);
|
|
|
|
/* Scale */
|
|
graphene_matrix_scale (transformation->model_matrix,
|
|
transformation->xscale, transformation->yscale, 1.0f);
|
|
|
|
/* Rotation */
|
|
graphene_matrix_rotate (transformation->model_matrix,
|
|
transformation->xrotation, graphene_vec3_x_axis ());
|
|
graphene_matrix_rotate (transformation->model_matrix,
|
|
transformation->yrotation, graphene_vec3_y_axis ());
|
|
graphene_matrix_rotate (transformation->model_matrix,
|
|
transformation->zrotation, graphene_vec3_z_axis ());
|
|
|
|
/* Translate back from pivot origin */
|
|
graphene_point3d_scale (&pivot_vector, -1.0, &negative_pivot_vector);
|
|
graphene_matrix_translate (transformation->model_matrix,
|
|
&negative_pivot_vector);
|
|
|
|
/* Translation */
|
|
graphene_matrix_translate (transformation->model_matrix,
|
|
&translation_vector);
|
|
|
|
if (transformation->ortho) {
|
|
graphene_matrix_init_ortho (transformation->projection_matrix,
|
|
-transformation->aspect, transformation->aspect,
|
|
-1, 1, transformation->znear, transformation->zfar);
|
|
} else {
|
|
graphene_matrix_init_perspective (transformation->projection_matrix,
|
|
transformation->fov,
|
|
transformation->aspect, transformation->znear, transformation->zfar);
|
|
}
|
|
|
|
graphene_matrix_init_look_at (transformation->view_matrix,
|
|
transformation->camera_position, ¢er, &up);
|
|
|
|
current_passthrough =
|
|
gst_base_transform_is_passthrough (GST_BASE_TRANSFORM (transformation));
|
|
passthrough = transformation->xtranslation == 0.
|
|
&& transformation->ytranslation == 0.
|
|
&& transformation->ztranslation == 0. && transformation->xrotation == 0.
|
|
&& transformation->yrotation == 0. && transformation->zrotation == 0.
|
|
&& transformation->xscale == 1. && transformation->yscale == 1.
|
|
&& gst_video_info_is_equal (&filter->in_info, &filter->out_info);
|
|
gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (transformation),
|
|
passthrough);
|
|
if (current_passthrough != passthrough) {
|
|
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (transformation));
|
|
}
|
|
}
|
|
|
|
graphene_matrix_multiply (transformation->model_matrix,
|
|
transformation->view_matrix, &modelview_matrix);
|
|
graphene_matrix_multiply (&modelview_matrix,
|
|
transformation->projection_matrix, transformation->mvp_matrix);
|
|
|
|
graphene_matrix_inverse (transformation->model_matrix,
|
|
transformation->inv_model_matrix);
|
|
graphene_matrix_inverse (transformation->view_matrix,
|
|
transformation->inv_view_matrix);
|
|
graphene_matrix_inverse (transformation->projection_matrix,
|
|
transformation->inv_projection_matrix);
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_set_property (GObject * object, guint prop_id,
|
|
const GValue * value, GParamSpec * pspec)
|
|
{
|
|
GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
|
|
|
|
switch (prop_id) {
|
|
case PROP_FOV:
|
|
filter->fov = g_value_get_float (value);
|
|
break;
|
|
case PROP_ORTHO:
|
|
filter->ortho = g_value_get_boolean (value);
|
|
break;
|
|
case PROP_TRANSLATION_X:
|
|
filter->xtranslation = g_value_get_float (value);
|
|
break;
|
|
case PROP_TRANSLATION_Y:
|
|
filter->ytranslation = g_value_get_float (value);
|
|
break;
|
|
case PROP_TRANSLATION_Z:
|
|
filter->ztranslation = g_value_get_float (value);
|
|
break;
|
|
case PROP_ROTATION_X:
|
|
filter->xrotation = g_value_get_float (value);
|
|
break;
|
|
case PROP_ROTATION_Y:
|
|
filter->yrotation = g_value_get_float (value);
|
|
break;
|
|
case PROP_ROTATION_Z:
|
|
filter->zrotation = g_value_get_float (value);
|
|
break;
|
|
case PROP_SCALE_X:
|
|
filter->xscale = g_value_get_float (value);
|
|
break;
|
|
case PROP_SCALE_Y:
|
|
filter->yscale = g_value_get_float (value);
|
|
break;
|
|
case PROP_PIVOT_X:
|
|
filter->xpivot = g_value_get_float (value);
|
|
break;
|
|
case PROP_PIVOT_Y:
|
|
filter->ypivot = g_value_get_float (value);
|
|
break;
|
|
case PROP_PIVOT_Z:
|
|
filter->zpivot = g_value_get_float (value);
|
|
break;
|
|
default:
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
break;
|
|
}
|
|
gst_gl_transformation_build_mvp (filter);
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_get_property (GObject * object, guint prop_id,
|
|
GValue * value, GParamSpec * pspec)
|
|
{
|
|
GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
|
|
|
|
switch (prop_id) {
|
|
case PROP_FOV:
|
|
g_value_set_float (value, filter->fov);
|
|
break;
|
|
case PROP_ORTHO:
|
|
g_value_set_boolean (value, filter->ortho);
|
|
break;
|
|
case PROP_TRANSLATION_X:
|
|
g_value_set_float (value, filter->xtranslation);
|
|
break;
|
|
case PROP_TRANSLATION_Y:
|
|
g_value_set_float (value, filter->ytranslation);
|
|
break;
|
|
case PROP_TRANSLATION_Z:
|
|
g_value_set_float (value, filter->ztranslation);
|
|
break;
|
|
case PROP_ROTATION_X:
|
|
g_value_set_float (value, filter->xrotation);
|
|
break;
|
|
case PROP_ROTATION_Y:
|
|
g_value_set_float (value, filter->yrotation);
|
|
break;
|
|
case PROP_ROTATION_Z:
|
|
g_value_set_float (value, filter->zrotation);
|
|
break;
|
|
case PROP_SCALE_X:
|
|
g_value_set_float (value, filter->xscale);
|
|
break;
|
|
case PROP_SCALE_Y:
|
|
g_value_set_float (value, filter->yscale);
|
|
break;
|
|
case PROP_PIVOT_X:
|
|
g_value_set_float (value, filter->xpivot);
|
|
break;
|
|
case PROP_PIVOT_Y:
|
|
g_value_set_float (value, filter->ypivot);
|
|
break;
|
|
case PROP_PIVOT_Z:
|
|
g_value_set_float (value, filter->zpivot);
|
|
break;
|
|
case PROP_MVP:
|
|
/* FIXME: need to decompose this to support navigation events */
|
|
g_value_set_boxed (value, (gconstpointer) filter->mvp_matrix);
|
|
break;
|
|
default:
|
|
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
|
|
break;
|
|
}
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_set_caps (GstGLFilter * filter, GstCaps * incaps,
|
|
GstCaps * outcaps)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
|
|
|
|
transformation->aspect =
|
|
(gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
|
|
(gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info);
|
|
|
|
transformation->caps_change = TRUE;
|
|
|
|
gst_gl_transformation_build_mvp (transformation);
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
static void
|
|
_intersect_plane_and_ray (graphene_plane_t * video_plane, graphene_ray_t * ray,
|
|
graphene_point3d_t * result)
|
|
{
|
|
float t = graphene_ray_get_distance_to_plane (ray, video_plane);
|
|
GST_TRACE ("Calculated a distance of %f to the plane", t);
|
|
graphene_ray_get_position_at (ray, t, result);
|
|
}
|
|
|
|
static void
|
|
_screen_coord_to_world_ray (GstGLTransformation * transformation, float x,
|
|
float y, graphene_ray_t * ray)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (transformation);
|
|
gfloat w = (gfloat) GST_VIDEO_INFO_WIDTH (&filter->in_info);
|
|
gfloat h = (gfloat) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
|
|
graphene_vec3_t ray_eye_vec3, ray_world_dir, *ray_origin, *ray_direction;
|
|
graphene_vec3_t ray_ortho_dir;
|
|
graphene_point3d_t ray_clip, ray_eye;
|
|
graphene_vec2_t screen_coord;
|
|
|
|
/* GL is y-flipped. i.e. 0, 0 is the bottom left corner in screen space */
|
|
graphene_vec2_init (&screen_coord, (2. * x / w - 1.) / transformation->aspect,
|
|
1. - 2. * y / h);
|
|
|
|
graphene_point3d_init (&ray_clip, graphene_vec2_get_x (&screen_coord),
|
|
graphene_vec2_get_y (&screen_coord), -1.);
|
|
graphene_matrix_transform_point3d (transformation->inv_projection_matrix,
|
|
&ray_clip, &ray_eye);
|
|
|
|
graphene_vec3_init (&ray_eye_vec3, ray_eye.x, ray_eye.y, -1.);
|
|
|
|
if (transformation->ortho) {
|
|
graphene_vec3_init (&ray_ortho_dir, 0., 0., 1.);
|
|
|
|
ray_origin = &ray_eye_vec3;
|
|
ray_direction = &ray_ortho_dir;
|
|
} else {
|
|
graphene_matrix_transform_vec3 (transformation->inv_view_matrix,
|
|
&ray_eye_vec3, &ray_world_dir);
|
|
graphene_vec3_normalize (&ray_world_dir, &ray_world_dir);
|
|
|
|
ray_origin = transformation->camera_position;
|
|
ray_direction = &ray_world_dir;
|
|
}
|
|
|
|
graphene_ray_init_from_vec3 (ray, ray_origin, ray_direction);
|
|
|
|
GST_TRACE_OBJECT (transformation, "Calculated ray origin: " VEC3_FORMAT
|
|
" direction: " VEC3_FORMAT " from screen coordinates: " VEC2_FORMAT
|
|
" with %s projection",
|
|
VEC3_ARGS (ray_origin), VEC3_ARGS (ray_direction),
|
|
VEC2_ARGS (&screen_coord),
|
|
transformation->ortho ? "ortho" : "perspection");
|
|
}
|
|
|
|
static void
|
|
_init_world_video_plane (GstGLTransformation * transformation,
|
|
graphene_plane_t * video_plane)
|
|
{
|
|
graphene_point3d_t bottom_left, bottom_right, top_left, top_right;
|
|
graphene_point3d_t world_bottom_left, world_bottom_right;
|
|
graphene_point3d_t world_top_left, world_top_right;
|
|
|
|
graphene_point3d_init (&top_left, -transformation->aspect, 1., 0.);
|
|
graphene_point3d_init (&top_right, transformation->aspect, 1., 0.);
|
|
graphene_point3d_init (&bottom_left, -transformation->aspect, -1., 0.);
|
|
graphene_point3d_init (&bottom_right, transformation->aspect, -1., 0.);
|
|
|
|
graphene_matrix_transform_point3d (transformation->model_matrix,
|
|
&bottom_left, &world_bottom_left);
|
|
graphene_matrix_transform_point3d (transformation->model_matrix,
|
|
&bottom_right, &world_bottom_right);
|
|
graphene_matrix_transform_point3d (transformation->model_matrix,
|
|
&top_left, &world_top_left);
|
|
graphene_matrix_transform_point3d (transformation->model_matrix,
|
|
&top_right, &world_top_right);
|
|
|
|
graphene_plane_init_from_points (video_plane, &world_bottom_left,
|
|
&world_top_right, &world_top_left);
|
|
}
|
|
|
|
static gboolean
|
|
_screen_coord_to_model_coord (GstGLTransformation * transformation,
|
|
double x, double y, double *res_x, double *res_y)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (transformation);
|
|
double w = (double) GST_VIDEO_INFO_WIDTH (&filter->in_info);
|
|
double h = (double) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
|
|
graphene_point3d_t world_point, model_coord;
|
|
graphene_plane_t video_plane;
|
|
graphene_ray_t ray;
|
|
double new_x, new_y;
|
|
|
|
_init_world_video_plane (transformation, &video_plane);
|
|
_screen_coord_to_world_ray (transformation, x, y, &ray);
|
|
_intersect_plane_and_ray (&video_plane, &ray, &world_point);
|
|
graphene_matrix_transform_point3d (transformation->inv_model_matrix,
|
|
&world_point, &model_coord);
|
|
|
|
/* ndc to pixels. We render the frame Y-flipped so need to unflip the
|
|
* y coordinate */
|
|
new_x = (model_coord.x + 1.) * w / 2;
|
|
new_y = (1. - model_coord.y) * h / 2;
|
|
|
|
if (new_x < 0. || new_x > w || new_y < 0. || new_y > h)
|
|
/* coords off video surface */
|
|
return FALSE;
|
|
|
|
GST_DEBUG_OBJECT (transformation, "converted %f,%f to %f,%f", x, y, new_x,
|
|
new_y);
|
|
|
|
if (res_x)
|
|
*res_x = new_x;
|
|
if (res_y)
|
|
*res_y = new_y;
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
#if 0
|
|
/* debugging facilities for transforming vertices from model space to screen
|
|
* space */
|
|
static void
|
|
_ndc_to_viewport (GstGLTransformation * transformation, graphene_vec3_t * ndc,
|
|
int x, int y, int w, int h, float near, float far, graphene_vec3_t * result)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (transformation);
|
|
/* center of the viewport */
|
|
int o_x = x + w / 2;
|
|
int o_y = y + h / 2;
|
|
|
|
graphene_vec3_init (result, graphene_vec3_get_x (ndc) * w / 2 + o_x,
|
|
graphene_vec3_get_y (ndc) * h / 2 + o_y,
|
|
(far - near) * graphene_vec3_get_z (ndc) / 2 + (far + near) / 2);
|
|
}
|
|
|
|
static void
|
|
_perspective_division (graphene_vec4_t * clip, graphene_vec3_t * result)
|
|
{
|
|
float w = graphene_vec4_get_w (clip);
|
|
|
|
graphene_vec3_init (result, graphene_vec4_get_x (clip) / w,
|
|
graphene_vec4_get_y (clip) / w, graphene_vec4_get_z (clip) / w);
|
|
}
|
|
|
|
static void
|
|
_vertex_to_screen_coord (GstGLTransformation * transformation,
|
|
graphene_vec4_t * vertex, graphene_vec3_t * view)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (transformation);
|
|
gint w = GST_VIDEO_INFO_WIDTH (&filter->in_info);
|
|
gint h = GST_VIDEO_INFO_HEIGHT (&filter->in_info);
|
|
graphene_vec4_t clip;
|
|
graphene_vec3_t ndc;
|
|
|
|
graphene_matrix_transform_vec4 (transformation->mvp_matrix, vertex, &clip);
|
|
_perspective_division (&clip, &ndc);
|
|
_ndc_to_viewport (transformation, &ndc, 0, 0, w, h, 0., 1., view);
|
|
}
|
|
#endif
|
|
|
|
static gboolean
|
|
gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
|
|
gboolean ret;
|
|
|
|
GST_DEBUG_OBJECT (trans, "handling %s event", GST_EVENT_TYPE_NAME (event));
|
|
|
|
switch (GST_EVENT_TYPE (event)) {
|
|
case GST_EVENT_NAVIGATION:{
|
|
gdouble x, y;
|
|
event = gst_event_make_writable (event);
|
|
|
|
if (gst_navigation_event_get_coordinates (event, &x, &y)) {
|
|
gdouble new_x, new_y;
|
|
|
|
if (!_screen_coord_to_model_coord (transformation, x, y, &new_x,
|
|
&new_y)) {
|
|
gst_event_unref (event);
|
|
return TRUE;
|
|
}
|
|
|
|
gst_navigation_event_set_coordinates (event, x, y);
|
|
}
|
|
break;
|
|
}
|
|
default:
|
|
break;
|
|
}
|
|
|
|
ret = GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (trans, event);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
|
|
GType api, const GstStructure * params)
|
|
{
|
|
if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
|
|
return TRUE;
|
|
|
|
if (api == GST_GL_SYNC_META_API_TYPE)
|
|
return TRUE;
|
|
|
|
return FALSE;
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
|
|
GstQuery * query)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
|
|
|
|
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
|
|
query))
|
|
return FALSE;
|
|
|
|
if (gst_query_find_allocation_meta (query,
|
|
GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
|
|
transformation->downstream_supports_affine_meta = TRUE;
|
|
} else {
|
|
transformation->downstream_supports_affine_meta = FALSE;
|
|
}
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
static void
|
|
gst_gl_transformation_gl_stop (GstGLBaseFilter * base_filter)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
|
|
const GstGLFuncs *gl = base_filter->context->gl_vtable;
|
|
|
|
if (transformation->vao) {
|
|
gl->DeleteVertexArrays (1, &transformation->vao);
|
|
transformation->vao = 0;
|
|
}
|
|
|
|
if (transformation->vertex_buffer) {
|
|
gl->DeleteBuffers (1, &transformation->vertex_buffer);
|
|
transformation->vertex_buffer = 0;
|
|
}
|
|
|
|
if (transformation->vbo_indices) {
|
|
gl->DeleteBuffers (1, &transformation->vbo_indices);
|
|
transformation->vbo_indices = 0;
|
|
}
|
|
|
|
if (transformation->shader) {
|
|
gst_object_unref (transformation->shader);
|
|
transformation->shader = NULL;
|
|
}
|
|
|
|
GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
|
|
|
|
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
|
|
return FALSE;
|
|
|
|
if (gst_gl_context_get_gl_api (base_filter->context)) {
|
|
gchar *frag_str;
|
|
gboolean ret;
|
|
|
|
frag_str =
|
|
gst_gl_shader_string_fragment_get_default (base_filter->context,
|
|
GST_GLSL_VERSION_NONE,
|
|
GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
|
|
|
|
/* blocking call, wait until the opengl thread has compiled the shader */
|
|
ret = gst_gl_context_gen_shader (base_filter->context,
|
|
gst_gl_shader_string_vertex_mat4_vertex_transform,
|
|
frag_str, &transformation->shader);
|
|
|
|
g_free (frag_str);
|
|
|
|
return ret;
|
|
}
|
|
return TRUE;
|
|
}
|
|
|
|
static GstFlowReturn
|
|
gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
|
|
GstBuffer * inbuf, GstBuffer ** outbuf)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
|
|
GstGLFilter *filter = GST_GL_FILTER (trans);
|
|
|
|
if (transformation->downstream_supports_affine_meta &&
|
|
gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
|
|
GstVideoAffineTransformationMeta *af_meta;
|
|
graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
|
|
float upstream[16], downstream[16];
|
|
|
|
*outbuf = gst_buffer_make_writable (inbuf);
|
|
|
|
af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
|
|
if (!af_meta)
|
|
af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
|
|
|
|
GST_LOG_OBJECT (trans, "applying transformation to existing affine "
|
|
"transformation meta");
|
|
|
|
gst_gl_get_affine_transformation_meta_as_ndc (af_meta, upstream);
|
|
|
|
/* apply the transformation to the existing affine meta */
|
|
graphene_matrix_init_from_float (&upstream_matrix, upstream);
|
|
graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
|
|
graphene_matrix_init_scale (&yflip, 1., -1., 1.);
|
|
|
|
/* invert the aspect effects */
|
|
graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
|
|
/* apply the transformation */
|
|
graphene_matrix_multiply (&tmp2, transformation->mvp_matrix, &tmp);
|
|
/* and undo yflip */
|
|
graphene_matrix_multiply (&tmp, &yflip, &tmp2);
|
|
|
|
graphene_matrix_to_float (&tmp2, downstream);
|
|
gst_gl_set_affine_transformation_meta_from_ndc (af_meta, downstream);
|
|
|
|
return GST_FLOW_OK;
|
|
}
|
|
|
|
return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
|
|
inbuf, outbuf);
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_filter (GstGLFilter * filter,
|
|
GstBuffer * inbuf, GstBuffer * outbuf)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
|
|
|
|
if (transformation->downstream_supports_affine_meta &&
|
|
gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
|
|
return TRUE;
|
|
} else {
|
|
return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
|
|
}
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_filter_texture (GstGLFilter * filter,
|
|
GstGLMemory * in_tex, GstGLMemory * out_tex)
|
|
{
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
|
|
|
|
transformation->in_tex = in_tex;
|
|
|
|
gst_gl_framebuffer_draw_to_texture (filter->fbo, out_tex,
|
|
(GstGLFramebufferFunc) gst_gl_transformation_callback, transformation);
|
|
|
|
return TRUE;
|
|
}
|
|
|
|
static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
|
|
|
|
static void
|
|
_upload_vertices (GstGLTransformation * transformation)
|
|
{
|
|
const GstGLFuncs *gl =
|
|
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
|
|
|
|
/* *INDENT-OFF* */
|
|
GLfloat vertices[] = {
|
|
-transformation->aspect, -1.0, 0.0, 1.0, 0.0, 0.0,
|
|
transformation->aspect, -1.0, 0.0, 1.0, 1.0, 0.0,
|
|
transformation->aspect, 1.0, 0.0, 1.0, 1.0, 1.0,
|
|
-transformation->aspect, 1.0, 0.0, 1.0, 0.0, 1.0,
|
|
};
|
|
/* *INDENT-ON* */
|
|
|
|
gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
|
|
|
|
gl->BufferData (GL_ARRAY_BUFFER, 4 * 6 * sizeof (GLfloat), vertices,
|
|
GL_STATIC_DRAW);
|
|
}
|
|
|
|
static void
|
|
_bind_buffer (GstGLTransformation * transformation)
|
|
{
|
|
const GstGLFuncs *gl =
|
|
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
|
|
|
|
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
|
|
gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
|
|
|
|
/* Load the vertex position */
|
|
gl->VertexAttribPointer (transformation->attr_position, 4, GL_FLOAT,
|
|
GL_FALSE, 6 * sizeof (GLfloat), (void *) 0);
|
|
|
|
/* Load the texture coordinate */
|
|
gl->VertexAttribPointer (transformation->attr_texture, 2, GL_FLOAT, GL_FALSE,
|
|
6 * sizeof (GLfloat), (void *) (4 * sizeof (GLfloat)));
|
|
|
|
gl->EnableVertexAttribArray (transformation->attr_position);
|
|
gl->EnableVertexAttribArray (transformation->attr_texture);
|
|
}
|
|
|
|
static void
|
|
_unbind_buffer (GstGLTransformation * transformation)
|
|
{
|
|
const GstGLFuncs *gl =
|
|
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
|
|
|
|
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
|
|
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
|
|
|
|
gl->DisableVertexAttribArray (transformation->attr_position);
|
|
gl->DisableVertexAttribArray (transformation->attr_texture);
|
|
}
|
|
|
|
static gboolean
|
|
gst_gl_transformation_callback (gpointer stuff)
|
|
{
|
|
GstGLFilter *filter = GST_GL_FILTER (stuff);
|
|
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
|
|
GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
|
|
|
|
GLfloat temp_matrix[16];
|
|
|
|
gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
|
|
gl->BindTexture (GL_TEXTURE_2D, 0);
|
|
|
|
gl->ClearColor (0.f, 0.f, 0.f, 0.f);
|
|
gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
|
|
|
|
gst_gl_shader_use (transformation->shader);
|
|
|
|
gl->ActiveTexture (GL_TEXTURE0);
|
|
gl->BindTexture (GL_TEXTURE_2D, transformation->in_tex->tex_id);
|
|
gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
|
|
|
|
graphene_matrix_to_float (transformation->mvp_matrix, temp_matrix);
|
|
gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
|
|
"u_transformation", 1, GL_FALSE, temp_matrix);
|
|
|
|
if (!transformation->vertex_buffer) {
|
|
transformation->attr_position =
|
|
gst_gl_shader_get_attribute_location (transformation->shader,
|
|
"a_position");
|
|
|
|
transformation->attr_texture =
|
|
gst_gl_shader_get_attribute_location (transformation->shader,
|
|
"a_texcoord");
|
|
|
|
if (gl->GenVertexArrays) {
|
|
gl->GenVertexArrays (1, &transformation->vao);
|
|
gl->BindVertexArray (transformation->vao);
|
|
}
|
|
|
|
gl->GenBuffers (1, &transformation->vertex_buffer);
|
|
|
|
gl->GenBuffers (1, &transformation->vbo_indices);
|
|
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
|
|
gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
|
|
GL_STATIC_DRAW);
|
|
|
|
transformation->caps_change = TRUE;
|
|
}
|
|
|
|
if (gl->GenVertexArrays)
|
|
gl->BindVertexArray (transformation->vao);
|
|
|
|
if (transformation->caps_change)
|
|
_upload_vertices (transformation);
|
|
_bind_buffer (transformation);
|
|
|
|
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
|
|
|
|
if (gl->GenVertexArrays)
|
|
gl->BindVertexArray (0);
|
|
else
|
|
_unbind_buffer (transformation);
|
|
|
|
gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
|
|
transformation->caps_change = FALSE;
|
|
|
|
return TRUE;
|
|
}
|