mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-07-07 07:05:52 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/subprojects/gst-plugins-base/ext/gl/gstgltransformation.c
Vivienne Watermeier 6c2f6c3bd4 all: Use new navigation interface and API 
Use and implement the new navigation interface in all relevant sink elements,
and use API functions everywhere instead of directy accessing the event structure.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1633>
2022-03-23 13:14:52 +00:00

982 lines
33 KiB
C
Raw Blame History

/*
* GStreamer
* Copyright (C) 2014 Lubosz Sarnecki <lubosz@gmail.com>
* Copyright (C) 2016 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-gltransformation
* @title: gltransformation
*
* Transforms video on the GPU.
*
* ## Examples
* |[
* gst-launch-1.0 gltestsrc ! gltransformation rotation-z=45 ! glimagesink
* ]| A pipeline to rotate by 45 degrees
* |[
* gst-launch-1.0 gltestsrc ! gltransformation translation-x=0.5 ! glimagesink
* ]| Translate the video by 0.5
* |[
* gst-launch-1.0 gltestsrc ! gltransformation scale-y=0.5 scale-x=0.5 ! glimagesink
* ]| Resize the video by 0.5
* |[
* gst-launch-1.0 gltestsrc ! gltransformation rotation-x=-45 ortho=True ! glimagesink
* ]| Rotate the video around the X-Axis by -45° with an orthographic projection
*
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstglelements.h"
#include "gstgltransformation.h"
#include <gst/gl/gstglapi.h>
#include <graphene-gobject.h>
#include "gstglutils.h"
#define GST_CAT_DEFAULT gst_gl_transformation_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
#define gst_gl_transformation_parent_class parent_class
#define VEC4_FORMAT "%f,%f,%f,%f"
#define VEC4_ARGS(v) graphene_vec4_get_x (v), graphene_vec4_get_y (v), graphene_vec4_get_z (v), graphene_vec4_get_w (v)
#define VEC3_FORMAT "%f,%f,%f"
#define VEC3_ARGS(v) graphene_vec3_get_x (v), graphene_vec3_get_y (v), graphene_vec3_get_z (v)
#define VEC2_FORMAT "%f,%f"
#define VEC2_ARGS(v) graphene_vec2_get_x (v), graphene_vec2_get_y (v)
#define POINT3D_FORMAT "%f,%f,%f"
#define POINT3D_ARGS(p) (p)->x, (p)->y, (p)->z
enum
{
PROP_0,
PROP_FOV,
PROP_ORTHO,
PROP_TRANSLATION_X,
PROP_TRANSLATION_Y,
PROP_TRANSLATION_Z,
PROP_ROTATION_X,
PROP_ROTATION_Y,
PROP_ROTATION_Z,
PROP_SCALE_X,
PROP_SCALE_Y,
PROP_MVP,
PROP_PIVOT_X,
PROP_PIVOT_Y,
PROP_PIVOT_Z,
};
#define DEBUG_INIT \
GST_DEBUG_CATEGORY_INIT (gst_gl_transformation_debug, "gltransformation", 0, "gltransformation element");
G_DEFINE_TYPE_WITH_CODE (GstGLTransformation, gst_gl_transformation,
GST_TYPE_GL_FILTER, DEBUG_INIT);
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gltransformation, "gltransformation",
GST_RANK_NONE, GST_TYPE_GL_TRANSFORMATION, gl_element_init (plugin));
static void gst_gl_transformation_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static void gst_gl_transformation_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static gboolean gst_gl_transformation_set_caps (GstGLFilter * filter,
GstCaps * incaps, GstCaps * outcaps);
static gboolean gst_gl_transformation_src_event (GstBaseTransform * trans,
GstEvent * event);
static gboolean gst_gl_transformation_filter_meta (GstBaseTransform * trans,
GstQuery * query, GType api, const GstStructure * params);
static gboolean gst_gl_transformation_decide_allocation (GstBaseTransform *
trans, GstQuery * query);
static void gst_gl_transformation_gl_stop (GstGLBaseFilter * filter);
static gboolean gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter);
static gboolean gst_gl_transformation_callback (gpointer stuff);
static void gst_gl_transformation_build_mvp (GstGLTransformation *
transformation);
static GstFlowReturn
gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
GstBuffer * inbuf, GstBuffer ** outbuf);
static gboolean gst_gl_transformation_filter (GstGLFilter * filter,
GstBuffer * inbuf, GstBuffer * outbuf);
static gboolean gst_gl_transformation_filter_texture (GstGLFilter * filter,
GstGLMemory * in_tex, GstGLMemory * out_tex);
static void
gst_gl_transformation_class_init (GstGLTransformationClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *element_class;
GstBaseTransformClass *base_transform_class;
gobject_class = (GObjectClass *) klass;
element_class = GST_ELEMENT_CLASS (klass);
base_transform_class = GST_BASE_TRANSFORM_CLASS (klass);
gst_gl_filter_add_rgba_pad_templates (GST_GL_FILTER_CLASS (klass));
gobject_class->set_property = gst_gl_transformation_set_property;
gobject_class->get_property = gst_gl_transformation_get_property;
base_transform_class->src_event = gst_gl_transformation_src_event;
base_transform_class->decide_allocation =
gst_gl_transformation_decide_allocation;
base_transform_class->filter_meta = gst_gl_transformation_filter_meta;
GST_GL_BASE_FILTER_CLASS (klass)->gl_start = gst_gl_transformation_gl_start;
GST_GL_BASE_FILTER_CLASS (klass)->gl_stop = gst_gl_transformation_gl_stop;
GST_GL_FILTER_CLASS (klass)->set_caps = gst_gl_transformation_set_caps;
GST_GL_FILTER_CLASS (klass)->filter = gst_gl_transformation_filter;
GST_GL_FILTER_CLASS (klass)->filter_texture =
gst_gl_transformation_filter_texture;
GST_BASE_TRANSFORM_CLASS (klass)->prepare_output_buffer =
gst_gl_transformation_prepare_output_buffer;
g_object_class_install_property (gobject_class, PROP_FOV,
g_param_spec_float ("fov", "Fov", "Field of view angle in degrees",
0.0, G_MAXFLOAT, 90.0, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_ORTHO,
g_param_spec_boolean ("ortho", "Orthographic",
"Use orthographic projection", FALSE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
/* Rotation */
g_object_class_install_property (gobject_class, PROP_ROTATION_X,
g_param_spec_float ("rotation-x", "X Rotation",
"Rotates the video around the X-Axis in degrees.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_ROTATION_Y,
g_param_spec_float ("rotation-y", "Y Rotation",
"Rotates the video around the Y-Axis in degrees.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_ROTATION_Z,
g_param_spec_float ("rotation-z", "Z Rotation",
"Rotates the video around the Z-Axis in degrees.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
/* Translation */
g_object_class_install_property (gobject_class, PROP_TRANSLATION_X,
g_param_spec_float ("translation-x", "X Translation",
"Translates the video at the X-Axis, in universal [0-1] coordinate.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_TRANSLATION_Y,
g_param_spec_float ("translation-y", "Y Translation",
"Translates the video at the Y-Axis, in universal [0-1] coordinate.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_TRANSLATION_Z,
g_param_spec_float ("translation-z", "Z Translation",
"Translates the video at the Z-Axis, in universal [0-1] coordinate.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
/* Scale */
g_object_class_install_property (gobject_class, PROP_SCALE_X,
g_param_spec_float ("scale-x", "X Scale",
"Scale multiplier for the X-Axis.",
-G_MAXFLOAT, G_MAXFLOAT, 1.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_SCALE_Y,
g_param_spec_float ("scale-y", "Y Scale",
"Scale multiplier for the Y-Axis.",
-G_MAXFLOAT, G_MAXFLOAT, 1.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
/* Pivot */
g_object_class_install_property (gobject_class, PROP_PIVOT_X,
g_param_spec_float ("pivot-x", "X Pivot",
"Rotation pivot point X coordinate, where 0 is the center,"
" -1 the left border, +1 the right border and <-1, >1 outside.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PIVOT_Y,
g_param_spec_float ("pivot-y", "Y Pivot",
"Rotation pivot point X coordinate, where 0 is the center,"
" -1 the left border, +1 the right border and <-1, >1 outside.",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_PIVOT_Z,
g_param_spec_float ("pivot-z", "Z Pivot",
"Relevant for rotation in 3D space. You look into the negative Z axis direction",
-G_MAXFLOAT, G_MAXFLOAT, 0.0,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
/* MVP */
g_object_class_install_property (gobject_class, PROP_MVP,
g_param_spec_boxed ("mvp-matrix",
"Modelview Projection Matrix",
"The final Graphene 4x4 Matrix for transformation",
GRAPHENE_TYPE_MATRIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gst_element_class_set_metadata (element_class, "OpenGL transformation filter",
"Filter/Effect/Video", "Transform video on the GPU",
"Lubosz Sarnecki <lubosz@gmail.com>\n"
"Matthew Waters <matthew@centricular.com>");
GST_GL_BASE_FILTER_CLASS (klass)->supported_gl_api =
GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2;
}
static void
gst_gl_transformation_init (GstGLTransformation * filter)
{
filter->shader = NULL;
filter->fov = 90;
filter->aspect = 1.0;
filter->znear = 0.1f;
filter->zfar = 100.0;
filter->xscale = 1.0;
filter->yscale = 1.0;
filter->in_tex = 0;
gst_gl_transformation_build_mvp (filter);
}
static void
gst_gl_transformation_build_mvp (GstGLTransformation * transformation)
{
GstGLFilter *filter = GST_GL_FILTER (transformation);
graphene_matrix_t modelview_matrix;
if (!filter->out_info.finfo) {
graphene_matrix_init_identity (&transformation->model_matrix);
graphene_matrix_init_identity (&transformation->view_matrix);
graphene_matrix_init_identity (&transformation->projection_matrix);
} else {
graphene_point3d_t translation_vector =
GRAPHENE_POINT3D_INIT (transformation->xtranslation * 2.0 *
transformation->aspect,
transformation->ytranslation * 2.0,
transformation->ztranslation * 2.0);
graphene_point3d_t pivot_vector =
GRAPHENE_POINT3D_INIT (-transformation->xpivot * transformation->aspect,
transformation->ypivot,
-transformation->zpivot);
graphene_point3d_t negative_pivot_vector;
graphene_vec3_t center;
graphene_vec3_t up;
gboolean current_passthrough;
gboolean passthrough;
graphene_vec3_init (&transformation->camera_position, 0.f, 0.f, 1.f);
graphene_vec3_init (&center, 0.f, 0.f, 0.f);
graphene_vec3_init (&up, 0.f, 1.f, 0.f);
/* Translate into pivot origin */
graphene_matrix_init_translate (&transformation->model_matrix,
&pivot_vector);
/* Scale */
graphene_matrix_scale (&transformation->model_matrix,
transformation->xscale, transformation->yscale, 1.0f);
/* Rotation */
graphene_matrix_rotate (&transformation->model_matrix,
transformation->xrotation, graphene_vec3_x_axis ());
graphene_matrix_rotate (&transformation->model_matrix,
transformation->yrotation, graphene_vec3_y_axis ());
graphene_matrix_rotate (&transformation->model_matrix,
transformation->zrotation, graphene_vec3_z_axis ());
/* Translate back from pivot origin */
graphene_point3d_scale (&pivot_vector, -1.0, &negative_pivot_vector);
graphene_matrix_translate (&transformation->model_matrix,
&negative_pivot_vector);
/* Translation */
graphene_matrix_translate (&transformation->model_matrix,
&translation_vector);
if (transformation->ortho) {
graphene_matrix_init_ortho (&transformation->projection_matrix,
-transformation->aspect, transformation->aspect,
-1, 1, transformation->znear, transformation->zfar);
} else {
graphene_matrix_init_perspective (&transformation->projection_matrix,
transformation->fov,
transformation->aspect, transformation->znear, transformation->zfar);
}
graphene_matrix_init_look_at (&transformation->view_matrix,
&transformation->camera_position, &center, &up);
current_passthrough =
gst_base_transform_is_passthrough (GST_BASE_TRANSFORM (transformation));
passthrough = transformation->xtranslation == 0.
&& transformation->ytranslation == 0.
&& transformation->ztranslation == 0. && transformation->xrotation == 0.
&& transformation->yrotation == 0. && transformation->zrotation == 0.
&& transformation->xscale == 1. && transformation->yscale == 1.
&& gst_video_info_is_equal (&filter->in_info, &filter->out_info);
gst_base_transform_set_passthrough (GST_BASE_TRANSFORM (transformation),
passthrough);
if (current_passthrough != passthrough) {
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (transformation));
}
}
graphene_matrix_multiply (&transformation->model_matrix,
&transformation->view_matrix, &modelview_matrix);
graphene_matrix_multiply (&modelview_matrix,
&transformation->projection_matrix, &transformation->mvp_matrix);
graphene_matrix_inverse (&transformation->model_matrix,
&transformation->inv_model_matrix);
graphene_matrix_inverse (&transformation->view_matrix,
&transformation->inv_view_matrix);
graphene_matrix_inverse (&transformation->projection_matrix,
&transformation->inv_projection_matrix);
}
static void
gst_gl_transformation_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
switch (prop_id) {
case PROP_FOV:
filter->fov = g_value_get_float (value);
break;
case PROP_ORTHO:
filter->ortho = g_value_get_boolean (value);
break;
case PROP_TRANSLATION_X:
filter->xtranslation = g_value_get_float (value);
break;
case PROP_TRANSLATION_Y:
filter->ytranslation = g_value_get_float (value);
break;
case PROP_TRANSLATION_Z:
filter->ztranslation = g_value_get_float (value);
break;
case PROP_ROTATION_X:
filter->xrotation = g_value_get_float (value);
break;
case PROP_ROTATION_Y:
filter->yrotation = g_value_get_float (value);
break;
case PROP_ROTATION_Z:
filter->zrotation = g_value_get_float (value);
break;
case PROP_SCALE_X:
filter->xscale = g_value_get_float (value);
break;
case PROP_SCALE_Y:
filter->yscale = g_value_get_float (value);
break;
case PROP_PIVOT_X:
filter->xpivot = g_value_get_float (value);
break;
case PROP_PIVOT_Y:
filter->ypivot = g_value_get_float (value);
break;
case PROP_PIVOT_Z:
filter->zpivot = g_value_get_float (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
gst_gl_transformation_build_mvp (filter);
}
static void
gst_gl_transformation_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstGLTransformation *filter = GST_GL_TRANSFORMATION (object);
switch (prop_id) {
case PROP_FOV:
g_value_set_float (value, filter->fov);
break;
case PROP_ORTHO:
g_value_set_boolean (value, filter->ortho);
break;
case PROP_TRANSLATION_X:
g_value_set_float (value, filter->xtranslation);
break;
case PROP_TRANSLATION_Y:
g_value_set_float (value, filter->ytranslation);
break;
case PROP_TRANSLATION_Z:
g_value_set_float (value, filter->ztranslation);
break;
case PROP_ROTATION_X:
g_value_set_float (value, filter->xrotation);
break;
case PROP_ROTATION_Y:
g_value_set_float (value, filter->yrotation);
break;
case PROP_ROTATION_Z:
g_value_set_float (value, filter->zrotation);
break;
case PROP_SCALE_X:
g_value_set_float (value, filter->xscale);
break;
case PROP_SCALE_Y:
g_value_set_float (value, filter->yscale);
break;
case PROP_PIVOT_X:
g_value_set_float (value, filter->xpivot);
break;
case PROP_PIVOT_Y:
g_value_set_float (value, filter->ypivot);
break;
case PROP_PIVOT_Z:
g_value_set_float (value, filter->zpivot);
break;
case PROP_MVP:
/* FIXME: need to decompose this to support navigation events */
g_value_set_boxed (value, (gconstpointer) & filter->mvp_matrix);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_gl_transformation_set_caps (GstGLFilter * filter, GstCaps * incaps,
GstCaps * outcaps)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
transformation->aspect =
(gdouble) GST_VIDEO_INFO_WIDTH (&filter->out_info) /
(gdouble) GST_VIDEO_INFO_HEIGHT (&filter->out_info);
transformation->caps_change = TRUE;
gst_gl_transformation_build_mvp (transformation);
return TRUE;
}
static void
_intersect_plane_and_ray (graphene_plane_t * video_plane, graphene_ray_t * ray,
graphene_point3d_t * result)
{
float t = graphene_ray_get_distance_to_plane (ray, video_plane);
GST_TRACE ("Calculated a distance of %f to the plane", t);
graphene_ray_get_position_at (ray, t, result);
}
static void
_screen_coord_to_world_ray (GstGLTransformation * transformation, float x,
float y, graphene_ray_t * ray)
{
GstGLFilter *filter = GST_GL_FILTER (transformation);
gfloat w = (gfloat) GST_VIDEO_INFO_WIDTH (&filter->in_info);
gfloat h = (gfloat) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
graphene_vec3_t ray_eye_vec3, ray_world_dir, *ray_origin, *ray_direction;
graphene_vec3_t ray_ortho_dir;
graphene_point3d_t ray_clip, ray_eye;
graphene_vec2_t screen_coord;
/* GL is y-flipped. i.e. 0, 0 is the bottom left corner in screen space */
graphene_vec2_init (&screen_coord, (2. * x / w - 1.) / transformation->aspect,
1. - 2. * y / h);
graphene_point3d_init (&ray_clip, graphene_vec2_get_x (&screen_coord),
graphene_vec2_get_y (&screen_coord), -1.);
graphene_matrix_transform_point3d (&transformation->inv_projection_matrix,
&ray_clip, &ray_eye);
graphene_vec3_init (&ray_eye_vec3, ray_eye.x, ray_eye.y, -1.);
if (transformation->ortho) {
graphene_vec3_init (&ray_ortho_dir, 0., 0., 1.);
ray_origin = &ray_eye_vec3;
ray_direction = &ray_ortho_dir;
} else {
graphene_matrix_transform_vec3 (&transformation->inv_view_matrix,
&ray_eye_vec3, &ray_world_dir);
graphene_vec3_normalize (&ray_world_dir, &ray_world_dir);
ray_origin = &transformation->camera_position;
ray_direction = &ray_world_dir;
}
graphene_ray_init_from_vec3 (ray, ray_origin, ray_direction);
GST_TRACE_OBJECT (transformation, "Calculated ray origin: " VEC3_FORMAT
" direction: " VEC3_FORMAT " from screen coordinates: " VEC2_FORMAT
" with %s projection",
VEC3_ARGS (ray_origin), VEC3_ARGS (ray_direction),
VEC2_ARGS (&screen_coord),
transformation->ortho ? "ortho" : "perspection");
}
static void
_init_world_video_plane (GstGLTransformation * transformation,
graphene_plane_t * video_plane)
{
graphene_point3d_t bottom_left, bottom_right, top_left, top_right;
graphene_point3d_t world_bottom_left, world_bottom_right;
graphene_point3d_t world_top_left, world_top_right;
graphene_point3d_init (&top_left, -transformation->aspect, 1., 0.);
graphene_point3d_init (&top_right, transformation->aspect, 1., 0.);
graphene_point3d_init (&bottom_left, -transformation->aspect, -1., 0.);
graphene_point3d_init (&bottom_right, transformation->aspect, -1., 0.);
graphene_matrix_transform_point3d (&transformation->model_matrix,
&bottom_left, &world_bottom_left);
graphene_matrix_transform_point3d (&transformation->model_matrix,
&bottom_right, &world_bottom_right);
graphene_matrix_transform_point3d (&transformation->model_matrix,
&top_left, &world_top_left);
graphene_matrix_transform_point3d (&transformation->model_matrix,
&top_right, &world_top_right);
graphene_plane_init_from_points (video_plane, &world_bottom_left,
&world_top_right, &world_top_left);
}
static gboolean
_screen_coord_to_model_coord (GstGLTransformation * transformation,
double x, double y, double *res_x, double *res_y)
{
GstGLFilter *filter = GST_GL_FILTER (transformation);
double w = (double) GST_VIDEO_INFO_WIDTH (&filter->in_info);
double h = (double) GST_VIDEO_INFO_HEIGHT (&filter->in_info);
graphene_point3d_t world_point, model_coord;
graphene_plane_t video_plane;
graphene_ray_t ray;
double new_x, new_y;
_init_world_video_plane (transformation, &video_plane);
_screen_coord_to_world_ray (transformation, x, y, &ray);
_intersect_plane_and_ray (&video_plane, &ray, &world_point);
graphene_matrix_transform_point3d (&transformation->inv_model_matrix,
&world_point, &model_coord);
/* ndc to pixels. We render the frame Y-flipped so need to unflip the
* y coordinate */
new_x = (model_coord.x + 1.) * w / 2;
new_y = (1. - model_coord.y) * h / 2;
if (new_x < 0. || new_x > w || new_y < 0. || new_y > h)
/* coords off video surface */
return FALSE;
GST_DEBUG_OBJECT (transformation, "converted %f,%f to %f,%f", x, y, new_x,
new_y);
if (res_x)
*res_x = new_x;
if (res_y)
*res_y = new_y;
return TRUE;
}
#if 0
/* debugging facilities for transforming vertices from model space to screen
* space */
static void
_ndc_to_viewport (GstGLTransformation * transformation, graphene_vec3_t * ndc,
int x, int y, int w, int h, float near, float far, graphene_vec3_t * result)
{
GstGLFilter *filter = GST_GL_FILTER (transformation);
/* center of the viewport */
int o_x = x + w / 2;
int o_y = y + h / 2;
graphene_vec3_init (result, graphene_vec3_get_x (ndc) * w / 2 + o_x,
graphene_vec3_get_y (ndc) * h / 2 + o_y,
(far - near) * graphene_vec3_get_z (ndc) / 2 + (far + near) / 2);
}
static void
_perspective_division (graphene_vec4_t * clip, graphene_vec3_t * result)
{
float w = graphene_vec4_get_w (clip);
graphene_vec3_init (result, graphene_vec4_get_x (clip) / w,
graphene_vec4_get_y (clip) / w, graphene_vec4_get_z (clip) / w);
}
static void
_vertex_to_screen_coord (GstGLTransformation * transformation,
graphene_vec4_t * vertex, graphene_vec3_t * view)
{
GstGLFilter *filter = GST_GL_FILTER (transformation);
gint w = GST_VIDEO_INFO_WIDTH (&filter->in_info);
gint h = GST_VIDEO_INFO_HEIGHT (&filter->in_info);
graphene_vec4_t clip;
graphene_vec3_t ndc;
graphene_matrix_transform_vec4 (&transformation->mvp_matrix, vertex, &clip);
_perspective_division (&clip, &ndc);
_ndc_to_viewport (transformation, &ndc, 0, 0, w, h, 0., 1., view);
}
#endif
static gboolean
gst_gl_transformation_src_event (GstBaseTransform * trans, GstEvent * event)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
gboolean ret;
GST_DEBUG_OBJECT (trans, "handling %s event", GST_EVENT_TYPE_NAME (event));
switch (GST_EVENT_TYPE (event)) {
case GST_EVENT_NAVIGATION:{
gdouble x, y;
event = gst_event_make_writable (event);
if (gst_navigation_event_get_coordinates (event, &x, &y)) {
gdouble new_x, new_y;
if (!_screen_coord_to_model_coord (transformation, x, y, &new_x,
&new_y)) {
gst_event_unref (event);
return TRUE;
}
gst_navigation_event_set_coordinates (event, x, y);
}
break;
}
default:
break;
}
ret = GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (trans, event);
return ret;
}
static gboolean
gst_gl_transformation_filter_meta (GstBaseTransform * trans, GstQuery * query,
GType api, const GstStructure * params)
{
if (api == GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE)
return TRUE;
if (api == GST_GL_SYNC_META_API_TYPE)
return TRUE;
return FALSE;
}
static gboolean
gst_gl_transformation_decide_allocation (GstBaseTransform * trans,
GstQuery * query)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->decide_allocation (trans,
query))
return FALSE;
if (gst_query_find_allocation_meta (query,
GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, NULL)) {
transformation->downstream_supports_affine_meta = TRUE;
} else {
transformation->downstream_supports_affine_meta = FALSE;
}
return TRUE;
}
static void
gst_gl_transformation_gl_stop (GstGLBaseFilter * base_filter)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
const GstGLFuncs *gl = base_filter->context->gl_vtable;
if (transformation->vao) {
gl->DeleteVertexArrays (1, &transformation->vao);
transformation->vao = 0;
}
if (transformation->vertex_buffer) {
gl->DeleteBuffers (1, &transformation->vertex_buffer);
transformation->vertex_buffer = 0;
}
if (transformation->vbo_indices) {
gl->DeleteBuffers (1, &transformation->vbo_indices);
transformation->vbo_indices = 0;
}
if (transformation->shader) {
gst_object_unref (transformation->shader);
transformation->shader = NULL;
}
GST_GL_BASE_FILTER_CLASS (parent_class)->gl_stop (base_filter);
}
static gboolean
gst_gl_transformation_gl_start (GstGLBaseFilter * base_filter)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (base_filter);
if (!GST_GL_BASE_FILTER_CLASS (parent_class)->gl_start (base_filter))
return FALSE;
if (gst_gl_context_get_gl_api (base_filter->context)) {
gchar *frag_str;
gboolean ret;
frag_str =
gst_gl_shader_string_fragment_get_default (base_filter->context,
GST_GLSL_VERSION_NONE,
GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY);
/* blocking call, wait until the opengl thread has compiled the shader */
ret = gst_gl_context_gen_shader (base_filter->context,
gst_gl_shader_string_vertex_mat4_vertex_transform,
frag_str, &transformation->shader);
g_free (frag_str);
return ret;
}
return TRUE;
}
static GstFlowReturn
gst_gl_transformation_prepare_output_buffer (GstBaseTransform * trans,
GstBuffer * inbuf, GstBuffer ** outbuf)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (trans);
GstGLFilter *filter = GST_GL_FILTER (trans);
if (transformation->downstream_supports_affine_meta &&
gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
GstVideoAffineTransformationMeta *af_meta;
graphene_matrix_t upstream_matrix, tmp, tmp2, inv_aspect, yflip;
float upstream[16], downstream[16];
*outbuf = gst_buffer_make_writable (inbuf);
af_meta = gst_buffer_get_video_affine_transformation_meta (inbuf);
if (!af_meta)
af_meta = gst_buffer_add_video_affine_transformation_meta (*outbuf);
GST_LOG_OBJECT (trans, "applying transformation to existing affine "
"transformation meta");
gst_gl_get_affine_transformation_meta_as_ndc (af_meta, upstream);
/* apply the transformation to the existing affine meta */
graphene_matrix_init_from_float (&upstream_matrix, upstream);
graphene_matrix_init_scale (&inv_aspect, transformation->aspect, -1., 1.);
graphene_matrix_init_scale (&yflip, 1., -1., 1.);
/* invert the aspect effects */
graphene_matrix_multiply (&upstream_matrix, &inv_aspect, &tmp2);
/* apply the transformation */
graphene_matrix_multiply (&tmp2, &transformation->mvp_matrix, &tmp);
/* and undo yflip */
graphene_matrix_multiply (&tmp, &yflip, &tmp2);
graphene_matrix_to_float (&tmp2, downstream);
gst_gl_set_affine_transformation_meta_from_ndc (af_meta, downstream);
return GST_FLOW_OK;
}
return GST_BASE_TRANSFORM_CLASS (parent_class)->prepare_output_buffer (trans,
inbuf, outbuf);
}
static gboolean
gst_gl_transformation_filter (GstGLFilter * filter,
GstBuffer * inbuf, GstBuffer * outbuf)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
if (transformation->downstream_supports_affine_meta &&
gst_video_info_is_equal (&filter->in_info, &filter->out_info)) {
return TRUE;
} else {
return gst_gl_filter_filter_texture (filter, inbuf, outbuf);
}
}
static gboolean
gst_gl_transformation_filter_texture (GstGLFilter * filter,
GstGLMemory * in_tex, GstGLMemory * out_tex)
{
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
transformation->in_tex = in_tex;
gst_gl_framebuffer_draw_to_texture (filter->fbo, out_tex,
(GstGLFramebufferFunc) gst_gl_transformation_callback, transformation);
return TRUE;
}
static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
static void
_upload_vertices (GstGLTransformation * transformation)
{
const GstGLFuncs *gl =
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
/* *INDENT-OFF* */
GLfloat vertices[] = {
-transformation->aspect, -1.0, 0.0, 1.0, 0.0, 0.0,
transformation->aspect, -1.0, 0.0, 1.0, 1.0, 0.0,
transformation->aspect, 1.0, 0.0, 1.0, 1.0, 1.0,
-transformation->aspect, 1.0, 0.0, 1.0, 0.0, 1.0,
};
/* *INDENT-ON* */
gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 6 * sizeof (GLfloat), vertices,
GL_STATIC_DRAW);
}
static void
_bind_buffer (GstGLTransformation * transformation)
{
const GstGLFuncs *gl =
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
gl->BindBuffer (GL_ARRAY_BUFFER, transformation->vertex_buffer);
/* Load the vertex position */
gl->VertexAttribPointer (transformation->attr_position, 4, GL_FLOAT,
GL_FALSE, 6 * sizeof (GLfloat), (void *) 0);
/* Load the texture coordinate */
gl->VertexAttribPointer (transformation->attr_texture, 2, GL_FLOAT, GL_FALSE,
6 * sizeof (GLfloat), (void *) (4 * sizeof (GLfloat)));
gl->EnableVertexAttribArray (transformation->attr_position);
gl->EnableVertexAttribArray (transformation->attr_texture);
}
static void
_unbind_buffer (GstGLTransformation * transformation)
{
const GstGLFuncs *gl =
GST_GL_BASE_FILTER (transformation)->context->gl_vtable;
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
gl->DisableVertexAttribArray (transformation->attr_position);
gl->DisableVertexAttribArray (transformation->attr_texture);
}
static gboolean
gst_gl_transformation_callback (gpointer stuff)
{
GstGLFilter *filter = GST_GL_FILTER (stuff);
GstGLTransformation *transformation = GST_GL_TRANSFORMATION (filter);
GstGLFuncs *gl = GST_GL_BASE_FILTER (filter)->context->gl_vtable;
GLfloat temp_matrix[16];
gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->ClearColor (0.f, 0.f, 0.f, 0.f);
gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
gst_gl_shader_use (transformation->shader);
gl->ActiveTexture (GL_TEXTURE0);
gl->BindTexture (GL_TEXTURE_2D, transformation->in_tex->tex_id);
gst_gl_shader_set_uniform_1i (transformation->shader, "texture", 0);
graphene_matrix_to_float (&transformation->mvp_matrix, temp_matrix);
gst_gl_shader_set_uniform_matrix_4fv (transformation->shader,
"u_transformation", 1, GL_FALSE, temp_matrix);
if (!transformation->vertex_buffer) {
transformation->attr_position =
gst_gl_shader_get_attribute_location (transformation->shader,
"a_position");
transformation->attr_texture =
gst_gl_shader_get_attribute_location (transformation->shader,
"a_texcoord");
if (gl->GenVertexArrays) {
gl->GenVertexArrays (1, &transformation->vao);
gl->BindVertexArray (transformation->vao);
}
gl->GenBuffers (1, &transformation->vertex_buffer);
gl->GenBuffers (1, &transformation->vbo_indices);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, transformation->vbo_indices);
gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
GL_STATIC_DRAW);
transformation->caps_change = TRUE;
}
if (gl->GenVertexArrays)
gl->BindVertexArray (transformation->vao);
if (transformation->caps_change)
_upload_vertices (transformation);
_bind_buffer (transformation);
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
if (gl->GenVertexArrays)
gl->BindVertexArray (0);
else
_unbind_buffer (transformation);
gst_gl_context_clear_shader (GST_GL_BASE_FILTER (filter)->context);
transformation->caps_change = FALSE;
return TRUE;
}
Reference in a new issue View git blame Copy permalink