diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c index 758177b664..4ad02ba5da 100644 --- a/ext/gl/gstglimagesink.c +++ b/ext/gl/gstglimagesink.c @@ -1698,6 +1698,8 @@ gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) gst_query_add_allocation_meta (query, GST_VIDEO_OVERLAY_COMPOSITION_META_API_TYPE, allocation_meta); + gst_query_add_allocation_meta (query, + GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, 0); if (allocation_meta) gst_structure_free (allocation_meta); @@ -1776,7 +1778,7 @@ gst_glimage_sink_thread_init_redisplay (GstGLImageSink * gl_sink) vert_stage = gst_glsl_stage_new_with_string (gl_sink->context, GL_VERTEX_SHADER, GST_GLSL_VERSION_NONE, GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, - gst_gl_shader_string_vertex_mat4_texture_transform); + gst_gl_shader_string_vertex_mat4_vertex_transform); if (gl_sink->texture_target == GST_GL_TEXTURE_TARGET_EXTERNAL_OES) { frag_stage = gst_glsl_stage_new_with_string (gl_sink->context, GL_FRAGMENT_SHADER, GST_GLSL_VERSION_NONE, @@ -1939,13 +1941,6 @@ gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height) GST_GLIMAGE_SINK_UNLOCK (gl_sink); } -static const gfloat identity_matrix[] = { - 1.0f, 0.0f, 0.0, 0.0f, - 0.0f, 1.0f, 0.0, 0.0f, - 0.0f, 0.0f, 1.0, 0.0f, - 0.0f, 0.0f, 0.0, 1.0f, -}; - static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) { @@ -2028,16 +2023,15 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) gst_gl_shader_set_uniform_1i (gl_sink->redisplay_shader, "tex", 0); { GstVideoAffineTransformationMeta *af_meta; + gfloat matrix[16]; af_meta = gst_buffer_get_video_affine_transformation_meta (gl_sink->stored_buffer[0]); - if (af_meta) - gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader, - "u_transformation", 1, FALSE, af_meta->matrix); - else - gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader, - "u_transformation", 1, FALSE, identity_matrix); + + gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader, + "u_transformation", 1, FALSE, matrix); } gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ba3caa4fd1..83905d0e68 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -454,6 +454,7 @@ enum #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); +#define gst_gl_video_mixer_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER, DEBUG_INIT); @@ -465,6 +466,9 @@ static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter); static GstCaps *_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps); +static gboolean gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * + base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, + GstQuery * query); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -856,6 +860,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GstElementClass *element_class; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -883,6 +888,8 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + mix_class->propose_allocation = gst_gl_video_mixer_propose_allocation; + GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2; } @@ -926,6 +933,20 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } +static gboolean +gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * base_mix, + GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) +{ + if (!GST_GL_BASE_MIXER_CLASS (parent_class)->propose_allocation (base_mix, + base_pad, decide_query, query)) + return FALSE; + + gst_query_add_allocation_meta (query, + GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, 0); + + return TRUE; +} + static void _mixer_pad_get_output_size (GstGLVideoMixer * mix, GstGLVideoMixerPad * mix_pad, gint out_par_n, gint out_par_d, gint * width, @@ -1130,7 +1151,7 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) video_mixer->shader); return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, - gst_gl_shader_string_vertex_mat4_texture_transform, + gst_gl_shader_string_vertex_mat4_vertex_transform, video_mixer_f_src, &video_mixer->shader); } @@ -1361,13 +1382,6 @@ _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad) return TRUE; } -static const gfloat identity_matrix[] = { - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, -}; - /* opengl scene, params: input texture (not the output mixer->texture) */ static void gst_gl_video_mixer_callback (gpointer stuff) @@ -1495,15 +1509,13 @@ gst_gl_video_mixer_callback (gpointer stuff) { GstVideoAffineTransformationMeta *af_meta; + gfloat matrix[16]; af_meta = gst_buffer_get_video_affine_transformation_meta (vagg_pad->buffer); - if (af_meta) - gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, - "u_transformation", 1, FALSE, af_meta->matrix); - else - gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, - "u_transformation", 1, FALSE, identity_matrix); + gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, + "u_transformation", 1, FALSE, matrix); } gl->EnableVertexAttribArray (attr_position_loc); diff --git a/gst-libs/gst/gl/gstglshaderstrings.c b/gst-libs/gst/gl/gstglshaderstrings.c index 729be6f751..e6bf5f33ce 100644 --- a/gst-libs/gst/gl/gstglshaderstrings.c +++ b/gst-libs/gst/gl/gstglshaderstrings.c @@ -46,6 +46,17 @@ const gchar *gst_gl_shader_string_vertex_mat4_texture_transform = " v_texcoord = (u_transformation * vec4(a_texcoord, 0, 1)).xy;\n" "}\n"; +const gchar *gst_gl_shader_string_vertex_mat4_vertex_transform = + "uniform mat4 u_transformation;\n" + "attribute vec4 a_position;\n" + "attribute vec2 a_texcoord;\n" + "varying vec2 v_texcoord;\n" + "void main()\n" + "{\n" + " gl_Position = u_transformation * a_position;\n" + " v_texcoord = a_texcoord;\n" + "}\n"; + const gchar *gst_gl_shader_string_fragment_default = "#ifdef GL_ES\n" "precision mediump float;\n" diff --git a/gst-libs/gst/gl/gstglshaderstrings.h b/gst-libs/gst/gl/gstglshaderstrings.h index 49ea8de427..89d88b0006 100644 --- a/gst-libs/gst/gl/gstglshaderstrings.h +++ b/gst-libs/gst/gl/gstglshaderstrings.h @@ -29,6 +29,7 @@ extern const gchar *gst_gl_shader_string_vertex_default; extern const gchar *gst_gl_shader_string_fragment_default; extern const gchar *gst_gl_shader_string_vertex_mat4_texture_transform; +extern const gchar *gst_gl_shader_string_vertex_mat4_vertex_transform; extern const gchar *gst_gl_shader_string_fragment_external_oes_default; G_END_DECLS diff --git a/gst-libs/gst/gl/gstglutils.c b/gst-libs/gst/gl/gstglutils.c index e2e04e407e..4b7639691a 100644 --- a/gst-libs/gst/gl/gstglutils.c +++ b/gst-libs/gst/gl/gstglutils.c @@ -1067,3 +1067,59 @@ gst_gl_value_set_texture_target_from_mask (GValue * value, return ret; } } + +static const gfloat identity_matrix[] = { + 1.0f, 0.0f, 0.0, 0.0f, + 0.0f, 1.0f, 0.0, 0.0f, + 0.0f, 0.0f, 1.0, 0.0f, + 0.0f, 0.0f, 0.0, 1.0f, +}; + +static const gfloat from_ndc_matrix[] = { + 0.5f, 0.0f, 0.0, 0.5f, + 0.0f, 0.5f, 0.0, 0.5f, + 0.0f, 0.0f, 0.5, 0.5f, + 0.0f, 0.0f, 0.0, 1.0f, +}; + +static const gfloat to_ndc_matrix[] = { + 2.0f, 0.0f, 0.0, -1.0f, + 0.0f, 2.0f, 0.0, -1.0f, + 0.0f, 0.0f, 2.0, -1.0f, + 0.0f, 0.0f, 0.0, 1.0f, +}; + +static void +_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result) +{ + int i, j, k; + + for (i = 0; i < 16; i++) + result[i] = 0.0f; + + for (i = 0; i < 4; i++) { + for (j = 0; j < 4; j++) { + for (k = 0; k < 4; k++) { + result[i + (j * 4)] += a[i + (k * 4)] * b[k + (j * 4)]; + } + } + } +} + +void +gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * + meta, gfloat * matrix) +{ + if (!meta) { + int i; + + for (i = 0; i < 16; i++) { + matrix[i] = identity_matrix[i]; + } + } else { + gfloat tmp[16] = { 0.0f }; + + _multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp); + _multiply_matrix4 (tmp, to_ndc_matrix, matrix); + } +} diff --git a/gst-libs/gst/gl/gstglutils.h b/gst-libs/gst/gl/gstglutils.h index 1c5ab12344..12d3b6b2fe 100644 --- a/gst-libs/gst/gl/gstglutils.h +++ b/gst-libs/gst/gl/gstglutils.h @@ -22,6 +22,7 @@ #define __GST_GL_UTILS_H__ #include +#include #include @@ -116,6 +117,8 @@ gboolean gst_gl_value_set_texture_target_from_mask (GValue * value, gboolean gst_gl_value_set_texture_target (GValue * value, GstGLTextureTarget target); GstGLTextureTarget gst_gl_value_get_texture_target_mask (const GValue * value); +void gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * meta, gfloat * matrix); + G_END_DECLS #endif /* __GST_GL_UTILS_H__ */ diff --git a/gst-libs/gst/gl/gstglviewconvert.c b/gst-libs/gst/gl/gstglviewconvert.c index 3b983ad9d9..ff770af66e 100644 --- a/gst-libs/gst/gl/gstglviewconvert.c +++ b/gst-libs/gst/gl/gstglviewconvert.c @@ -1742,7 +1742,7 @@ _init_view_convert (GstGLViewConvert * viewconvert) tmp = _gst_glsl_mangle_shader - (gst_gl_shader_string_vertex_mat4_texture_transform, GL_VERTEX_SHADER, + (gst_gl_shader_string_vertex_mat4_vertex_transform, GL_VERTEX_SHADER, GST_GL_TEXTURE_TARGET_2D, viewconvert->from_texture_target, gl_api, gl_major, gl_minor, &version, &profile); @@ -1869,7 +1869,6 @@ _do_view_convert_draw (GstGLContext * context, GstGLViewConvert * viewconvert) GstVideoMultiviewMode out_mode = priv->output_mode; guint from_gl_target = gst_gl_texture_target_to_gl (viewconvert->from_texture_target); - GstVideoAffineTransformationMeta *af_meta; gl = context->gl_vtable; out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info); @@ -1883,10 +1882,16 @@ _do_view_convert_draw (GstGLContext * context, GstGLViewConvert * viewconvert) } /* FIXME: the auxillary buffer could have a different transform matrix */ - af_meta = gst_buffer_get_video_affine_transformation_meta (priv->primary_in); - if (af_meta) + { + GstVideoAffineTransformationMeta *af_meta; + gfloat matrix[16]; + + af_meta = + gst_buffer_get_video_affine_transformation_meta (priv->primary_in); + gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader, - "u_transformation", 1, FALSE, af_meta->matrix); + "u_transformation", 1, FALSE, matrix); + } /* attach the texture to the FBO to renderer to */ for (i = 0; i < out_views; i++) {