From 3cf27dd1c8ec8598d3a39d04b8c5bc295872e0f8 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 16 May 2016 20:02:28 +0800 Subject: [PATCH] glimagesink: support video rotation using transform matrix Add "rotate-method" to glimagesink and apply transform matrix to vertex coordinate to control rotation. https://bugzilla.gnome.org/show_bug.cgi?id=765795 --- ext/gl/gstglimagesink.c | 249 ++++++++++++++++++++++++++++++++++- ext/gl/gstglimagesink.h | 18 +++ gst-libs/gst/gl/gstglutils.c | 18 ++- gst-libs/gst/gl/gstglutils.h | 4 +- 4 files changed, 279 insertions(+), 10 deletions(-) diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c index 4ad02ba5da..f176c235c5 100644 --- a/ext/gl/gstglimagesink.c +++ b/ext/gl/gstglimagesink.c @@ -116,6 +116,7 @@ G_DEFINE_TYPE (GstGLImageSinkBin, gst_gl_image_sink_bin, GST_TYPE_GL_SINK_BIN); enum { PROP_BIN_0, + PROP_BIN_ROTATE_METHOD, PROP_BIN_FORCE_ASPECT_RATIO, PROP_BIN_PIXEL_ASPECT_RATIO, PROP_BIN_HANDLE_EVENTS, @@ -177,6 +178,39 @@ _on_client_draw (GstGLImageSink * sink, GstGLContext * context, return ret; } +#define DEFAULT_ROTATE_METHOD GST_GL_ROTATE_METHOD_IDENTITY + +#define GST_TYPE_GL_ROTATE_METHOD (gst_gl_rotate_method_get_type()) + +static const GEnumValue rotate_methods[] = { + {GST_GL_ROTATE_METHOD_IDENTITY, "Identity (no rotation)", "none"}, + {GST_GL_ROTATE_METHOD_90R, "Rotate clockwise 90 degrees", "clockwise"}, + {GST_GL_ROTATE_METHOD_180, "Rotate 180 degrees", "rotate-180"}, + {GST_GL_ROTATE_METHOD_90L, "Rotate counter-clockwise 90 degrees", + "counterclockwise"}, + {GST_GL_ROTATE_METHOD_FLIP_HORIZ, "Flip horizontally", "horizontal-flip"}, + {GST_GL_ROTATE_METHOD_FLIP_VERT, "Flip vertically", "vertical-flip"}, + {GST_GL_ROTATE_METHOD_FLIP_UL_LR, + "Flip across upper left/lower right diagonal", "upper-left-diagonal"}, + {GST_GL_ROTATE_METHOD_FLIP_UR_LL, + "Flip across upper right/lower left diagonal", "upper-right-diagonal"}, + {GST_GL_ROTATE_METHOD_AUTO, + "Select rotate method based on image-orientation tag", "automatic"}, + {0, NULL, NULL}, +}; + +static GType +gst_gl_rotate_method_get_type (void) +{ + static GType rotate_method_type = 0; + + if (!rotate_method_type) { + rotate_method_type = g_enum_register_static ("GstGLRotateMethod", + rotate_methods); + } + return rotate_method_type; +} + static void gst_gl_image_sink_bin_init (GstGLImageSinkBin * self) { @@ -199,6 +233,12 @@ gst_gl_image_sink_bin_class_init (GstGLImageSinkBinClass * klass) gobject_class->set_property = gst_gl_image_sink_bin_set_property; /* gl sink */ + g_object_class_install_property (gobject_class, PROP_BIN_ROTATE_METHOD, + g_param_spec_enum ("rotate-method", + "rotate method", + "rotate method", + GST_TYPE_GL_ROTATE_METHOD, DEFAULT_ROTATE_METHOD, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_BIN_FORCE_ASPECT_RATIO, g_param_spec_boolean ("force-aspect-ratio", "Force aspect ratio", @@ -287,6 +327,7 @@ static void gst_glimage_sink_set_property (GObject * object, guint prop_id, static void gst_glimage_sink_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * param_spec); +static gboolean gst_glimage_sink_event (GstBaseSink * sink, GstEvent * event); static gboolean gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query); static void gst_glimage_sink_set_context (GstElement * element, GstContext * context); @@ -343,6 +384,7 @@ enum { ARG_0, ARG_DISPLAY, + PROP_ROTATE_METHOD, PROP_FORCE_ASPECT_RATIO, PROP_PIXEL_ASPECT_RATIO, PROP_CONTEXT, @@ -400,6 +442,126 @@ _display_size_to_stream_size (GstGLImageSink * gl_sink, gdouble x, GST_TRACE ("transform %fx%f into %fx%f", x, y, *stream_x, *stream_y); } +/* rotate 90 */ +static const gfloat clockwise_matrix[] = { + 0.0f, -1.0f, 0.0f, 0.0f, + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* rotate 180 */ +static const gfloat clockwise_180_matrix[] = { + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* rotate 270 */ +static const gfloat counterclockwise_matrix[] = { + 0.0f, 1.0f, 0.0f, 0.0f, + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* horizontal-flip */ +static const gfloat horizontal_flip_matrix[] = { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* vertical-flip */ +static const gfloat vertical_flip_matrix[] = { + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* upper-left-diagonal */ +static const gfloat upper_left_matrix[] = { + 0.0f, 1.0f, 0.0f, 0.0f, + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +/* upper-right-diagonal */ +static const gfloat upper_right_matrix[] = { + 0.0f, -1.0f, 0.0f, 0.0f, + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + +static void +gst_glimage_sink_set_rotate_method (GstGLImageSink * gl_sink, + GstGLRotateMethod method, gboolean from_tag) +{ + GstGLRotateMethod tag_method = DEFAULT_ROTATE_METHOD; + GST_GLIMAGE_SINK_LOCK (gl_sink); + if (from_tag) + tag_method = method; + else + gl_sink->rotate_method = method; + + if (gl_sink->rotate_method == GST_GL_ROTATE_METHOD_AUTO) + method = tag_method; + else + method = gl_sink->rotate_method; + + if (method != gl_sink->current_rotate_method) { + GST_DEBUG_OBJECT (gl_sink, "Changing method from %s to %s", + rotate_methods[gl_sink->current_rotate_method].value_nick, + rotate_methods[method].value_nick); + + switch (method) { + case GST_GL_ROTATE_METHOD_IDENTITY: + gl_sink->transform_matrix = NULL; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_90R: + gl_sink->transform_matrix = clockwise_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_180: + gl_sink->transform_matrix = clockwise_180_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_90L: + gl_sink->transform_matrix = counterclockwise_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_FLIP_HORIZ: + gl_sink->transform_matrix = horizontal_flip_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_FLIP_VERT: + gl_sink->transform_matrix = vertical_flip_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_FLIP_UL_LR: + gl_sink->transform_matrix = upper_left_matrix; + gl_sink->output_mode_changed = TRUE; + break; + case GST_GL_ROTATE_METHOD_FLIP_UR_LL: + gl_sink->transform_matrix = upper_right_matrix; + gl_sink->output_mode_changed = TRUE; + break; + default: + g_assert_not_reached (); + break; + } + + gl_sink->current_rotate_method = method; + } + GST_GLIMAGE_SINK_UNLOCK (gl_sink); +} + static void gst_glimage_sink_navigation_send_event (GstNavigation * navigation, GstStructure * structure) @@ -481,6 +643,13 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) gobject_class->set_property = gst_glimage_sink_set_property; gobject_class->get_property = gst_glimage_sink_get_property; + g_object_class_install_property (gobject_class, PROP_ROTATE_METHOD, + g_param_spec_enum ("rotate-method", + "rotate method", + "rotate method", + GST_TYPE_GL_ROTATE_METHOD, DEFAULT_ROTATE_METHOD, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_FORCE_ASPECT_RATIO, g_param_spec_boolean ("force-aspect-ratio", "Force aspect ratio", "When enabled, scaling will respect original aspect ratio", @@ -573,6 +742,7 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) gstelement_class->change_state = gst_glimage_sink_change_state; gstelement_class->set_context = gst_glimage_sink_set_context; + gstbasesink_class->event = gst_glimage_sink_event; gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_glimage_sink_query); gstbasesink_class->set_caps = gst_glimage_sink_set_caps; gstbasesink_class->get_caps = gst_glimage_sink_get_caps; @@ -602,6 +772,9 @@ gst_glimage_sink_init (GstGLImageSink * glimage_sink) glimage_sink->mview_output_flags = DEFAULT_MULTIVIEW_FLAGS; glimage_sink->mview_downmix_mode = DEFAULT_MULTIVIEW_DOWNMIX; + glimage_sink->current_rotate_method = DEFAULT_ROTATE_METHOD; + glimage_sink->transform_matrix = NULL; + g_mutex_init (&glimage_sink->drawing_lock); } @@ -616,6 +789,10 @@ gst_glimage_sink_set_property (GObject * object, guint prop_id, glimage_sink = GST_GLIMAGE_SINK (object); switch (prop_id) { + case PROP_ROTATE_METHOD: + gst_glimage_sink_set_rotate_method (glimage_sink, + g_value_get_enum (value), FALSE); + break; case PROP_FORCE_ASPECT_RATIO: { glimage_sink->keep_aspect_ratio = g_value_get_boolean (value); @@ -683,6 +860,9 @@ gst_glimage_sink_get_property (GObject * object, guint prop_id, glimage_sink = GST_GLIMAGE_SINK (object); switch (prop_id) { + case PROP_ROTATE_METHOD: + g_value_set_enum (value, glimage_sink->current_rotate_method); + break; case PROP_FORCE_ASPECT_RATIO: g_value_set_boolean (value, glimage_sink->keep_aspect_ratio); break; @@ -839,6 +1019,58 @@ context_error: } } +static gboolean +gst_glimage_sink_event (GstBaseSink * sink, GstEvent * event) +{ + GstGLImageSink *gl_sink = GST_GLIMAGE_SINK (sink); + GstTagList *taglist; + gchar *orientation; + gboolean ret; + + GST_DEBUG_OBJECT (gl_sink, "handling %s event", GST_EVENT_TYPE_NAME (event)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_TAG: + gst_event_parse_tag (event, &taglist); + + if (gst_tag_list_get_string (taglist, "image-orientation", &orientation)) { + if (!g_strcmp0 ("rotate-0", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, + GST_GL_ROTATE_METHOD_IDENTITY, TRUE); + else if (!g_strcmp0 ("rotate-90", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, GST_GL_ROTATE_METHOD_90R, + TRUE); + else if (!g_strcmp0 ("rotate-180", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, GST_GL_ROTATE_METHOD_180, + TRUE); + else if (!g_strcmp0 ("rotate-270", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, GST_GL_ROTATE_METHOD_90L, + TRUE); + else if (!g_strcmp0 ("flip-rotate-0", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, + GST_GL_ROTATE_METHOD_FLIP_HORIZ, TRUE); + else if (!g_strcmp0 ("flip-rotate-90", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, + GST_GL_ROTATE_METHOD_FLIP_UR_LL, TRUE); + else if (!g_strcmp0 ("flip-rotate-180", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, + GST_GL_ROTATE_METHOD_FLIP_VERT, TRUE); + else if (!g_strcmp0 ("flip-rotate-270", orientation)) + gst_glimage_sink_set_rotate_method (gl_sink, + GST_GL_ROTATE_METHOD_FLIP_UL_LR, TRUE); + + g_free (orientation); + } + break; + default: + break; + } + + ret = GST_BASE_SINK_CLASS (parent_class)->event (sink, event); + + return ret; +} + static gboolean gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query) { @@ -1910,8 +2142,17 @@ gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height) src.x = 0; src.y = 0; - src.w = GST_VIDEO_SINK_WIDTH (gl_sink); - src.h = GST_VIDEO_SINK_HEIGHT (gl_sink); + if (gl_sink->current_rotate_method == GST_GL_ROTATE_METHOD_90R + || gl_sink->current_rotate_method == GST_GL_ROTATE_METHOD_90L + || gl_sink->current_rotate_method == GST_GL_ROTATE_METHOD_FLIP_UL_LR + || gl_sink->current_rotate_method == + GST_GL_ROTATE_METHOD_FLIP_UR_LL) { + src.h = GST_VIDEO_SINK_WIDTH (gl_sink); + src.w = GST_VIDEO_SINK_HEIGHT (gl_sink); + } else { + src.w = GST_VIDEO_SINK_WIDTH (gl_sink); + src.h = GST_VIDEO_SINK_HEIGHT (gl_sink); + } dst.x = 0; dst.y = 0; @@ -2030,6 +2271,10 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) (gl_sink->stored_buffer[0]); gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + + if (gl_sink->transform_matrix) + gst_gl_multiply_matrix4 (gl_sink->transform_matrix, matrix, matrix); + gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader, "u_transformation", 1, FALSE, matrix); } diff --git a/ext/gl/gstglimagesink.h b/ext/gl/gstglimagesink.h index 6e9b98e74a..8833103073 100644 --- a/ext/gl/gstglimagesink.h +++ b/ext/gl/gstglimagesink.h @@ -44,6 +44,19 @@ GST_DEBUG_CATEGORY_EXTERN (gst_debug_glimage_sink); #define GST_IS_GLIMAGE_SINK_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GLIMAGE_SINK)) +typedef enum +{ + GST_GL_ROTATE_METHOD_IDENTITY, + GST_GL_ROTATE_METHOD_90R, + GST_GL_ROTATE_METHOD_180, + GST_GL_ROTATE_METHOD_90L, + GST_GL_ROTATE_METHOD_FLIP_HORIZ, + GST_GL_ROTATE_METHOD_FLIP_VERT, + GST_GL_ROTATE_METHOD_FLIP_UL_LR, + GST_GL_ROTATE_METHOD_FLIP_UR_LL, + GST_GL_ROTATE_METHOD_AUTO, +}GstGLRotateMethod; + typedef struct _GstGLImageSink GstGLImageSink; typedef struct _GstGLImageSinkClass GstGLImageSinkClass; @@ -122,6 +135,11 @@ struct _GstGLImageSink GstGLStereoDownmix mview_downmix_mode; GstGLOverlayCompositor *overlay_compositor; + + /* current video flip method */ + GstGLRotateMethod current_rotate_method; + GstGLRotateMethod rotate_method; + const gfloat *transform_matrix; }; struct _GstGLImageSinkClass diff --git a/gst-libs/gst/gl/gstglutils.c b/gst-libs/gst/gl/gstglutils.c index 4b7639691a..a38772f10b 100644 --- a/gst-libs/gst/gl/gstglutils.c +++ b/gst-libs/gst/gl/gstglutils.c @@ -1089,21 +1089,25 @@ static const gfloat to_ndc_matrix[] = { 0.0f, 0.0f, 0.0, 1.0f, }; -static void -_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result) +void +gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result) { int i, j, k; + gfloat tmp[16] = { 0.0f }; - for (i = 0; i < 16; i++) - result[i] = 0.0f; + if (!a || !b || !result) + return; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) { for (k = 0; k < 4; k++) { - result[i + (j * 4)] += a[i + (k * 4)] * b[k + (j * 4)]; + tmp[i + (j * 4)] += a[i + (k * 4)] * b[k + (j * 4)]; } } } + + for (i = 0; i < 16; i++) + result[i] = tmp[i]; } void @@ -1119,7 +1123,7 @@ gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * } else { gfloat tmp[16] = { 0.0f }; - _multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp); - _multiply_matrix4 (tmp, to_ndc_matrix, matrix); + gst_gl_multiply_matrix4 (from_ndc_matrix, meta->matrix, tmp); + gst_gl_multiply_matrix4 (tmp, to_ndc_matrix, matrix); } } diff --git a/gst-libs/gst/gl/gstglutils.h b/gst-libs/gst/gl/gstglutils.h index 12d3b6b2fe..6e44c765d9 100644 --- a/gst-libs/gst/gl/gstglutils.h +++ b/gst-libs/gst/gl/gstglutils.h @@ -117,7 +117,9 @@ gboolean gst_gl_value_set_texture_target_from_mask (GValue * value, gboolean gst_gl_value_set_texture_target (GValue * value, GstGLTextureTarget target); GstGLTextureTarget gst_gl_value_get_texture_target_mask (const GValue * value); -void gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * meta, gfloat * matrix); +void gst_gl_multiply_matrix4 (const gfloat * a, const gfloat * b, gfloat * result); +void gst_gl_get_affine_transformation_meta_as_ndc (GstVideoAffineTransformationMeta * + meta, gfloat * matrix); G_END_DECLS