diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c index 8b45459ae5..7000477473 100644 --- a/ext/gl/gstglimagesink.c +++ b/ext/gl/gstglimagesink.c @@ -98,6 +98,8 @@ #include #endif +#include + GST_DEBUG_CATEGORY (gst_debug_glimage_sink); #define GST_CAT_DEFAULT gst_debug_glimage_sink @@ -106,6 +108,10 @@ GST_DEBUG_CATEGORY (gst_debug_glimage_sink); #define DEFAULT_FORCE_ASPECT_RATIO TRUE #define DEFAULT_IGNORE_ALPHA TRUE +#define DEFAULT_MULTIVIEW_MODE GST_VIDEO_MULTIVIEW_MODE_MONO +#define DEFAULT_MULTIVIEW_FLAGS GST_VIDEO_MULTIVIEW_FLAGS_NONE +#define DEFAULT_MULTIVIEW_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS + typedef GstGLSinkBin GstGLImageSinkBin; typedef GstGLSinkBinClass GstGLImageSinkBinClass; @@ -120,6 +126,9 @@ enum PROP_BIN_CONTEXT, PROP_BIN_IGNORE_ALPHA, PROP_BIN_SHOW_PREROLL_FRAME, + PROP_BIN_OUTPUT_MULTIVIEW_LAYOUT, + PROP_BIN_OUTPUT_MULTIVIEW_FLAGS, + PROP_BIN_OUTPUT_MULTIVIEW_DOWNMIX_MODE }; enum @@ -223,10 +232,29 @@ gst_gl_image_sink_bin_class_init (GstGLImageSinkBinClass * klass) DEFAULT_SHOW_PREROLL_FRAME, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_BIN_OUTPUT_MULTIVIEW_LAYOUT, + g_param_spec_enum ("output-multiview-mode", "Output Multiview Mode", + "Choose output mode for multiview/3D video", + GST_TYPE_VIDEO_MULTIVIEW_MODE, DEFAULT_MULTIVIEW_MODE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_BIN_OUTPUT_MULTIVIEW_FLAGS, + g_param_spec_flags ("output-multiview-flags", "Output Multiview Flags", + "Output multiview layout modifier flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGS, DEFAULT_MULTIVIEW_FLAGS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_BIN_OUTPUT_MULTIVIEW_DOWNMIX_MODE, + g_param_spec_enum ("output-multiview-downmix-mode", + "Mode for mono downmixed output", + "Output anaglyph type to generate when downmixing to mono", + GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_MULTIVIEW_DOWNMIX, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW] = - g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass), - G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic, - G_TYPE_BOOLEAN, 2, GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE); + g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST, + 0, NULL, NULL, g_cclosure_marshal_generic, G_TYPE_BOOLEAN, 2, + GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE); gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE] = g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass), @@ -254,6 +282,8 @@ static void gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink); static void gst_glimage_sink_on_close (GstGLImageSink * gl_sink); static void gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height); +static void gst_glimage_sink_do_resize (GstGLImageSink * gl_sink, + gint width, gint height); static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink); static gboolean gst_glimage_sink_redisplay (GstGLImageSink * gl_sink); @@ -291,6 +321,7 @@ static void gst_glimage_sink_set_render_rectangle (GstVideoOverlay * overlay, gint x, gint y, gint width, gint height); static void gst_glimage_sink_handle_events (GstVideoOverlay * overlay, gboolean handle_events); +static gboolean update_output_format (GstGLImageSink * glimage_sink); static GstStaticPadTemplate gst_glimage_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -310,6 +341,9 @@ enum PROP_CONTEXT, PROP_HANDLE_EVENTS, PROP_IGNORE_ALPHA, + PROP_OUTPUT_MULTIVIEW_LAYOUT, + PROP_OUTPUT_MULTIVIEW_FLAGS, + PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE }; enum @@ -426,6 +460,26 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) "When enabled, alpha will be ignored and converted to black", DEFAULT_IGNORE_ALPHA, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT_MULTIVIEW_LAYOUT, + g_param_spec_enum ("output-multiview-mode", + "Output Multiview Mode", + "Choose output mode for multiview/3D video", + GST_TYPE_VIDEO_MULTIVIEW_MODE, DEFAULT_MULTIVIEW_MODE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT_MULTIVIEW_FLAGS, + g_param_spec_flags ("output-multiview-flags", + "Output Multiview Flags", + "Output multiview layout modifier flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGS, DEFAULT_MULTIVIEW_FLAGS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE, + g_param_spec_enum ("output-multiview-downmix-mode", + "Mode for mono downmixed output", + "Output anaglyph type to generate when downmixing to mono", + GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_MULTIVIEW_DOWNMIX, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + gst_element_class_set_metadata (element_class, "OpenGL video sink", "Sink/Video", "A videosink based on OpenGL", "Julien Isorce "); @@ -493,11 +547,14 @@ gst_glimage_sink_init (GstGLImageSink * glimage_sink) glimage_sink->keep_aspect_ratio = TRUE; glimage_sink->par_n = 0; glimage_sink->par_d = 1; - glimage_sink->stored_buffer = NULL; glimage_sink->redisplay_texture = 0; glimage_sink->handle_events = TRUE; glimage_sink->ignore_alpha = TRUE; + glimage_sink->mview_output_mode = DEFAULT_MULTIVIEW_MODE; + glimage_sink->mview_output_flags = DEFAULT_MULTIVIEW_FLAGS; + glimage_sink->mview_downmix_mode = DEFAULT_MULTIVIEW_DOWNMIX; + g_mutex_init (&glimage_sink->drawing_lock); } @@ -530,6 +587,24 @@ gst_glimage_sink_set_property (GObject * object, guint prop_id, case PROP_IGNORE_ALPHA: glimage_sink->ignore_alpha = g_value_get_boolean (value); break; + case PROP_OUTPUT_MULTIVIEW_LAYOUT: + GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->mview_output_mode = g_value_get_enum (value); + glimage_sink->output_mode_changed = TRUE; + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + break; + case PROP_OUTPUT_MULTIVIEW_FLAGS: + GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->mview_output_flags = g_value_get_flags (value); + glimage_sink->output_mode_changed = TRUE; + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + break; + case PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE: + GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->mview_downmix_mode = g_value_get_enum (value); + glimage_sink->output_mode_changed = TRUE; + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -577,6 +652,15 @@ gst_glimage_sink_get_property (GObject * object, guint prop_id, case PROP_IGNORE_ALPHA: g_value_set_boolean (value, glimage_sink->ignore_alpha); break; + case PROP_OUTPUT_MULTIVIEW_LAYOUT: + g_value_set_enum (value, glimage_sink->mview_output_mode); + break; + case PROP_OUTPUT_MULTIVIEW_FLAGS: + g_value_set_flags (value, glimage_sink->mview_output_flags); + break; + case PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE: + g_value_set_enum (value, glimage_sink->mview_downmix_mode); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -608,7 +692,7 @@ _ensure_gl_setup (GstGLImageSink * gl_sink) { GError *error = NULL; - GST_DEBUG_OBJECT (gl_sink, "Ensuring setup"); + GST_TRACE_OBJECT (gl_sink, "Ensuring setup"); if (!gl_sink->context) { if (GST_GL_IS_CONTEXT_GPU_PROCESS (gl_sink->other_context)) { @@ -708,7 +792,7 @@ _ensure_gl_setup (GstGLImageSink * gl_sink) } while (!gst_gl_display_add_context (gl_sink->display, gl_sink->context)); GST_OBJECT_UNLOCK (gl_sink->display); } else - GST_DEBUG_OBJECT (gl_sink, "Already have a context"); + GST_TRACE_OBJECT (gl_sink, "Already have a context"); return TRUE; @@ -775,18 +859,22 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query) } case GST_QUERY_DRAIN: { - GstBuffer *buf = NULL; + GstBuffer *buf[2]; GST_GLIMAGE_SINK_LOCK (glimage_sink); glimage_sink->redisplay_texture = 0; - buf = glimage_sink->stored_buffer; - glimage_sink->stored_buffer = NULL; + buf[0] = glimage_sink->stored_buffer[0]; + buf[1] = glimage_sink->stored_buffer[1]; + glimage_sink->stored_buffer[0] = glimage_sink->stored_buffer[1] = NULL; GST_GLIMAGE_SINK_UNLOCK (glimage_sink); - if (buf) - gst_buffer_unref (buf); + gst_buffer_replace (buf, NULL); + gst_buffer_replace (buf + 1, NULL); + gst_buffer_replace (&glimage_sink->input_buffer, NULL); + gst_buffer_replace (&glimage_sink->input_buffer2, NULL); gst_buffer_replace (&glimage_sink->next_buffer, NULL); + gst_buffer_replace (&glimage_sink->next_buffer2, NULL); gst_buffer_replace (&glimage_sink->next_sync, NULL); res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query); @@ -853,32 +941,44 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition) break; case GST_STATE_CHANGE_PAUSED_TO_READY: { + GstBuffer *buf[2]; + + GST_GLIMAGE_SINK_LOCK (glimage_sink); /* mark the redisplay_texture as unavailable (=0) * to avoid drawing */ - GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->redisplay_texture = 0; + buf[0] = glimage_sink->stored_buffer[0]; + buf[1] = glimage_sink->stored_buffer[1]; + glimage_sink->stored_buffer[0] = glimage_sink->stored_buffer[1] = NULL; + if (glimage_sink->stored_sync) gst_buffer_unref (glimage_sink->stored_sync); glimage_sink->stored_sync = NULL; - glimage_sink->redisplay_texture = 0; - if (glimage_sink->stored_buffer) { - gst_buffer_unref (glimage_sink->stored_buffer); - glimage_sink->stored_buffer = NULL; - } - gst_buffer_replace (&glimage_sink->next_buffer, NULL); - gst_buffer_replace (&glimage_sink->next_sync, NULL); GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + gst_buffer_replace (buf, NULL); + gst_buffer_replace (buf + 1, NULL); + + gst_object_replace ((GstObject **) & glimage_sink->convert_views, NULL); + gst_buffer_replace (&glimage_sink->input_buffer, NULL); + gst_buffer_replace (&glimage_sink->input_buffer2, NULL); + gst_buffer_replace (&glimage_sink->next_buffer, NULL); + gst_buffer_replace (&glimage_sink->next_buffer2, NULL); + gst_buffer_replace (&glimage_sink->next_sync, NULL); + + gst_object_replace ((GstObject **) & glimage_sink->convert_views, NULL); + glimage_sink->window_id = 0; /* but do not reset glimage_sink->new_window_id */ GST_VIDEO_SINK_WIDTH (glimage_sink) = 1; GST_VIDEO_SINK_HEIGHT (glimage_sink) = 1; /* Clear cached caps */ - if (glimage_sink->caps) { - gst_caps_unref (glimage_sink->caps); - glimage_sink->caps = NULL; + if (glimage_sink->out_caps) { + gst_caps_unref (glimage_sink->out_caps); + glimage_sink->out_caps = NULL; } if (glimage_sink->context) { @@ -935,11 +1035,11 @@ gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf, if (GST_BUFFER_DURATION_IS_VALID (buf)) *end = *start + GST_BUFFER_DURATION (buf); else { - if (GST_VIDEO_INFO_FPS_N (&glimagesink->info) > 0) { + if (GST_VIDEO_INFO_FPS_N (&glimagesink->out_info) > 0) { *end = *start + gst_util_uint64_scale_int (GST_SECOND, - GST_VIDEO_INFO_FPS_D (&glimagesink->info), - GST_VIDEO_INFO_FPS_N (&glimagesink->info)); + GST_VIDEO_INFO_FPS_D (&glimagesink->out_info), + GST_VIDEO_INFO_FPS_N (&glimagesink->out_info)); } } } @@ -954,6 +1054,9 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink)); if (filter) { + GST_DEBUG_OBJECT (bsink, "intersecting with filter caps %" GST_PTR_FORMAT, + filter); + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (tmp); } else { @@ -966,30 +1069,21 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) } static gboolean -gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) +configure_display_from_info (GstGLImageSink * glimage_sink, + GstVideoInfo * vinfo) { - GstGLImageSink *glimage_sink; gint width; gint height; gboolean ok; gint par_n, par_d; gint display_par_n, display_par_d; guint display_ratio_num, display_ratio_den; - GstVideoInfo vinfo; - GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps); + width = GST_VIDEO_INFO_WIDTH (vinfo); + height = GST_VIDEO_INFO_HEIGHT (vinfo); - glimage_sink = GST_GLIMAGE_SINK (bsink); - - ok = gst_video_info_from_caps (&vinfo, caps); - if (!ok) - return FALSE; - - width = GST_VIDEO_INFO_WIDTH (&vinfo); - height = GST_VIDEO_INFO_HEIGHT (&vinfo); - - par_n = GST_VIDEO_INFO_PAR_N (&vinfo); - par_d = GST_VIDEO_INFO_PAR_D (&vinfo); + par_n = GST_VIDEO_INFO_PAR_N (vinfo); + par_d = GST_VIDEO_INFO_PAR_D (vinfo); if (!par_n) par_n = 1; @@ -1034,24 +1128,222 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (glimage_sink), GST_VIDEO_SINK_HEIGHT (glimage_sink)); - glimage_sink->info = vinfo; - glimage_sink->caps = gst_caps_ref (caps); + return TRUE; +} + +/* Called with GST_GLIMAGE_SINK lock held, to + * copy in_info to out_info and update out_caps */ +static gboolean +update_output_format (GstGLImageSink * glimage_sink) +{ + GstVideoInfo *out_info = &glimage_sink->out_info; + gboolean input_is_mono = FALSE; + GstVideoMultiviewMode mv_mode; + gboolean ret; + + *out_info = glimage_sink->in_info; + + mv_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info); + + if (mv_mode == GST_VIDEO_MULTIVIEW_MODE_NONE || + mv_mode == GST_VIDEO_MULTIVIEW_MODE_MONO || + mv_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT || + mv_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT) + input_is_mono = TRUE; + + if (input_is_mono == FALSE && + glimage_sink->mview_output_mode != GST_VIDEO_MULTIVIEW_MODE_NONE) { + /* Input is multiview, and output wants a conversion - configure 3d converter now, + * otherwise defer it until either the caps or the 3D output mode changes */ + gst_video_multiview_video_info_change_mode (out_info, + glimage_sink->mview_output_mode, glimage_sink->mview_output_flags); + + if (glimage_sink->convert_views == NULL) { + glimage_sink->convert_views = gst_gl_view_convert_new (); + gst_gl_view_convert_set_context (glimage_sink->convert_views, + glimage_sink->context); + } + } else { + if (glimage_sink->convert_views) { + gst_object_unref (glimage_sink->convert_views); + glimage_sink->convert_views = NULL; + } + } + + ret = configure_display_from_info (glimage_sink, out_info); + + if (glimage_sink->convert_views) { + /* Match actual output window size for pixel-aligned output, + * even though we can't necessarily match the starting left/right + * view parity properly */ + glimage_sink->out_info.width = MAX (1, glimage_sink->display_rect.w); + glimage_sink->out_info.height = MAX (1, glimage_sink->display_rect.h); + GST_LOG_OBJECT (glimage_sink, "Set 3D output scale to %d,%d", + glimage_sink->display_rect.w, glimage_sink->display_rect.h); + + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + gst_gl_view_convert_set_format (glimage_sink->convert_views, + &glimage_sink->in_info, &glimage_sink->out_info); + g_object_set (glimage_sink->convert_views, "downmix-mode", + glimage_sink->mview_downmix_mode, NULL); + GST_GLIMAGE_SINK_LOCK (glimage_sink); + } + + glimage_sink->output_mode_changed = FALSE; + glimage_sink->caps_change = TRUE; + + if (glimage_sink->out_caps) + gst_caps_unref (glimage_sink->out_caps); + glimage_sink->out_caps = gst_video_info_to_caps (out_info); + + return ret; +} + +static gboolean +gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) +{ + GstGLImageSink *glimage_sink; + gboolean ok; + GstVideoInfo vinfo; + + GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps); + + glimage_sink = GST_GLIMAGE_SINK (bsink); + + ok = gst_video_info_from_caps (&vinfo, caps); + if (!ok) + return FALSE; if (!_ensure_gl_setup (glimage_sink)) return FALSE; - glimage_sink->caps_change = TRUE; + GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->in_info = vinfo; + ok = update_output_format (glimage_sink); + + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + + return ok; +} + +/* Take the input_buffer and run it through 3D conversion if needed. + * Called with glimagesink lock, but might drop it temporarily */ +static gboolean +prepare_next_buffer (GstGLImageSink * glimage_sink) +{ + GstBuffer *in_buffer, *next_buffer, *old_buffer; + GstBuffer *in_buffer2 = NULL, *next_buffer2 = NULL, *old_buffer2; + GstBuffer *next_sync, *old_sync; + GstGLSyncMeta *sync_meta; + GstVideoFrame gl_frame; + GstGLViewConvert *convert_views = NULL; + GstVideoInfo *info; + + if (glimage_sink->input_buffer == NULL) + return TRUE; /* No input buffer to process */ + + if (GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) == + GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) { + if (glimage_sink->input_buffer2 == NULL) + return TRUE; /* Need 2nd input buffer to process */ + in_buffer2 = gst_buffer_ref (glimage_sink->input_buffer2); + } + + in_buffer = gst_buffer_ref (glimage_sink->input_buffer); + if (glimage_sink->convert_views && + (GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) != + GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->out_info) || + GST_VIDEO_INFO_MULTIVIEW_FLAGS (&glimage_sink->in_info) != + GST_VIDEO_INFO_MULTIVIEW_FLAGS (&glimage_sink->out_info))) + convert_views = gst_object_ref (glimage_sink->convert_views); + + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + + if (convert_views) { + info = &glimage_sink->out_info; + + if (gst_gl_view_convert_submit_input_buffer (glimage_sink->convert_views, + GST_BUFFER_IS_DISCONT (in_buffer), in_buffer) != GST_FLOW_OK) { + gst_buffer_replace (&in_buffer2, NULL); + goto fail; + } + if (in_buffer2) { + if (gst_gl_view_convert_submit_input_buffer (glimage_sink->convert_views, + GST_BUFFER_IS_DISCONT (in_buffer2), in_buffer2) != GST_FLOW_OK) { + goto fail; + } + } + + if (gst_gl_view_convert_get_output (glimage_sink->convert_views, + &next_buffer) != GST_FLOW_OK) + goto fail; + if (GST_VIDEO_INFO_MULTIVIEW_MODE (info) == + GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) { + if (gst_gl_view_convert_get_output (glimage_sink->convert_views, + &next_buffer2) != GST_FLOW_OK) + goto fail; + } + gst_object_unref (convert_views); + + if (next_buffer == NULL) { + /* Not ready to paint a buffer yet */ + GST_GLIMAGE_SINK_LOCK (glimage_sink); + return TRUE; + } + } else { + next_buffer = in_buffer; + info = &glimage_sink->in_info; + } + /* in_buffer invalid now */ + if (!gst_video_frame_map (&gl_frame, info, next_buffer, + GST_MAP_READ | GST_MAP_GL)) { + gst_buffer_unref (next_buffer); + goto fail; + } + + next_sync = gst_buffer_new (); + sync_meta = gst_buffer_add_gl_sync_meta (glimage_sink->context, next_sync); + gst_gl_sync_meta_set_sync_point (sync_meta, glimage_sink->context); + + GST_GLIMAGE_SINK_LOCK (glimage_sink); + glimage_sink->next_tex = *(guint *) gl_frame.data[0]; + + old_buffer = glimage_sink->next_buffer; + glimage_sink->next_buffer = next_buffer; + old_buffer2 = glimage_sink->next_buffer2; + glimage_sink->next_buffer2 = next_buffer2; + + old_sync = glimage_sink->next_sync; + glimage_sink->next_sync = next_sync; + + /* Need to drop the lock again, to avoid a deadlock if we're + * dropping the last ref on this buffer and it goes back to our + * allocator */ + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + + if (old_buffer) + gst_buffer_unref (old_buffer); + if (old_buffer2) + gst_buffer_unref (old_buffer2); + if (old_sync) + gst_buffer_unref (old_sync); + gst_video_frame_unmap (&gl_frame); + + GST_GLIMAGE_SINK_LOCK (glimage_sink); return TRUE; + +fail: + GST_GLIMAGE_SINK_LOCK (glimage_sink); + return FALSE; } static GstFlowReturn gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) { GstGLImageSink *glimage_sink; - GstVideoFrame gl_frame; - GstBuffer *next_sync, *old_sync, *old_buffer; - GstGLSyncMeta *sync_meta; + GstBuffer **target; + GstBuffer *old_input; glimage_sink = GST_GLIMAGE_SINK (bsink); @@ -1065,31 +1357,29 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) if (!_ensure_gl_setup (glimage_sink)) return GST_FLOW_NOT_NEGOTIATED; - if (!gst_video_frame_map (&gl_frame, &glimage_sink->info, buf, - GST_MAP_READ | GST_MAP_GL)) { - goto upload_failed; - } - - next_sync = gst_buffer_new (); - sync_meta = gst_buffer_add_gl_sync_meta (glimage_sink->context, next_sync); - gst_gl_sync_meta_set_sync_point (sync_meta, glimage_sink->context); - GST_GLIMAGE_SINK_LOCK (glimage_sink); - glimage_sink->next_tex = *(guint *) gl_frame.data[0]; + target = &glimage_sink->input_buffer; + if (GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) == + GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME && + !GST_BUFFER_FLAG_IS_SET (buf, GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE)) { + target = &glimage_sink->input_buffer2; + } + old_input = *target; + *target = gst_buffer_ref (buf); - old_buffer = glimage_sink->next_buffer; - glimage_sink->next_buffer = gst_buffer_ref (buf); + if (glimage_sink->output_mode_changed) + update_output_format (glimage_sink); - old_sync = glimage_sink->next_sync; - glimage_sink->next_sync = next_sync; + if (!prepare_next_buffer (glimage_sink)) { + GST_GLIMAGE_SINK_UNLOCK (glimage_sink); + if (old_input) + gst_buffer_unref (old_input); + goto convert_views_failed; + } GST_GLIMAGE_SINK_UNLOCK (glimage_sink); - if (old_buffer) - gst_buffer_unref (old_buffer); - if (old_sync) - gst_buffer_unref (old_sync); - - gst_video_frame_unmap (&gl_frame); + if (old_input) + gst_buffer_unref (old_input); if (glimage_sink->window_id != glimage_sink->new_window_id) { GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context); @@ -1101,11 +1391,10 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) } return GST_FLOW_OK; - -upload_failed: +convert_views_failed: { GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND, - ("%s", "Failed to upload buffer"), (NULL)); + ("%s", "Failed to convert multiview video buffer"), (NULL)); return GST_FLOW_ERROR; } } @@ -1114,40 +1403,23 @@ static GstFlowReturn gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) { GstGLImageSink *glimage_sink; - GstBuffer *stored_buffer, *old_sync; GST_TRACE ("rendering buffer:%p", buf); glimage_sink = GST_GLIMAGE_SINK (vsink); GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u", - glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->info), - GST_VIDEO_INFO_HEIGHT (&glimage_sink->info), + glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->out_info), + GST_VIDEO_INFO_HEIGHT (&glimage_sink->out_info), GST_VIDEO_SINK_WIDTH (glimage_sink), GST_VIDEO_SINK_HEIGHT (glimage_sink)); - /* Avoid to release the texture while drawing */ - GST_GLIMAGE_SINK_LOCK (glimage_sink); - glimage_sink->redisplay_texture = glimage_sink->next_tex; - stored_buffer = glimage_sink->stored_buffer; - glimage_sink->stored_buffer = gst_buffer_ref (glimage_sink->next_buffer); - - old_sync = glimage_sink->stored_sync; - glimage_sink->stored_sync = gst_buffer_ref (glimage_sink->next_sync); - - GST_GLIMAGE_SINK_UNLOCK (glimage_sink); - /* Ask the underlying window to redraw its content */ if (!gst_glimage_sink_redisplay (glimage_sink)) goto redisplay_failed; GST_TRACE ("post redisplay"); - if (stored_buffer) - gst_buffer_unref (stored_buffer); - if (old_sync) - gst_buffer_unref (old_sync); - if (g_atomic_int_get (&glimage_sink->to_quit) != 0) { GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND, ("%s", gst_gl_context_get_error ()), (NULL)); @@ -1419,17 +1691,28 @@ gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink) static void gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height) { - /* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps. - * It means that they cannot not change between two set_caps + GST_DEBUG_OBJECT (gl_sink, "GL Window resized to %ux%u", width, height); + + GST_GLIMAGE_SINK_LOCK (gl_sink); + gl_sink->output_mode_changed = TRUE; + gst_glimage_sink_do_resize (gl_sink, width, height); + GST_GLIMAGE_SINK_UNLOCK (gl_sink); +} + +/* Called with object lock held */ +static void +gst_glimage_sink_do_resize (GstGLImageSink * gl_sink, gint width, gint height) +{ + /* Here gl_sink members (ex:gl_sink->out_info) have a life time of set_caps. + * It means that they cannot change between two set_caps */ - const GstGLFuncs *gl = gl_sink->context->gl_vtable; gboolean do_reshape; - GST_TRACE ("GL Window resized to %ux%u", width, height); - + GST_GLIMAGE_SINK_UNLOCK (gl_sink); /* check if a client reshape callback is registered */ g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL], 0, gl_sink->context, width, height, &do_reshape); + GST_GLIMAGE_SINK_LOCK (gl_sink); width = MAX (1, width); height = MAX (1, height); @@ -1453,17 +1736,27 @@ gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height) dst.h = height; gst_video_sink_center_rect (src, dst, &result, TRUE); - gl->Viewport (result.x, result.y, result.w, result.h); + gl_sink->output_mode_changed |= (result.w != gl_sink->display_rect.w); + gl_sink->output_mode_changed |= (result.h != gl_sink->display_rect.h); + gl_sink->display_rect = result; } else { - gl->Viewport (0, 0, width, height); + gl_sink->output_mode_changed |= (width != gl_sink->display_rect.w); + gl_sink->output_mode_changed |= (height != gl_sink->display_rect.h); + + gl_sink->display_rect.x = 0; + gl_sink->display_rect.y = 0; + gl_sink->display_rect.w = width; + gl_sink->display_rect.h = height; } + gl_sink->update_viewport = TRUE; + } } static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) { - /* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps. + /* Here gl_sink members (ex:gl_sink->out_info) have a life time of set_caps. * It means that they cannot not change between two set_caps as well as * for the redisplay_texture size. * Whereas redisplay_texture id changes every sink_render @@ -1495,12 +1788,18 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) if (gl_sink->caps_change && gl_sink->window_width > 0 && gl_sink->window_height > 0) { - GST_GLIMAGE_SINK_UNLOCK (gl_sink); - gst_glimage_sink_on_resize (gl_sink, gl_sink->window_width, + gst_glimage_sink_do_resize (gl_sink, gl_sink->window_width, gl_sink->window_height); - GST_GLIMAGE_SINK_LOCK (gl_sink); gl_sink->caps_change = FALSE; } + if (gl_sink->update_viewport == TRUE) { + gl->Viewport (gl_sink->display_rect.x, gl_sink->display_rect.y, + gl_sink->display_rect.w, gl_sink->display_rect.h); + GST_DEBUG_OBJECT (gl_sink, "GL output area now %u,%u %ux%u", + gl_sink->display_rect.x, gl_sink->display_rect.y, + gl_sink->display_rect.w, gl_sink->display_rect.h); + gl_sink->update_viewport = FALSE; + } sync_meta = gst_buffer_get_gl_sync_meta (gl_sink->stored_sync); if (sync_meta) @@ -1514,14 +1813,20 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink) gl->Disable (GL_TEXTURE_2D); #endif - sample = gst_sample_new (gl_sink->stored_buffer, - gl_sink->caps, &GST_BASE_SINK (gl_sink)->segment, NULL); - + sample = gst_sample_new (gl_sink->stored_buffer[0], + gl_sink->out_caps, &GST_BASE_SINK (gl_sink)->segment, NULL); g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0, gl_sink->context, sample, &do_redisplay); - gst_sample_unref (sample); + if (gl_sink->stored_buffer[1]) { + sample = gst_sample_new (gl_sink->stored_buffer[1], + gl_sink->out_caps, &GST_BASE_SINK (gl_sink)->segment, NULL); + g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0, + gl_sink->context, sample, &do_redisplay); + gst_sample_unref (sample); + } + if (!do_redisplay) { gfloat alpha = gl_sink->ignore_alpha ? 1.0f : 0.0f; @@ -1591,6 +1896,7 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink) { GstGLWindow *window; gboolean alive; + GstBuffer *old_stored_buffer[2], *old_sync; window = gst_gl_context_get_window (gl_sink->context); if (!window) @@ -1618,6 +1924,39 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink) gst_gl_window_show (window); } + /* Recreate the output texture if needed */ + GST_GLIMAGE_SINK_LOCK (gl_sink); + if (gl_sink->output_mode_changed && gl_sink->input_buffer != NULL) { + GST_DEBUG ("Recreating output after mode/size change"); + update_output_format (gl_sink); + prepare_next_buffer (gl_sink); + } + + if (gl_sink->next_buffer == NULL) { + /* Nothing to display yet */ + GST_GLIMAGE_SINK_UNLOCK (gl_sink); + return TRUE; + } + + /* Avoid to release the texture while drawing */ + gl_sink->redisplay_texture = gl_sink->next_tex; + old_stored_buffer[0] = gl_sink->stored_buffer[0]; + old_stored_buffer[1] = gl_sink->stored_buffer[1]; + gl_sink->stored_buffer[0] = gst_buffer_ref (gl_sink->next_buffer); + if (gl_sink->next_buffer2) + gl_sink->stored_buffer[1] = gst_buffer_ref (gl_sink->next_buffer2); + else + gl_sink->stored_buffer[1] = NULL; + + old_sync = gl_sink->stored_sync; + gl_sink->stored_sync = gst_buffer_ref (gl_sink->next_sync); + GST_GLIMAGE_SINK_UNLOCK (gl_sink); + + gst_buffer_replace (old_stored_buffer, NULL); + gst_buffer_replace (old_stored_buffer + 1, NULL); + if (old_sync) + gst_buffer_unref (old_sync); + /* Drawing is asynchronous: gst_gl_window_draw is not blocking * It means that it does not wait for stuff to be executed in other threads */ diff --git a/ext/gl/gstglimagesink.h b/ext/gl/gstglimagesink.h index b8c256ba11..a60a8b8ad9 100644 --- a/ext/gl/gstglimagesink.h +++ b/ext/gl/gstglimagesink.h @@ -62,9 +62,12 @@ struct _GstGLImageSink gint width; gint height; - //caps - GstVideoInfo info; - GstCaps *caps; + /* Input info before 3d stereo output conversion, if any */ + GstVideoInfo in_info; + + /* format/caps we actually hand off to the app */ + GstVideoInfo out_info; + GstCaps *out_caps; GstGLDisplay *display; GstGLContext *context; @@ -72,8 +75,17 @@ struct _GstGLImageSink gboolean handle_events; gboolean ignore_alpha; + GstGLViewConvert *convert_views; + + /* Original input RGBA buffer, ready for display, + * or possible reconversion through the views filter */ + GstBuffer *input_buffer; + /* Secondary view buffer - when operating in frame-by-frame mode */ + GstBuffer *input_buffer2; + guint next_tex; GstBuffer *next_buffer; + GstBuffer *next_buffer2; /* frame-by-frame 2nd view */ GstBuffer *next_sync; volatile gint to_quit; @@ -82,13 +94,16 @@ struct _GstGLImageSink /* avoid replacing the stored_buffer while drawing */ GMutex drawing_lock; - GstBuffer *stored_buffer; + GstBuffer *stored_buffer[2]; GstBuffer *stored_sync; GLuint redisplay_texture; gboolean caps_change; guint window_width; guint window_height; + gboolean update_viewport; + + GstVideoRectangle display_rect; GstGLShader *redisplay_shader; GLuint vao; @@ -96,6 +111,11 @@ struct _GstGLImageSink GLuint vertex_buffer; GLint attr_position; GLint attr_texture; + + GstVideoMultiviewMode mview_output_mode; + GstVideoMultiviewFlags mview_output_flags; + gboolean output_mode_changed; + GstGLStereoDownmix mview_downmix_mode; }; struct _GstGLImageSinkClass