mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-28 19:20:35 +00:00
Code Issues 1 Projects Releases Wiki Activity

glimagesink: Support multiview/stereoscopic video

Browse source 
Support video with multiview info in the caps, transform
it to mono anaglyph by default, but allow for configuring
other output modes and handoff to the app via
the draw signal.

https://bugzilla.gnome.org/show_bug.cgi?id=611157
This commit is contained in:
Jan Schmidt 2015-05-30 02:26:32 +10:00
parent 1aa3911d40
commit 00d6fe9068
2 changed files with 466 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

545
ext/gl/gstglimagesink.c
View file

@ -98,6 +98,8 @@
#include <gst/gl/egl/gsteglimagememory.h>
#endif
#include <gst/gl/gstglviewconvert.h>
GST_DEBUG_CATEGORY (gst_debug_glimage_sink);
#define GST_CAT_DEFAULT gst_debug_glimage_sink
@ -106,6 +108,10 @@ GST_DEBUG_CATEGORY (gst_debug_glimage_sink);
#define DEFAULT_FORCE_ASPECT_RATIO TRUE
#define DEFAULT_IGNORE_ALPHA TRUE
#define DEFAULT_MULTIVIEW_MODE GST_VIDEO_MULTIVIEW_MODE_MONO
#define DEFAULT_MULTIVIEW_FLAGS GST_VIDEO_MULTIVIEW_FLAGS_NONE
#define DEFAULT_MULTIVIEW_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
typedef GstGLSinkBin GstGLImageSinkBin;
typedef GstGLSinkBinClass GstGLImageSinkBinClass;
@ -120,6 +126,9 @@ enum
PROP_BIN_CONTEXT,
PROP_BIN_IGNORE_ALPHA,
PROP_BIN_SHOW_PREROLL_FRAME,
PROP_BIN_OUTPUT_MULTIVIEW_LAYOUT,
PROP_BIN_OUTPUT_MULTIVIEW_FLAGS,
PROP_BIN_OUTPUT_MULTIVIEW_DOWNMIX_MODE
};
enum
@ -223,10 +232,29 @@ gst_gl_image_sink_bin_class_init (GstGLImageSinkBinClass * klass)
DEFAULT_SHOW_PREROLL_FRAME,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_BIN_OUTPUT_MULTIVIEW_LAYOUT,
g_param_spec_enum ("output-multiview-mode", "Output Multiview Mode",
"Choose output mode for multiview/3D video",
GST_TYPE_VIDEO_MULTIVIEW_MODE, DEFAULT_MULTIVIEW_MODE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_BIN_OUTPUT_MULTIVIEW_FLAGS,
g_param_spec_flags ("output-multiview-flags", "Output Multiview Flags",
"Output multiview layout modifier flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, DEFAULT_MULTIVIEW_FLAGS,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_BIN_OUTPUT_MULTIVIEW_DOWNMIX_MODE,
g_param_spec_enum ("output-multiview-downmix-mode",
"Mode for mono downmixed output",
"Output anaglyph type to generate when downmixing to mono",
GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_MULTIVIEW_DOWNMIX,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW] =
g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass),
G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic,
G_TYPE_BOOLEAN, 2, GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE);
g_signal_new ("client-draw", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_LAST,
0, NULL, NULL, g_cclosure_marshal_generic, G_TYPE_BOOLEAN, 2,
GST_GL_TYPE_CONTEXT, GST_TYPE_SAMPLE);
gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_RESHAPE] =
g_signal_new ("client-reshape", G_TYPE_FROM_CLASS (klass),
@ -254,6 +282,8 @@ static void gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink);
static void gst_glimage_sink_on_close (GstGLImageSink * gl_sink);
static void gst_glimage_sink_on_resize (GstGLImageSink * gl_sink,
gint width, gint height);
static void gst_glimage_sink_do_resize (GstGLImageSink * gl_sink,
gint width, gint height);
static void gst_glimage_sink_on_draw (GstGLImageSink * gl_sink);
static gboolean gst_glimage_sink_redisplay (GstGLImageSink * gl_sink);
@ -291,6 +321,7 @@ static void gst_glimage_sink_set_render_rectangle (GstVideoOverlay * overlay,
gint x, gint y, gint width, gint height);
static void gst_glimage_sink_handle_events (GstVideoOverlay * overlay,
gboolean handle_events);
static gboolean update_output_format (GstGLImageSink * glimage_sink);
static GstStaticPadTemplate gst_glimage_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
@ -310,6 +341,9 @@ enum
PROP_CONTEXT,
PROP_HANDLE_EVENTS,
PROP_IGNORE_ALPHA,
PROP_OUTPUT_MULTIVIEW_LAYOUT,
PROP_OUTPUT_MULTIVIEW_FLAGS,
PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE
};
enum
@ -426,6 +460,26 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass)
"When enabled, alpha will be ignored and converted to black",
DEFAULT_IGNORE_ALPHA, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_MULTIVIEW_LAYOUT,
g_param_spec_enum ("output-multiview-mode",
"Output Multiview Mode",
"Choose output mode for multiview/3D video",
GST_TYPE_VIDEO_MULTIVIEW_MODE, DEFAULT_MULTIVIEW_MODE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_MULTIVIEW_FLAGS,
g_param_spec_flags ("output-multiview-flags",
"Output Multiview Flags",
"Output multiview layout modifier flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, DEFAULT_MULTIVIEW_FLAGS,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class,
PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE,
g_param_spec_enum ("output-multiview-downmix-mode",
"Mode for mono downmixed output",
"Output anaglyph type to generate when downmixing to mono",
GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_MULTIVIEW_DOWNMIX,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gst_element_class_set_metadata (element_class, "OpenGL video sink",
"Sink/Video", "A videosink based on OpenGL",
"Julien Isorce <julien.isorce@gmail.com>");
@ -493,11 +547,14 @@ gst_glimage_sink_init (GstGLImageSink * glimage_sink)
glimage_sink->keep_aspect_ratio = TRUE;
glimage_sink->par_n = 0;
glimage_sink->par_d = 1;
glimage_sink->stored_buffer = NULL;
glimage_sink->redisplay_texture = 0;
glimage_sink->handle_events = TRUE;
glimage_sink->ignore_alpha = TRUE;
glimage_sink->mview_output_mode = DEFAULT_MULTIVIEW_MODE;
glimage_sink->mview_output_flags = DEFAULT_MULTIVIEW_FLAGS;
glimage_sink->mview_downmix_mode = DEFAULT_MULTIVIEW_DOWNMIX;
g_mutex_init (&glimage_sink->drawing_lock);
}
@ -530,6 +587,24 @@ gst_glimage_sink_set_property (GObject * object, guint prop_id,
case PROP_IGNORE_ALPHA:
glimage_sink->ignore_alpha = g_value_get_boolean (value);
break;
case PROP_OUTPUT_MULTIVIEW_LAYOUT:
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->mview_output_mode = g_value_get_enum (value);
glimage_sink->output_mode_changed = TRUE;
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
break;
case PROP_OUTPUT_MULTIVIEW_FLAGS:
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->mview_output_flags = g_value_get_flags (value);
glimage_sink->output_mode_changed = TRUE;
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
break;
case PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE:
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->mview_downmix_mode = g_value_get_enum (value);
glimage_sink->output_mode_changed = TRUE;
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@ -577,6 +652,15 @@ gst_glimage_sink_get_property (GObject * object, guint prop_id,
case PROP_IGNORE_ALPHA:
g_value_set_boolean (value, glimage_sink->ignore_alpha);
break;
case PROP_OUTPUT_MULTIVIEW_LAYOUT:
g_value_set_enum (value, glimage_sink->mview_output_mode);
break;
case PROP_OUTPUT_MULTIVIEW_FLAGS:
g_value_set_flags (value, glimage_sink->mview_output_flags);
break;
case PROP_OUTPUT_MULTIVIEW_DOWNMIX_MODE:
g_value_set_enum (value, glimage_sink->mview_downmix_mode);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
@ -608,7 +692,7 @@ _ensure_gl_setup (GstGLImageSink * gl_sink)
{
GError *error = NULL;
GST_DEBUG_OBJECT (gl_sink, "Ensuring setup");
GST_TRACE_OBJECT (gl_sink, "Ensuring setup");
if (!gl_sink->context) {
if (GST_GL_IS_CONTEXT_GPU_PROCESS (gl_sink->other_context)) {
@ -708,7 +792,7 @@ _ensure_gl_setup (GstGLImageSink * gl_sink)
} while (!gst_gl_display_add_context (gl_sink->display, gl_sink->context));
GST_OBJECT_UNLOCK (gl_sink->display);
} else
GST_DEBUG_OBJECT (gl_sink, "Already have a context");
GST_TRACE_OBJECT (gl_sink, "Already have a context");
return TRUE;
@ -775,18 +859,22 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query)
}
case GST_QUERY_DRAIN:
{
GstBuffer *buf = NULL;
GstBuffer *buf[2];
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->redisplay_texture = 0;
buf = glimage_sink->stored_buffer;
glimage_sink->stored_buffer = NULL;
buf[0] = glimage_sink->stored_buffer[0];
buf[1] = glimage_sink->stored_buffer[1];
glimage_sink->stored_buffer[0] = glimage_sink->stored_buffer[1] = NULL;
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
if (buf)
gst_buffer_unref (buf);
gst_buffer_replace (buf, NULL);
gst_buffer_replace (buf + 1, NULL);
gst_buffer_replace (&glimage_sink->input_buffer, NULL);
gst_buffer_replace (&glimage_sink->input_buffer2, NULL);
gst_buffer_replace (&glimage_sink->next_buffer, NULL);
gst_buffer_replace (&glimage_sink->next_buffer2, NULL);
gst_buffer_replace (&glimage_sink->next_sync, NULL);
res = GST_BASE_SINK_CLASS (parent_class)->query (bsink, query);
@ -853,32 +941,44 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
break;
case GST_STATE_CHANGE_PAUSED_TO_READY:
{
GstBuffer *buf[2];
GST_GLIMAGE_SINK_LOCK (glimage_sink);
/* mark the redisplay_texture as unavailable (=0)
* to avoid drawing
*/
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->redisplay_texture = 0;
buf[0] = glimage_sink->stored_buffer[0];
buf[1] = glimage_sink->stored_buffer[1];
glimage_sink->stored_buffer[0] = glimage_sink->stored_buffer[1] = NULL;
if (glimage_sink->stored_sync)
gst_buffer_unref (glimage_sink->stored_sync);
glimage_sink->stored_sync = NULL;
glimage_sink->redisplay_texture = 0;
if (glimage_sink->stored_buffer) {
gst_buffer_unref (glimage_sink->stored_buffer);
glimage_sink->stored_buffer = NULL;
}
gst_buffer_replace (&glimage_sink->next_buffer, NULL);
gst_buffer_replace (&glimage_sink->next_sync, NULL);
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
gst_buffer_replace (buf, NULL);
gst_buffer_replace (buf + 1, NULL);
gst_object_replace ((GstObject **) & glimage_sink->convert_views, NULL);
gst_buffer_replace (&glimage_sink->input_buffer, NULL);
gst_buffer_replace (&glimage_sink->input_buffer2, NULL);
gst_buffer_replace (&glimage_sink->next_buffer, NULL);
gst_buffer_replace (&glimage_sink->next_buffer2, NULL);
gst_buffer_replace (&glimage_sink->next_sync, NULL);
gst_object_replace ((GstObject **) & glimage_sink->convert_views, NULL);
glimage_sink->window_id = 0;
/* but do not reset glimage_sink->new_window_id */
GST_VIDEO_SINK_WIDTH (glimage_sink) = 1;
GST_VIDEO_SINK_HEIGHT (glimage_sink) = 1;
/* Clear cached caps */
if (glimage_sink->caps) {
gst_caps_unref (glimage_sink->caps);
glimage_sink->caps = NULL;
if (glimage_sink->out_caps) {
gst_caps_unref (glimage_sink->out_caps);
glimage_sink->out_caps = NULL;
}
if (glimage_sink->context) {
@ -935,11 +1035,11 @@ gst_glimage_sink_get_times (GstBaseSink * bsink, GstBuffer * buf,
if (GST_BUFFER_DURATION_IS_VALID (buf))
*end = *start + GST_BUFFER_DURATION (buf);
else {
if (GST_VIDEO_INFO_FPS_N (&glimagesink->info) > 0) {
if (GST_VIDEO_INFO_FPS_N (&glimagesink->out_info) > 0) {
*end = *start +
gst_util_uint64_scale_int (GST_SECOND,
GST_VIDEO_INFO_FPS_D (&glimagesink->info),
GST_VIDEO_INFO_FPS_N (&glimagesink->info));
GST_VIDEO_INFO_FPS_D (&glimagesink->out_info),
GST_VIDEO_INFO_FPS_N (&glimagesink->out_info));
}
}
}
@ -954,6 +1054,9 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink));
if (filter) {
GST_DEBUG_OBJECT (bsink, "intersecting with filter caps %" GST_PTR_FORMAT,
filter);
result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (tmp);
} else {
@ -966,30 +1069,21 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter)
}
static gboolean
gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
configure_display_from_info (GstGLImageSink * glimage_sink,
GstVideoInfo * vinfo)
{
GstGLImageSink *glimage_sink;
gint width;
gint height;
gboolean ok;
gint par_n, par_d;
gint display_par_n, display_par_d;
guint display_ratio_num, display_ratio_den;
GstVideoInfo vinfo;
GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps);
width = GST_VIDEO_INFO_WIDTH (vinfo);
height = GST_VIDEO_INFO_HEIGHT (vinfo);
glimage_sink = GST_GLIMAGE_SINK (bsink);
ok = gst_video_info_from_caps (&vinfo, caps);
if (!ok)
return FALSE;
width = GST_VIDEO_INFO_WIDTH (&vinfo);
height = GST_VIDEO_INFO_HEIGHT (&vinfo);
par_n = GST_VIDEO_INFO_PAR_N (&vinfo);
par_d = GST_VIDEO_INFO_PAR_D (&vinfo);
par_n = GST_VIDEO_INFO_PAR_N (vinfo);
par_d = GST_VIDEO_INFO_PAR_D (vinfo);
if (!par_n)
par_n = 1;
@ -1034,24 +1128,222 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
GST_DEBUG ("scaling to %dx%d", GST_VIDEO_SINK_WIDTH (glimage_sink),
GST_VIDEO_SINK_HEIGHT (glimage_sink));
glimage_sink->info = vinfo;
glimage_sink->caps = gst_caps_ref (caps);
return TRUE;
}
/* Called with GST_GLIMAGE_SINK lock held, to
* copy in_info to out_info and update out_caps */
static gboolean
update_output_format (GstGLImageSink * glimage_sink)
{
GstVideoInfo *out_info = &glimage_sink->out_info;
gboolean input_is_mono = FALSE;
GstVideoMultiviewMode mv_mode;
gboolean ret;
*out_info = glimage_sink->in_info;
mv_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info);
if (mv_mode == GST_VIDEO_MULTIVIEW_MODE_NONE ||
mv_mode == GST_VIDEO_MULTIVIEW_MODE_MONO ||
mv_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT ||
mv_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT)
input_is_mono = TRUE;
if (input_is_mono == FALSE &&
glimage_sink->mview_output_mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* Input is multiview, and output wants a conversion - configure 3d converter now,
* otherwise defer it until either the caps or the 3D output mode changes */
gst_video_multiview_video_info_change_mode (out_info,
glimage_sink->mview_output_mode, glimage_sink->mview_output_flags);
if (glimage_sink->convert_views == NULL) {
glimage_sink->convert_views = gst_gl_view_convert_new ();
gst_gl_view_convert_set_context (glimage_sink->convert_views,
glimage_sink->context);
}
} else {
if (glimage_sink->convert_views) {
gst_object_unref (glimage_sink->convert_views);
glimage_sink->convert_views = NULL;
}
}
ret = configure_display_from_info (glimage_sink, out_info);
if (glimage_sink->convert_views) {
/* Match actual output window size for pixel-aligned output,
* even though we can't necessarily match the starting left/right
* view parity properly */
glimage_sink->out_info.width = MAX (1, glimage_sink->display_rect.w);
glimage_sink->out_info.height = MAX (1, glimage_sink->display_rect.h);
GST_LOG_OBJECT (glimage_sink, "Set 3D output scale to %d,%d",
glimage_sink->display_rect.w, glimage_sink->display_rect.h);
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
gst_gl_view_convert_set_format (glimage_sink->convert_views,
&glimage_sink->in_info, &glimage_sink->out_info);
g_object_set (glimage_sink->convert_views, "downmix-mode",
glimage_sink->mview_downmix_mode, NULL);
GST_GLIMAGE_SINK_LOCK (glimage_sink);
}
glimage_sink->output_mode_changed = FALSE;
glimage_sink->caps_change = TRUE;
if (glimage_sink->out_caps)
gst_caps_unref (glimage_sink->out_caps);
glimage_sink->out_caps = gst_video_info_to_caps (out_info);
return ret;
}
static gboolean
gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps)
{
GstGLImageSink *glimage_sink;
gboolean ok;
GstVideoInfo vinfo;
GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps);
glimage_sink = GST_GLIMAGE_SINK (bsink);
ok = gst_video_info_from_caps (&vinfo, caps);
if (!ok)
return FALSE;
if (!_ensure_gl_setup (glimage_sink))
return FALSE;
glimage_sink->caps_change = TRUE;
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->in_info = vinfo;
ok = update_output_format (glimage_sink);
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
return ok;
}
/* Take the input_buffer and run it through 3D conversion if needed.
* Called with glimagesink lock, but might drop it temporarily */
static gboolean
prepare_next_buffer (GstGLImageSink * glimage_sink)
{
GstBuffer *in_buffer, *next_buffer, *old_buffer;
GstBuffer *in_buffer2 = NULL, *next_buffer2 = NULL, *old_buffer2;
GstBuffer *next_sync, *old_sync;
GstGLSyncMeta *sync_meta;
GstVideoFrame gl_frame;
GstGLViewConvert *convert_views = NULL;
GstVideoInfo *info;
if (glimage_sink->input_buffer == NULL)
return TRUE; /* No input buffer to process */
if (GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) ==
GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (glimage_sink->input_buffer2 == NULL)
return TRUE; /* Need 2nd input buffer to process */
in_buffer2 = gst_buffer_ref (glimage_sink->input_buffer2);
}
in_buffer = gst_buffer_ref (glimage_sink->input_buffer);
if (glimage_sink->convert_views &&
(GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) !=
GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->out_info) ||
GST_VIDEO_INFO_MULTIVIEW_FLAGS (&glimage_sink->in_info) !=
GST_VIDEO_INFO_MULTIVIEW_FLAGS (&glimage_sink->out_info)))
convert_views = gst_object_ref (glimage_sink->convert_views);
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
if (convert_views) {
info = &glimage_sink->out_info;
if (gst_gl_view_convert_submit_input_buffer (glimage_sink->convert_views,
GST_BUFFER_IS_DISCONT (in_buffer), in_buffer) != GST_FLOW_OK) {
gst_buffer_replace (&in_buffer2, NULL);
goto fail;
}
if (in_buffer2) {
if (gst_gl_view_convert_submit_input_buffer (glimage_sink->convert_views,
GST_BUFFER_IS_DISCONT (in_buffer2), in_buffer2) != GST_FLOW_OK) {
goto fail;
}
}
if (gst_gl_view_convert_get_output (glimage_sink->convert_views,
&next_buffer) != GST_FLOW_OK)
goto fail;
if (GST_VIDEO_INFO_MULTIVIEW_MODE (info) ==
GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (gst_gl_view_convert_get_output (glimage_sink->convert_views,
&next_buffer2) != GST_FLOW_OK)
goto fail;
}
gst_object_unref (convert_views);
if (next_buffer == NULL) {
/* Not ready to paint a buffer yet */
GST_GLIMAGE_SINK_LOCK (glimage_sink);
return TRUE;
}
} else {
next_buffer = in_buffer;
info = &glimage_sink->in_info;
}
/* in_buffer invalid now */
if (!gst_video_frame_map (&gl_frame, info, next_buffer,
GST_MAP_READ | GST_MAP_GL)) {
gst_buffer_unref (next_buffer);
goto fail;
}
next_sync = gst_buffer_new ();
sync_meta = gst_buffer_add_gl_sync_meta (glimage_sink->context, next_sync);
gst_gl_sync_meta_set_sync_point (sync_meta, glimage_sink->context);
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->next_tex = *(guint *) gl_frame.data[0];
old_buffer = glimage_sink->next_buffer;
glimage_sink->next_buffer = next_buffer;
old_buffer2 = glimage_sink->next_buffer2;
glimage_sink->next_buffer2 = next_buffer2;
old_sync = glimage_sink->next_sync;
glimage_sink->next_sync = next_sync;
/* Need to drop the lock again, to avoid a deadlock if we're
* dropping the last ref on this buffer and it goes back to our
* allocator */
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
if (old_buffer)
gst_buffer_unref (old_buffer);
if (old_buffer2)
gst_buffer_unref (old_buffer2);
if (old_sync)
gst_buffer_unref (old_sync);
gst_video_frame_unmap (&gl_frame);
GST_GLIMAGE_SINK_LOCK (glimage_sink);
return TRUE;
fail:
GST_GLIMAGE_SINK_LOCK (glimage_sink);
return FALSE;
}
static GstFlowReturn
gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
{
GstGLImageSink *glimage_sink;
GstVideoFrame gl_frame;
GstBuffer *next_sync, *old_sync, *old_buffer;
GstGLSyncMeta *sync_meta;
GstBuffer **target;
GstBuffer *old_input;
glimage_sink = GST_GLIMAGE_SINK (bsink);
@ -1065,31 +1357,29 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
if (!_ensure_gl_setup (glimage_sink))
return GST_FLOW_NOT_NEGOTIATED;
if (!gst_video_frame_map (&gl_frame, &glimage_sink->info, buf,
GST_MAP_READ | GST_MAP_GL)) {
goto upload_failed;
}
next_sync = gst_buffer_new ();
sync_meta = gst_buffer_add_gl_sync_meta (glimage_sink->context, next_sync);
gst_gl_sync_meta_set_sync_point (sync_meta, glimage_sink->context);
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->next_tex = *(guint *) gl_frame.data[0];
target = &glimage_sink->input_buffer;
if (GST_VIDEO_INFO_MULTIVIEW_MODE (&glimage_sink->in_info) ==
GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME &&
!GST_BUFFER_FLAG_IS_SET (buf, GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE)) {
target = &glimage_sink->input_buffer2;
}
old_input = *target;
*target = gst_buffer_ref (buf);
old_buffer = glimage_sink->next_buffer;
glimage_sink->next_buffer = gst_buffer_ref (buf);
if (glimage_sink->output_mode_changed)
update_output_format (glimage_sink);
old_sync = glimage_sink->next_sync;
glimage_sink->next_sync = next_sync;
if (!prepare_next_buffer (glimage_sink)) {
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
if (old_input)
gst_buffer_unref (old_input);
goto convert_views_failed;
}
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
if (old_buffer)
gst_buffer_unref (old_buffer);
if (old_sync)
gst_buffer_unref (old_sync);
gst_video_frame_unmap (&gl_frame);
if (old_input)
gst_buffer_unref (old_input);
if (glimage_sink->window_id != glimage_sink->new_window_id) {
GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
@ -1101,11 +1391,10 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf)
}
return GST_FLOW_OK;
upload_failed:
convert_views_failed:
{
GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
("%s", "Failed to upload buffer"), (NULL));
("%s", "Failed to convert multiview video buffer"), (NULL));
return GST_FLOW_ERROR;
}
}
@ -1114,40 +1403,23 @@ static GstFlowReturn
gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
{
GstGLImageSink *glimage_sink;
GstBuffer *stored_buffer, *old_sync;
GST_TRACE ("rendering buffer:%p", buf);
glimage_sink = GST_GLIMAGE_SINK (vsink);
GST_TRACE ("redisplay texture:%u of size:%ux%u, window size:%ux%u",
glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->info),
GST_VIDEO_INFO_HEIGHT (&glimage_sink->info),
glimage_sink->next_tex, GST_VIDEO_INFO_WIDTH (&glimage_sink->out_info),
GST_VIDEO_INFO_HEIGHT (&glimage_sink->out_info),
GST_VIDEO_SINK_WIDTH (glimage_sink),
GST_VIDEO_SINK_HEIGHT (glimage_sink));
/* Avoid to release the texture while drawing */
GST_GLIMAGE_SINK_LOCK (glimage_sink);
glimage_sink->redisplay_texture = glimage_sink->next_tex;
stored_buffer = glimage_sink->stored_buffer;
glimage_sink->stored_buffer = gst_buffer_ref (glimage_sink->next_buffer);
old_sync = glimage_sink->stored_sync;
glimage_sink->stored_sync = gst_buffer_ref (glimage_sink->next_sync);
GST_GLIMAGE_SINK_UNLOCK (glimage_sink);
/* Ask the underlying window to redraw its content */
if (!gst_glimage_sink_redisplay (glimage_sink))
goto redisplay_failed;
GST_TRACE ("post redisplay");
if (stored_buffer)
gst_buffer_unref (stored_buffer);
if (old_sync)
gst_buffer_unref (old_sync);
if (g_atomic_int_get (&glimage_sink->to_quit) != 0) {
GST_ELEMENT_ERROR (glimage_sink, RESOURCE, NOT_FOUND,
("%s", gst_gl_context_get_error ()), (NULL));
@ -1419,17 +1691,28 @@ gst_glimage_sink_cleanup_glthread (GstGLImageSink * gl_sink)
static void
gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height)
{
/* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps.
* It means that they cannot not change between two set_caps
GST_DEBUG_OBJECT (gl_sink, "GL Window resized to %ux%u", width, height);
GST_GLIMAGE_SINK_LOCK (gl_sink);
gl_sink->output_mode_changed = TRUE;
gst_glimage_sink_do_resize (gl_sink, width, height);
GST_GLIMAGE_SINK_UNLOCK (gl_sink);
}
/* Called with object lock held */
static void
gst_glimage_sink_do_resize (GstGLImageSink * gl_sink, gint width, gint height)
{
/* Here gl_sink members (ex:gl_sink->out_info) have a life time of set_caps.
* It means that they cannot change between two set_caps
*/
const GstGLFuncs *gl = gl_sink->context->gl_vtable;
gboolean do_reshape;
GST_TRACE ("GL Window resized to %ux%u", width, height);
GST_GLIMAGE_SINK_UNLOCK (gl_sink);
/* check if a client reshape callback is registered */
g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_RESHAPE_SIGNAL], 0,
gl_sink->context, width, height, &do_reshape);
GST_GLIMAGE_SINK_LOCK (gl_sink);
width = MAX (1, width);
height = MAX (1, height);
@ -1453,17 +1736,27 @@ gst_glimage_sink_on_resize (GstGLImageSink * gl_sink, gint width, gint height)
dst.h = height;
gst_video_sink_center_rect (src, dst, &result, TRUE);
gl->Viewport (result.x, result.y, result.w, result.h);
gl_sink->output_mode_changed |= (result.w != gl_sink->display_rect.w);
gl_sink->output_mode_changed |= (result.h != gl_sink->display_rect.h);
gl_sink->display_rect = result;
} else {
gl->Viewport (0, 0, width, height);
gl_sink->output_mode_changed |= (width != gl_sink->display_rect.w);
gl_sink->output_mode_changed |= (height != gl_sink->display_rect.h);
gl_sink->display_rect.x = 0;
gl_sink->display_rect.y = 0;
gl_sink->display_rect.w = width;
gl_sink->display_rect.h = height;
}
gl_sink->update_viewport = TRUE;
}
}
static void
gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
{
/* Here gl_sink members (ex:gl_sink->info) have a life time of set_caps.
/* Here gl_sink members (ex:gl_sink->out_info) have a life time of set_caps.
* It means that they cannot not change between two set_caps as well as
* for the redisplay_texture size.
* Whereas redisplay_texture id changes every sink_render
@ -1495,12 +1788,18 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
if (gl_sink->caps_change && gl_sink->window_width > 0
&& gl_sink->window_height > 0) {
GST_GLIMAGE_SINK_UNLOCK (gl_sink);
gst_glimage_sink_on_resize (gl_sink, gl_sink->window_width,
gst_glimage_sink_do_resize (gl_sink, gl_sink->window_width,
gl_sink->window_height);
GST_GLIMAGE_SINK_LOCK (gl_sink);
gl_sink->caps_change = FALSE;
}
if (gl_sink->update_viewport == TRUE) {
gl->Viewport (gl_sink->display_rect.x, gl_sink->display_rect.y,
gl_sink->display_rect.w, gl_sink->display_rect.h);
GST_DEBUG_OBJECT (gl_sink, "GL output area now %u,%u %ux%u",
gl_sink->display_rect.x, gl_sink->display_rect.y,
gl_sink->display_rect.w, gl_sink->display_rect.h);
gl_sink->update_viewport = FALSE;
}
sync_meta = gst_buffer_get_gl_sync_meta (gl_sink->stored_sync);
if (sync_meta)
@ -1514,14 +1813,20 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
gl->Disable (GL_TEXTURE_2D);
#endif
sample = gst_sample_new (gl_sink->stored_buffer,
gl_sink->caps, &GST_BASE_SINK (gl_sink)->segment, NULL);
sample = gst_sample_new (gl_sink->stored_buffer[0],
gl_sink->out_caps, &GST_BASE_SINK (gl_sink)->segment, NULL);
g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0,
gl_sink->context, sample, &do_redisplay);
gst_sample_unref (sample);
if (gl_sink->stored_buffer[1]) {
sample = gst_sample_new (gl_sink->stored_buffer[1],
gl_sink->out_caps, &GST_BASE_SINK (gl_sink)->segment, NULL);
g_signal_emit (gl_sink, gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0,
gl_sink->context, sample, &do_redisplay);
gst_sample_unref (sample);
}
if (!do_redisplay) {
gfloat alpha = gl_sink->ignore_alpha ? 1.0f : 0.0f;
@ -1591,6 +1896,7 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink)
{
GstGLWindow *window;
gboolean alive;
GstBuffer *old_stored_buffer[2], *old_sync;
window = gst_gl_context_get_window (gl_sink->context);
if (!window)
@ -1618,6 +1924,39 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink)
gst_gl_window_show (window);
}
/* Recreate the output texture if needed */
GST_GLIMAGE_SINK_LOCK (gl_sink);
if (gl_sink->output_mode_changed && gl_sink->input_buffer != NULL) {
GST_DEBUG ("Recreating output after mode/size change");
update_output_format (gl_sink);
prepare_next_buffer (gl_sink);
}
if (gl_sink->next_buffer == NULL) {
/* Nothing to display yet */
GST_GLIMAGE_SINK_UNLOCK (gl_sink);
return TRUE;
}
/* Avoid to release the texture while drawing */
gl_sink->redisplay_texture = gl_sink->next_tex;
old_stored_buffer[0] = gl_sink->stored_buffer[0];
old_stored_buffer[1] = gl_sink->stored_buffer[1];
gl_sink->stored_buffer[0] = gst_buffer_ref (gl_sink->next_buffer);
if (gl_sink->next_buffer2)
gl_sink->stored_buffer[1] = gst_buffer_ref (gl_sink->next_buffer2);
else
gl_sink->stored_buffer[1] = NULL;
old_sync = gl_sink->stored_sync;
gl_sink->stored_sync = gst_buffer_ref (gl_sink->next_sync);
GST_GLIMAGE_SINK_UNLOCK (gl_sink);
gst_buffer_replace (old_stored_buffer, NULL);
gst_buffer_replace (old_stored_buffer + 1, NULL);
if (old_sync)
gst_buffer_unref (old_sync);
/* Drawing is asynchronous: gst_gl_window_draw is not blocking
* It means that it does not wait for stuff to be executed in other threads
*/

28
ext/gl/gstglimagesink.h
View file

@ -62,9 +62,12 @@ struct _GstGLImageSink
gint width;
gint height;
//caps
GstVideoInfo info;
GstCaps *caps;
/* Input info before 3d stereo output conversion, if any */
GstVideoInfo in_info;
/* format/caps we actually hand off to the app */
GstVideoInfo out_info;
GstCaps *out_caps;
GstGLDisplay *display;
GstGLContext *context;
@ -72,8 +75,17 @@ struct _GstGLImageSink
gboolean handle_events;
gboolean ignore_alpha;
GstGLViewConvert *convert_views;
/* Original input RGBA buffer, ready for display,
* or possible reconversion through the views filter */
GstBuffer *input_buffer;
/* Secondary view buffer - when operating in frame-by-frame mode */
GstBuffer *input_buffer2;
guint next_tex;
GstBuffer *next_buffer;
GstBuffer *next_buffer2; /* frame-by-frame 2nd view */
GstBuffer *next_sync;
volatile gint to_quit;
@ -82,13 +94,16 @@ struct _GstGLImageSink
/* avoid replacing the stored_buffer while drawing */
GMutex drawing_lock;
GstBuffer *stored_buffer;
GstBuffer *stored_buffer[2];
GstBuffer *stored_sync;
GLuint redisplay_texture;
gboolean caps_change;
guint window_width;
guint window_height;
gboolean update_viewport;
GstVideoRectangle display_rect;
GstGLShader *redisplay_shader;
GLuint vao;
@ -96,6 +111,11 @@ struct _GstGLImageSink
GLuint vertex_buffer;
GLint attr_position;
GLint attr_texture;
GstVideoMultiviewMode mview_output_mode;
GstVideoMultiviewFlags mview_output_flags;
gboolean output_mode_changed;
GstGLStereoDownmix mview_downmix_mode;
};
struct _GstGLImageSinkClass