mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-13 20:06:35 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/gst-libs/gst/gl/gstglviewconvert.c
Alessandro Decina 6961945110 glmemory: add gst_gl_memory_allocator_get_default 
Add gst_gl_memory_allocator_get_default to get the default allocator based on
the opengl version. Allows us to stop hardcoding the PBO allocator which isn't
supported on gles2.

Fixes GL upload on iOS9 among other things.
2015-12-18 14:25:32 +11:00

2270 lines
78 KiB
C
Raw Blame History

/*
* GStreamer
* Copyright (C) 2009 Julien Isorce <julien.isorce@mail.com>
* Copyright (C) 2014 Jan Schmidt <jan@centricular.com>
* Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:viewconvert
*
* Convert stereoscopic/multiview video using fragment shaders.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstglviewconvert.h"
#include <gst/video/gstvideoaffinetransformationmeta.h>
#define USING_OPENGL(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL, 1, 0))
#define USING_OPENGL3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_OPENGL3, 3, 1))
#define USING_GLES(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES, 1, 0))
#define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
#define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
static GstStaticCaps caps_template =
GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), "
"format = (string) RGBA, "
"width = " GST_VIDEO_SIZE_RANGE ", "
"height = " GST_VIDEO_SIZE_RANGE ", "
"framerate = " GST_VIDEO_FPS_RANGE ", "
"texture-target = (string) { 2D, rectangle, external-oes } ");
#define GST_CAT_DEFAULT gst_gl_view_convert_debug
GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
enum
{
PROP_0,
PROP_INPUT_LAYOUT,
PROP_INPUT_FLAGS,
PROP_OUTPUT_LAYOUT,
PROP_OUTPUT_FLAGS,
PROP_OUTPUT_DOWNMIX_MODE
};
#define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
struct _GstGLViewConvertPrivate
{
gboolean result;
GstVideoMultiviewMode input_mode;
GstVideoMultiviewFlags input_flags;
GstVideoMultiviewMode output_mode;
GstVideoMultiviewFlags output_flags;
GstBuffer *primary_in;
GstBuffer *auxilliary_in;
GstBuffer *primary_out;
GstBuffer *auxilliary_out;
GstGLMemory *in_tex[GST_VIDEO_MAX_PLANES];
GstGLMemory *out_tex[GST_VIDEO_MAX_PLANES];
guint n_out_tex;
GLuint vao;
GLuint vertex_buffer;
GLuint vbo_indices;
GLuint attr_position;
GLuint attr_texture;
};
#define GST_GL_VIEW_CONVERT_GET_PRIVATE(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), \
GST_TYPE_GL_VIEW_CONVERT, GstGLViewConvertPrivate))
#define DEBUG_INIT \
GST_DEBUG_CATEGORY_INIT (gst_gl_view_convert_debug, "glviewconvert", 0, "glviewconvert object");
G_DEFINE_TYPE_WITH_CODE (GstGLViewConvert, gst_gl_view_convert,
GST_TYPE_OBJECT, DEBUG_INIT);
static void gst_gl_view_convert_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_gl_view_convert_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static void gst_gl_view_convert_finalize (GObject * object);
static void _do_view_convert (GstGLContext * context,
GstGLViewConvert * viewconvert);
GType
gst_gl_stereo_downmix_mode_get_type (void)
{
static volatile gsize g_define_type_id__volatile = 0;
if (g_once_init_enter (&g_define_type_id__volatile)) {
static const GEnumValue values[] = {
{GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS,
"Dubois optimised Green-Magenta anaglyph", "green-magenta-dubois"},
{GST_GL_STEREO_DOWNMIX_ANAGLYPH_RED_CYAN_DUBOIS,
"Dubois optimised Red-Cyan anaglyph",
"red-cyan-dubois"},
{GST_GL_STEREO_DOWNMIX_ANAGLYPH_AMBER_BLUE_DUBOIS,
"Dubois optimised Amber-Blue anaglyph", "amber-blue-dubois"},
{0, NULL, NULL}
};
GType g_define_type_id =
g_enum_register_static ("GstGLStereoDownmix", values);
g_once_init_leave (&g_define_type_id__volatile, g_define_type_id);
}
return g_define_type_id__volatile;
}
/* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
static GLfloat downmix_matrices[][2][9] = {
{ /* Green-Magenta Dubois */
{-0.062, 0.284, -0.015, -0.158, 0.668, -0.027, -0.039, 0.143, 0.021},
{0.529, -0.016, 0.009, 0.705, -0.015, 0.075, 0.024, -0.065, 0.937}
},
{ /* Red-Cyan Dubois */
/* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
{0.437, -0.062, -0.048, 0.449, -0.062, -0.050, 0.164, -0.024, -0.017},
{-0.011, 0.377, -0.026, -0.032, 0.761, -0.093, -0.007, 0.009, 1.234}
},
{ /* Amber-blue Dubois */
{1.062, -0.026, -0.038, -0.205, 0.908, -0.173, 0.299, 0.068, 0.022},
{-0.016, 0.006, 0.094, -0.123, 0.062, 0.185, -0.017, -0.017, 0.911}
}
};
static gfloat identity_matrix[] = {
1.0f, 0.0f, 0.0f, 0.0f,
0.0f, 1.0f, 0.0f, 0.0f,
0.0f, 0.0f, 1.0f, 0.0f,
0.0f, 0.0f, 0.0f, 1.0f,
};
/* *INDENT-OFF* */
static const gchar *fragment_source =
"#ifdef GL_ES\n"
"precision mediump float;\n"
"#endif\n"
"varying vec2 v_texcoord;\n"
"uniform sampler2D tex_l;\n"
"uniform sampler2D tex_r;\n"
"uniform float width;\n"
"uniform float height;\n"
"uniform mat3 downmix[2];\n"
"uniform vec2 tex_scale[2];\n"
"uniform vec2 offsets[2];\n"
"void main () {\n"
"vec4 l, r;\n"
/* input */
"%s"
/* now have left and right pixel into l and r */
/* output */
"%s"
"}\n";
static const gchar *frag_input =
" vec2 l_tex = v_texcoord * tex_scale[0] + offsets[0];\n"
" vec2 r_tex = v_texcoord * tex_scale[1] + offsets[1];\n"
" l = texture2D(tex_l, l_tex).rgba;\n"
" r = texture2D(tex_r, r_tex).rgba;\n";
static const gchar *frag_output_downmix =
" vec3 lcol = l.rgb * l.a + vec3(1.0-l.a);\n"
" vec3 rcol = r.rgb * r.a + vec3(1.0-r.a);\n"
" if (l.a + r.a > 0.0) {\n"
" lcol = clamp (downmix[0] * lcol, 0.0, 1.0);\n"
" rcol = clamp (downmix[1] * rcol, 0.0, 1.0);\n"
" gl_FragColor = vec4 (lcol + rcol, 1.0);\n"
" } else {\n"
" gl_FragColor = vec4 (0.0);\n"
" }\n";
static const gchar *frag_output_left =
" gl_FragColor = l;\n";
static const gchar *frag_output_right =
" gl_FragColor = r;\n";
static const gchar *frag_output_side_by_side =
" if (v_texcoord.x < 0.5) {\n"
" gl_FragColor = l;\n"
" } else {\n"
" gl_FragColor = r;\n"
" };\n";
static const gchar *frag_output_top_bottom =
"if (v_texcoord.y < 0.5) {\n"
" gl_FragColor = l;\n"
"} else {\n"
" gl_FragColor = r;\n"
"};\n";
static const gchar *frag_output_column_interleaved =
"if (int(mod(l_tex.x * width, 2.0)) == 0) {\n"
" gl_FragColor = l;\n"
"} else {\n"
" gl_FragColor = r;\n"
"};\n";
static const gchar *frag_output_row_interleaved =
"if (int(mod(l_tex.y * height, 2.0)) == 0) {\n"
" gl_FragColor = l;\n"
"} else {\n"
" gl_FragColor = r;\n"
"};\n";
static const gchar *frag_output_checkerboard =
"if (int(mod(l_tex.x * width, 2.0)) == \n"
" int(mod(l_tex.y * height, 2.0))) {\n"
" gl_FragColor = l;\n"
"} else {\n"
" gl_FragColor = r;\n"
"};\n";
static const gchar *frag_output_separated =
"gl_FragData[0] = l;\n"
"gl_FragData[1] = r;\n";
/* *INDENT-ON* */
static const GLfloat vertices[] = {
1.0f, -1.0f, 0.0f, 1.0f, 0.0f,
-1.0f, -1.0f, 0.0f, 0.0f, 0.0f,
-1.0f, 1.0f, 0.0f, 0.0f, 1.0f,
1.0f, 1.0f, 0.0f, 1.0f, 1.0f
};
static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 };
static void
gst_gl_view_convert_class_init (GstGLViewConvertClass * klass)
{
GObjectClass *gobject_class = (GObjectClass *) klass;
g_type_class_add_private (klass, sizeof (GstGLViewConvertPrivate));
gobject_class->set_property = gst_gl_view_convert_set_property;
gobject_class->get_property = gst_gl_view_convert_get_property;
gobject_class->finalize = gst_gl_view_convert_finalize;
g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
g_param_spec_enum ("input-mode-override",
"Input Multiview Mode Override",
"Override any input information about multiview layout",
GST_TYPE_VIDEO_MULTIVIEW_MODE,
GST_VIDEO_MULTIVIEW_MODE_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
g_param_spec_flags ("input-flags-override",
"Input Multiview Flags Override",
"Override any input information about multiview layout flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
g_param_spec_enum ("output-mode-override",
"Output Multiview Mode Override",
"Override automatic output mode selection for multiview layout",
GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
g_param_spec_flags ("output-flags-override",
"Output Multiview Flags Override",
"Override automatic negotiation for output multiview layout flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
"Output anaglyph type to generate when downmixing to mono",
GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_DOWNMIX,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
}
static void
gst_gl_view_convert_init (GstGLViewConvert * convert)
{
convert->priv = GST_GL_VIEW_CONVERT_GET_PRIVATE (convert);
convert->shader = NULL;
convert->downmix_mode = DEFAULT_DOWNMIX;
convert->priv->input_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
convert->priv->input_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
convert->priv->output_mode = GST_VIDEO_MULTIVIEW_MODE_NONE;
convert->priv->output_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
convert->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
convert->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
convert->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
convert->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
gst_video_info_init (&convert->in_info);
gst_video_info_init (&convert->out_info);
}
static void
gst_gl_view_convert_finalize (GObject * object)
{
GstGLViewConvert *viewconvert;
viewconvert = GST_GL_VIEW_CONVERT (object);
gst_gl_view_convert_reset (viewconvert);
gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
if (viewconvert->context) {
gst_object_unref (viewconvert->context);
viewconvert->context = NULL;
}
G_OBJECT_CLASS (gst_gl_view_convert_parent_class)->finalize (object);
}
GstGLViewConvert *
gst_gl_view_convert_new (void)
{
return g_object_new (GST_TYPE_GL_VIEW_CONVERT, NULL);
}
void
gst_gl_view_convert_set_context (GstGLViewConvert * viewconvert,
GstGLContext * context)
{
g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
if (gst_object_replace ((GstObject **) & viewconvert->context,
GST_OBJECT (context)))
gst_gl_view_convert_reset (viewconvert);
}
static gboolean
_view_convert_set_format (GstGLViewConvert * viewconvert,
GstVideoInfo * in_info, GstVideoInfo * out_info)
{
g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
if (gst_video_info_is_equal (in_info, &viewconvert->in_info) &&
gst_video_info_is_equal (out_info, &viewconvert->out_info))
return TRUE;
if (GST_VIDEO_INFO_FORMAT (in_info) != GST_VIDEO_FORMAT_RGBA ||
GST_VIDEO_INFO_FORMAT (out_info) != GST_VIDEO_FORMAT_RGBA) {
GST_ERROR_OBJECT (viewconvert,
"Multiview conversion can currently only be performed on RGBA textures");
return FALSE;
}
/* FIXME: Compare what changed and decide if we need a full reset or not */
GST_OBJECT_LOCK (viewconvert);
gst_gl_view_convert_reset (viewconvert);
viewconvert->in_info = *in_info;
viewconvert->out_info = *out_info;
gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
gst_buffer_replace (&viewconvert->priv->primary_out, NULL);
gst_buffer_replace (&viewconvert->priv->auxilliary_out, NULL);
GST_OBJECT_UNLOCK (viewconvert);
return TRUE;
}
/**
* gst_gl_view_convert_set_caps:
* @viewconvert: a #GstGLViewConvert
* @in_caps: input #GstCaps
* @out_caps: output #GstCaps
*
* Initializes @viewconvert with the information required for conversion.
*/
gboolean
gst_gl_view_convert_set_caps (GstGLViewConvert * viewconvert,
GstCaps * in_caps, GstCaps * out_caps)
{
GstVideoInfo in_info, out_info;
GstCapsFeatures *in_features, *out_features;
g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), FALSE);
g_return_val_if_fail (GST_IS_CAPS (in_caps), FALSE);
g_return_val_if_fail (GST_IS_CAPS (out_caps), FALSE);
GST_INFO_OBJECT (viewconvert,
"Configuring multiview conversion from caps %" GST_PTR_FORMAT
" to %" GST_PTR_FORMAT, in_caps, out_caps);
in_features = gst_caps_get_features (in_caps, 0);
out_features = gst_caps_get_features (out_caps, 0);
if (!gst_caps_features_contains (in_features,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
return FALSE;
if (!gst_caps_features_contains (out_features,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY))
return FALSE;
if (!gst_video_info_from_caps (&in_info, in_caps))
return FALSE;
if (!gst_video_info_from_caps (&out_info, out_caps))
return FALSE;
return _view_convert_set_format (viewconvert, &in_info, &out_info);
}
/* Function that can halve the value
* of ints, fractions, int/fraction ranges and lists of ints/fractions */
static gboolean
_halve_value (GValue * out, const GValue * in_value)
{
/* Fundamental fixed types first */
if (G_VALUE_HOLDS_INT (in_value)) {
g_value_init (out, G_TYPE_INT);
g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
} else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
gint num, den;
num = gst_value_get_fraction_numerator (in_value);
den = gst_value_get_fraction_denominator (in_value);
g_value_init (out, GST_TYPE_FRACTION);
/* Don't adjust 'infinite' fractions */
if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
/* FIXME - could do better approximation when den > G_MAXINT/2? */
den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
}
gst_value_set_fraction (out, num, den);
} else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
gint range_min = gst_value_get_int_range_min (in_value);
gint range_max = gst_value_get_int_range_max (in_value);
gint range_step = gst_value_get_int_range_step (in_value);
g_value_init (out, GST_TYPE_INT_RANGE);
if (range_min != 1)
range_min = MAX (1, range_min / 2);
if (range_max != G_MAXINT)
range_max = MAX (1, range_max / 2);
gst_value_set_int_range_step (out, range_min,
range_max, MAX (1, range_step / 2));
} else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
GValue min_out = G_VALUE_INIT;
GValue max_out = G_VALUE_INIT;
const GValue *range_min = gst_value_get_fraction_range_min (in_value);
const GValue *range_max = gst_value_get_fraction_range_max (in_value);
_halve_value (&min_out, range_min);
_halve_value (&max_out, range_max);
g_value_init (out, GST_TYPE_FRACTION_RANGE);
gst_value_set_fraction_range (out, &min_out, &max_out);
g_value_unset (&min_out);
g_value_unset (&max_out);
} else if (GST_VALUE_HOLDS_LIST (in_value)) {
gint i;
g_value_init (out, GST_TYPE_LIST);
for (i = 0; i < gst_value_list_get_size (in_value); i++) {
const GValue *entry;
GValue tmp = G_VALUE_INIT;
entry = gst_value_list_get_value (in_value, i);
/* Random list values might not be the right type */
if (!_halve_value (&tmp, entry))
goto fail;
gst_value_list_append_and_take_value (out, &tmp);
}
} else {
return FALSE;
}
return TRUE;
fail:
g_value_unset (out);
return FALSE;
}
static GstStructure *
_halve_structure_field (const GstStructure * in, const gchar * field_name)
{
GstStructure *out;
const GValue *in_value = gst_structure_get_value (in, field_name);
GValue tmp = G_VALUE_INIT;
if (G_UNLIKELY (in_value == NULL))
return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
if (!_halve_value (&tmp, in_value))
return NULL;
out = gst_structure_copy (in);
gst_structure_set_value (out, field_name, &tmp);
g_value_unset (&tmp);
return out;
}
/* Function that can double the value
* of ints, fractions, int/fraction ranges and lists of ints/fractions */
static gboolean
_double_value (GValue * out, const GValue * in_value)
{
/* Fundamental fixed types first */
if (G_VALUE_HOLDS_INT (in_value)) {
gint n = g_value_get_int (in_value);
g_value_init (out, G_TYPE_INT);
if (n <= G_MAXINT / 2)
g_value_set_int (out, n * 2);
else
g_value_set_int (out, G_MAXINT);
} else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
gint num, den;
num = gst_value_get_fraction_numerator (in_value);
den = gst_value_get_fraction_denominator (in_value);
g_value_init (out, GST_TYPE_FRACTION);
/* Don't adjust 'infinite' fractions */
if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
/* FIXME - could do better approximation when num > G_MAXINT/2? */
num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
}
gst_value_set_fraction (out, num, den);
} else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
gint range_min = gst_value_get_int_range_min (in_value);
gint range_max = gst_value_get_int_range_max (in_value);
gint range_step = gst_value_get_int_range_step (in_value);
if (range_min != 1) {
range_min = MIN (G_MAXINT / 2, range_min);
range_min *= 2;
}
if (range_max != G_MAXINT) {
range_max = MIN (G_MAXINT / 2, range_max);
range_max *= 2;
}
range_step = MIN (G_MAXINT / 2, range_step);
g_value_init (out, GST_TYPE_INT_RANGE);
gst_value_set_int_range_step (out, range_min, range_max, range_step);
} else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
GValue min_out = G_VALUE_INIT;
GValue max_out = G_VALUE_INIT;
const GValue *range_min = gst_value_get_fraction_range_min (in_value);
const GValue *range_max = gst_value_get_fraction_range_max (in_value);
_double_value (&min_out, range_min);
_double_value (&max_out, range_max);
g_value_init (out, GST_TYPE_FRACTION_RANGE);
gst_value_set_fraction_range (out, &min_out, &max_out);
g_value_unset (&min_out);
g_value_unset (&max_out);
} else if (GST_VALUE_HOLDS_LIST (in_value)) {
gint i;
g_value_init (out, GST_TYPE_LIST);
for (i = 0; i < gst_value_list_get_size (in_value); i++) {
const GValue *entry;
GValue tmp = G_VALUE_INIT;
entry = gst_value_list_get_value (in_value, i);
/* Random list values might not be the right type */
if (!_double_value (&tmp, entry))
goto fail;
gst_value_list_append_and_take_value (out, &tmp);
}
} else {
return FALSE;
}
return TRUE;
fail:
g_value_unset (out);
return FALSE;
}
static GstStructure *
_double_structure_field (const GstStructure * in, const gchar * field_name)
{
GstStructure *out;
const GValue *in_value = gst_structure_get_value (in, field_name);
GValue tmp = G_VALUE_INIT;
if (G_UNLIKELY (in_value == NULL))
return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
if (!_double_value (&tmp, in_value))
return NULL;
out = gst_structure_copy (in);
gst_structure_set_value (out, field_name, &tmp);
g_value_unset (&tmp);
return out;
}
/* Return a copy of the caps with the requested field halved in value/range */
#if 0
static GstCaps *
_halve_caps_field (const GstCaps * in, const gchar * field_name)
{
gint i;
GstCaps *out = gst_caps_new_empty ();
for (i = 0; i < gst_caps_get_size (in); i++) {
const GstStructure *cur = gst_caps_get_structure (in, i);
GstCapsFeatures *f = gst_caps_get_features (in, i);
GstStructure *res = _halve_structure_field (cur, field_name);
out =
gst_caps_merge_structure_full (out, res,
f ? gst_caps_features_copy (f) : NULL);
}
return out;
}
#endif
/* Return a copy of the caps with the requested field doubled in value/range */
static GstCaps *
_double_caps_field (const GstCaps * in, const gchar * field_name)
{
gint i;
GstCaps *out = gst_caps_new_empty ();
for (i = 0; i < gst_caps_get_size (in); i++) {
const GstStructure *cur = gst_caps_get_structure (in, i);
GstCapsFeatures *f = gst_caps_get_features (in, i);
GstStructure *res = _double_structure_field (cur, field_name);
out =
gst_caps_merge_structure_full (out, res,
f ? gst_caps_features_copy (f) : NULL);
}
return out;
}
/* Takes ownership of the input caps */
static GstCaps *
_expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
{
guint mview_flags, mview_flags_mask;
GstCaps *out;
GstStructure *tmp;
out = gst_caps_new_empty ();
while (gst_caps_get_size (in) > 0) {
GstStructure *s;
GstCapsFeatures *features;
features = gst_caps_get_features (in, 0);
if (features)
features = gst_caps_features_copy (features);
s = gst_caps_steal_structure (in, 0);
if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
gst_caps_append_structure_full (out, s, features);
continue;
}
/* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
gst_caps_append_structure_full (out, s, features);
continue;
}
if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
/* No par field, dont-care the half-aspect flag */
gst_structure_set (s, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_caps_append_structure_full (out, s, features);
continue;
}
/* Halve or double PAR base on inputs input specified. */
/* Append a copy with the half-aspect flag as-is */
tmp = gst_structure_copy (s);
out = gst_caps_merge_structure_full (out, tmp,
features ? gst_caps_features_copy (features) : NULL);
/* and then a copy inverted */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
/* Input is half-aspect. Double/halve the PAR, clear the flag */
if (vertical_half_aspect)
tmp = _halve_structure_field (s, "pixel-aspect-ratio");
else
tmp = _double_structure_field (s, "pixel-aspect-ratio");
/* Clear the flag */
gst_structure_set (tmp, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
} else {
if (vertical_half_aspect)
tmp = _double_structure_field (s, "pixel-aspect-ratio");
else
tmp = _halve_structure_field (s, "pixel-aspect-ratio");
/* Set the flag */
gst_structure_set (tmp, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
out = gst_caps_merge_structure_full (out, tmp,
features ? gst_caps_features_copy (features) : NULL);
gst_structure_free (s);
if (features)
gst_caps_features_free (features);
}
gst_caps_unref (in);
return out;
}
/* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
* If input supports left-right, checkerboard, quincunx or column-interleaved,
* we may halve width to mono frames.
* For output of top-bottom or row-interleaved, we may double the mono height
* For output of left-right, checkerboard, quincunx or column-interleaved,
* we may double the mono width.
* In all cases, if input has half-aspect and output does not, we may double the PAR
* And if input does *not* have half-aspect flag and output does not, we may halve the PAR
*/
static GstCaps *
_expand_structure (GstGLViewConvert * viewconvert,
GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
{
GstCaps *expanded_caps, *tmp;
GstCaps *mono_caps;
const gchar *default_mview_mode_str = NULL;
guint mview_flags, mview_flags_mask;
const GValue *in_modes;
gint i;
/* Empty caps to accumulate into */
expanded_caps = gst_caps_new_empty ();
/* First, set defaults if multiview flags are missing */
default_mview_mode_str =
gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
if (!gst_structure_has_field (structure, "multiview-mode")) {
gst_structure_set (structure,
"multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
}
if (!gst_structure_has_field (structure, "multiview-flags")) {
gst_structure_set (structure,
"multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
mview_flags_mask, NULL);
} else {
gst_structure_get_flagset (structure, "multiview-flags",
&mview_flags, &mview_flags_mask);
}
in_modes = gst_structure_get_value (structure, "multiview-mode");
mono_caps = gst_caps_new_empty ();
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_mono_modes ())) {
GstStructure *new_struct = gst_structure_copy (structure);
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Half-aspect makes no sense for mono or unpacked, get rid of it */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
gst_structure_set (new_struct, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
gst_caps_append_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_unpacked_modes ())) {
GstStructure *new_struct = gst_structure_copy (structure);
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Half-aspect makes no sense for mono or unpacked, get rid of it */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
gst_structure_set (new_struct, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
gst_caps_append_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_height_modes ())) {
/* Append mono formats with height halved */
GstStructure *new_struct = _halve_structure_field (structure, "height");
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Normalise the half-aspect flag away */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
GstStructure *s =
_halve_structure_field (new_struct, "pixel-aspect-ratio");
gst_structure_set (structure, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_structure_free (new_struct);
new_struct = s;
}
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_width_modes ())) {
/* Append mono formats with width halved */
GstStructure *new_struct = _halve_structure_field (structure, "width");
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Normalise the half-aspect flag away */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
GstStructure *s =
_double_structure_field (new_struct, "pixel-aspect-ratio");
gst_structure_set (structure, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_structure_free (new_struct);
new_struct = s;
}
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_size_modes ())) {
/* Append checkerboard/doubled size formats with width & height halved */
GstStructure *new_struct_w = _halve_structure_field (structure, "width");
GstStructure *new_struct_wh =
_halve_structure_field (new_struct_w, "height");
gst_structure_free (new_struct_w);
gst_structure_set_value (new_struct_wh, "multiview-mode",
gst_video_multiview_get_mono_modes ());
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
features ? gst_caps_features_copy (features) : NULL);
}
/* Everything is normalised now, unset the flags we can change */
/* Remove the views field, as these are all 'mono' modes
* Need to do this before we expand caps back out to frame packed modes */
for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
GstStructure *s = gst_caps_get_structure (mono_caps, i);
gst_structure_remove_fields (s, "views", NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* Preserve only the half-aspect and mixed-mono flags, for now.
* The rest we can change */
mview_flags_mask &=
(GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
GST_TRACE_OBJECT (viewconvert,
"Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
/* Put unpacked and mono modes first. We don't care about flags. Clear them */
tmp = gst_caps_copy (mono_caps);
for (i = 0; i < gst_caps_get_size (tmp); i++) {
GstStructure *s = gst_caps_get_structure (tmp, i);
gst_structure_remove_fields (s, "views", NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
mview_flags_mask = 0;
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Unpacked output modes have 2 views, for now */
tmp = gst_caps_copy (mono_caps);
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_unpacked_modes ());
for (i = 0; i < gst_caps_get_size (tmp); i++) {
GstStructure *s = gst_caps_get_structure (tmp, i);
gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
mview_flags_mask = 0;
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double height output modes */
tmp = _double_caps_field (mono_caps, "height");
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_height_modes ());
tmp = _expand_par_for_half_aspect (tmp, TRUE);
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double width output modes */
tmp = _double_caps_field (mono_caps, "width");
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_width_modes ());
tmp = _expand_par_for_half_aspect (tmp, FALSE);
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double size output modes */
{
GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
tmp = _double_caps_field (tmp_w, "height");
gst_caps_unref (tmp_w);
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_size_modes ());
expanded_caps = gst_caps_merge (expanded_caps, tmp);
}
/* We're done with the mono caps now */
gst_caps_unref (mono_caps);
GST_TRACE_OBJECT (viewconvert,
"expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
if (gst_caps_is_empty (expanded_caps)) {
gst_caps_unref (expanded_caps);
return out_caps;
}
/* Really, we can rescale - so at this point we can append full-range
* height/width/PAR as an unpreferred final option. */
tmp = gst_caps_copy (expanded_caps);
gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
"height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
out_caps = gst_caps_merge (out_caps, expanded_caps);
out_caps = gst_caps_merge (out_caps, tmp);
return out_caps;
}
static GstCaps *
_intersect_with_mview_mode (GstCaps * caps,
GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
{
GstCaps *filter, *result;
const gchar *caps_str;
caps_str = gst_video_multiview_mode_to_caps_string (mode);
filter = gst_caps_new_simple ("video/x-raw",
"multiview-mode", G_TYPE_STRING,
caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
GST_FLAG_SET_MASK_EXACT, NULL);
if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
" with caps %" GST_PTR_FORMAT, caps, filter);
result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (filter);
return result;
}
static GstCaps *
_intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
{
GstCaps *filter, *result;
filter = gst_caps_new_empty_simple ("video/x-raw");
gst_caps_set_value (filter, "multiview-mode", modes);
gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
" with caps %" GST_PTR_FORMAT, caps, filter);
result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (filter);
return result;
}
GstCaps *
gst_gl_view_convert_transform_caps (GstGLViewConvert * viewconvert,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
gint i, n;
GstCaps *base_caps = gst_static_caps_get (&caps_template);
GstCaps *out_caps, *tmp_caps;
g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
GST_DEBUG_OBJECT (viewconvert, "Direction %s "
"input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
/* We can only process GLmemory RGBA caps, start from that */
caps = gst_caps_intersect (caps, base_caps);
gst_caps_unref (base_caps);
/* Change input/output to the formats we can convert to/from,
* but keep the original caps at the start - we will always prefer
* passthrough */
if (direction == GST_PAD_SINK) {
out_caps = gst_caps_copy (caps);
if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
GstVideoMultiviewMode mode = viewconvert->input_mode_override;
GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
/* Coerce the input caps before transforming, so the sizes come out right */
gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
GST_FLAG_SET_MASK_EXACT, NULL);
}
} else {
out_caps = gst_caps_new_empty ();
}
for (i = 0; i < gst_caps_get_size (caps); i++) {
GstStructure *structure = gst_caps_get_structure (caps, i);
GstCapsFeatures *features = gst_caps_get_features (caps, i);
out_caps = _expand_structure (viewconvert, out_caps, structure, features);
}
if (gst_caps_is_empty (out_caps))
goto out;
/* If we have an output mode override, limit things to that */
if (direction == GST_PAD_SINK &&
viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
tmp_caps = _intersect_with_mview_mode (out_caps,
viewconvert->output_mode_override, viewconvert->output_flags_override);
gst_caps_unref (out_caps);
out_caps = tmp_caps;
} else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* Prepend a copy of our preferred input caps in case the peer
* can handle them */
tmp_caps = _intersect_with_mview_mode (out_caps,
viewconvert->input_mode_override, viewconvert->input_flags_override);
out_caps = gst_caps_merge (out_caps, tmp_caps);
}
if (direction == GST_PAD_SRC) {
GstStructure *s;
/* When generating input caps, we also need a copy of the mono caps
* without multiview-mode or flags for backwards compat, at the end */
tmp_caps = _intersect_with_mview_mode (caps,
GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
if (!gst_caps_is_empty (tmp_caps)) {
s = gst_caps_get_structure (tmp_caps, 0);
gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
NULL);
out_caps = gst_caps_merge (out_caps, tmp_caps);
} else
gst_caps_unref (tmp_caps);
}
out:
gst_caps_unref (caps);
n = gst_caps_get_size (out_caps);
for (i = 0; i < n; i++) {
GstStructure *s = gst_caps_get_structure (out_caps, i);
gst_structure_remove_fields (s, "texture-target", NULL);
}
GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
return out_caps;
}
static guint
_get_target_bitmask_from_g_value (const GValue * targets)
{
guint new_targets = 0;
if (targets == NULL) {
new_targets = 1 << GST_GL_TEXTURE_TARGET_2D;
} else if (G_TYPE_CHECK_VALUE_TYPE (targets, G_TYPE_STRING)) {
GstGLTextureTarget target;
const gchar *str;
str = g_value_get_string (targets);
target = gst_gl_texture_target_from_string (str);
if (target)
new_targets |= 1 << target;
} else if (G_TYPE_CHECK_VALUE_TYPE (targets, GST_TYPE_LIST)) {
gint j, m;
m = gst_value_list_get_size (targets);
for (j = 0; j < m; j++) {
const GValue *val = gst_value_list_get_value (targets, j);
GstGLTextureTarget target;
const gchar *str;
str = g_value_get_string (val);
target = gst_gl_texture_target_from_string (str);
if (target)
new_targets |= 1 << target;
}
}
return new_targets;
}
static GstCaps *
_fixate_texture_target (GstGLViewConvert * viewconvert,
GstPadDirection direction, GstCaps * caps, GstCaps * other)
{
GValue item = G_VALUE_INIT;
const GValue *targets, *other_targets;
guint targets_mask = 0, other_targets_mask = 0, result_mask;
GstStructure *s, *s_other;
other = gst_caps_make_writable (other);
s = gst_caps_get_structure (caps, 0);
s_other = gst_caps_get_structure (other, 0);
other_targets = gst_structure_get_value (s_other, "texture-target");
targets = gst_structure_get_value (s, "texture-target");
targets_mask = _get_target_bitmask_from_g_value (targets);
other_targets_mask = _get_target_bitmask_from_g_value (other_targets);
result_mask = targets_mask & other_targets_mask;
if (result_mask == 0) {
/* nothing we can do here */
return gst_caps_fixate (other);
}
if (direction == GST_PAD_SINK) {
result_mask &=
(1 << GST_GL_TEXTURE_TARGET_2D | 1 << GST_GL_TEXTURE_TARGET_RECTANGLE);
} else {
/* if the src caps has 2D support we can 'convert' to anything */
if (targets_mask & (1 << GST_GL_TEXTURE_TARGET_2D))
result_mask = -1;
else
result_mask = other_targets_mask;
}
g_value_init (&item, G_TYPE_STRING);
if (result_mask & (1 << GST_GL_TEXTURE_TARGET_2D)) {
g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_2D_STR);
} else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_RECTANGLE)) {
g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_RECTANGLE_STR);
} else if (result_mask & (1 << GST_GL_TEXTURE_TARGET_EXTERNAL_OES)) {
g_value_set_static_string (&item, GST_GL_TEXTURE_TARGET_EXTERNAL_OES_STR);
}
gst_structure_set_value (s, "texture-target", &item);
g_value_unset (&item);
return gst_caps_fixate (other);
}
GstCaps *
gst_gl_view_convert_fixate_caps (GstGLViewConvert * viewconvert,
GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
{
GstVideoMultiviewMode mode = viewconvert->output_mode_override;
GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
GstCaps *tmp;
g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), NULL);
othercaps = gst_caps_make_writable (othercaps);
GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
" against caps %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
if (direction == GST_PAD_SINK) {
if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* We have a requested output mode and are fixating source caps, try and enforce it */
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
gst_caps_unref (othercaps);
othercaps = tmp;
} else {
/* See if we can do passthrough */
GstVideoInfo info;
if (gst_video_info_from_caps (&info, caps)) {
GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
mode = viewconvert->input_mode_override;
flags = viewconvert->input_flags_override;
}
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
if (gst_caps_is_empty (tmp)) {
/* Nope, we can't pass our input caps downstream */
gst_caps_unref (tmp);
} else {
gst_caps_unref (othercaps);
othercaps = tmp;
goto done;
}
}
/* Prefer an unpacked mode for output */
tmp =
_intersect_with_mview_modes (othercaps,
gst_video_multiview_get_unpacked_modes ());
if (!gst_caps_is_empty (tmp)) {
gst_caps_unref (othercaps);
othercaps = tmp;
} else {
gst_caps_unref (tmp);
}
}
} else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* See if we can coerce the caps into matching input mode/flags,
* in case it doesn't care at all, but allow it not to too */
mode = viewconvert->input_mode_override;
flags = viewconvert->input_flags_override;
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
if (gst_caps_is_empty (tmp)) {
/* Nope, we can pass our input caps downstream */
gst_caps_unref (tmp);
} else {
gst_caps_unref (othercaps);
othercaps = tmp;
}
}
tmp = _fixate_texture_target (viewconvert, direction, caps, othercaps);
gst_caps_unref (othercaps);
othercaps = tmp;
done:
GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
" against caps %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
return othercaps;
}
void
gst_gl_view_convert_reset (GstGLViewConvert * viewconvert)
{
g_return_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert));
if (viewconvert->shader)
gst_gl_context_del_shader (viewconvert->context, viewconvert->shader);
viewconvert->shader = NULL;
viewconvert->initted = FALSE;
viewconvert->reconfigure = FALSE;
}
static void
gst_gl_view_convert_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
switch (prop_id) {
case PROP_INPUT_LAYOUT:
convert->input_mode_override = g_value_get_enum (value);
break;
case PROP_INPUT_FLAGS:
convert->input_flags_override = g_value_get_flags (value);
break;
case PROP_OUTPUT_LAYOUT:
convert->output_mode_override = g_value_get_enum (value);
break;
case PROP_OUTPUT_FLAGS:
convert->output_flags_override = g_value_get_flags (value);
break;
case PROP_OUTPUT_DOWNMIX_MODE:
convert->downmix_mode = g_value_get_enum (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_LOCK (convert);
convert->reconfigure = TRUE;
GST_OBJECT_UNLOCK (convert);
}
static void
gst_gl_view_convert_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstGLViewConvert *convert = GST_GL_VIEW_CONVERT (object);
switch (prop_id) {
case PROP_INPUT_LAYOUT:
g_value_set_enum (value, convert->input_mode_override);
break;
case PROP_INPUT_FLAGS:
g_value_set_flags (value, convert->input_flags_override);
break;
case PROP_OUTPUT_LAYOUT:
g_value_set_enum (value, convert->output_mode_override);
break;
case PROP_OUTPUT_FLAGS:
g_value_set_flags (value, convert->output_flags_override);
break;
case PROP_OUTPUT_DOWNMIX_MODE:
g_value_set_enum (value, convert->downmix_mode);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
GstBuffer *
gst_gl_view_convert_perform (GstGLViewConvert * viewconvert, GstBuffer * inbuf)
{
GstBuffer *out;
if (gst_gl_view_convert_submit_input_buffer (viewconvert,
GST_BUFFER_IS_DISCONT (inbuf), gst_buffer_ref (inbuf)) != GST_FLOW_OK)
return NULL;
if (gst_gl_view_convert_get_output (viewconvert, &out) != GST_FLOW_OK)
return NULL;
return out;
}
/* called by _init_convert (in the gl thread) */
static gboolean
_init_view_convert_fbo (GstGLViewConvert * viewconvert)
{
GstGLFuncs *gl;
guint out_width, out_height;
GLuint fake_texture = 0; /* a FBO must hava texture to init */
GLenum internal_format;
gboolean ret = TRUE;
gl = viewconvert->context->gl_vtable;
out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
if (!gl->GenFramebuffers) {
/* turn off the pipeline because Frame buffer object is a not present */
gst_gl_context_set_error (viewconvert->context,
"Frambuffer objects unsupported");
return FALSE;
}
/* setup FBO */
gl->GenFramebuffers (1, &viewconvert->fbo);
gl->BindFramebuffer (GL_FRAMEBUFFER, viewconvert->fbo);
/* setup the render buffer for depth */
gl->GenRenderbuffers (1, &viewconvert->depth_buffer);
gl->BindRenderbuffer (GL_RENDERBUFFER, viewconvert->depth_buffer);
if (USING_OPENGL (viewconvert->context)
|| USING_OPENGL3 (viewconvert->context)) {
gl->RenderbufferStorage (GL_RENDERBUFFER, GL_DEPTH_COMPONENT, out_width,
out_height);
gl->RenderbufferStorage (GL_RENDERBUFFER, GL_DEPTH24_STENCIL8,
out_width, out_height);
}
if (USING_GLES2 (viewconvert->context)) {
gl->RenderbufferStorage (GL_RENDERBUFFER, GL_DEPTH_COMPONENT16,
out_width, out_height);
}
/* a fake texture is attached to the convert FBO (cannot init without it) */
gl->GenTextures (1, &fake_texture);
gl->BindTexture (GL_TEXTURE_2D, fake_texture);
internal_format =
gst_gl_sized_gl_format_from_gl_format_type (viewconvert->context, GL_RGBA,
GL_UNSIGNED_BYTE);
gl->TexImage2D (GL_TEXTURE_2D, 0, internal_format, out_width, out_height,
0, GL_RGBA, GL_UNSIGNED_BYTE, NULL);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
gl->TexParameteri (GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
/* attach the texture to the FBO to renderer to */
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_TEXTURE_2D, fake_texture, 0);
/* attach the depth render buffer to the FBO */
gl->FramebufferRenderbuffer (GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, viewconvert->depth_buffer);
if (USING_OPENGL (viewconvert->context)) {
gl->FramebufferRenderbuffer (GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT,
GL_RENDERBUFFER, viewconvert->depth_buffer);
}
if (!gst_gl_context_check_framebuffer_status (viewconvert->context)) {
gst_gl_context_set_error (viewconvert->context,
"GL framebuffer status incomplete");
ret = FALSE;
}
/* unbind the FBO */
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
gl->DeleteTextures (1, &fake_texture);
return ret;
}
static gchar *
_mangle_texture_access (const gchar * str, GstGLTextureTarget from,
GstGLTextureTarget to)
{
const gchar *from_str = NULL, *to_str = NULL;
gchar *ret, *tmp;
gchar *regex_find;
GRegex *regex;
if (from == GST_GL_TEXTURE_TARGET_2D)
from_str = "texture2D";
if (from == GST_GL_TEXTURE_TARGET_RECTANGLE)
from_str = "texture2DRect";
if (from == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
from_str = "texture2D";
if (to == GST_GL_TEXTURE_TARGET_2D)
to_str = "texture2D";
if (to == GST_GL_TEXTURE_TARGET_RECTANGLE)
to_str = "texture2DRect";
if (to == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
to_str = "texture2D";
/* followed by any amount of whitespace then a bracket */
regex_find = g_strdup_printf ("%s(?=\\s*\\()", from_str);
regex = g_regex_new (regex_find, 0, 0, NULL);
tmp = g_regex_replace_literal (regex, str, -1, 0, to_str, 0, NULL);
g_free (regex_find);
g_regex_unref (regex);
if (tmp) {
ret = tmp;
} else {
GST_FIXME ("Couldn't mangle texture access successfully from %s to %s",
from_str, to_str);
ret = g_strdup (str);
}
return ret;
}
static gchar *
_mangle_sampler_type (const gchar * str, GstGLTextureTarget from,
GstGLTextureTarget to)
{
const gchar *from_str = NULL, *to_str = NULL;
gchar *ret, *tmp;
gchar *regex_find;
GRegex *regex;
if (from == GST_GL_TEXTURE_TARGET_2D)
from_str = "sampler2D";
if (from == GST_GL_TEXTURE_TARGET_RECTANGLE)
from_str = "sampler2DRect";
if (from == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
from_str = "samplerExternalOES";
if (to == GST_GL_TEXTURE_TARGET_2D)
to_str = "sampler2D";
if (to == GST_GL_TEXTURE_TARGET_RECTANGLE)
to_str = "sampler2DRect";
if (to == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
to_str = "samplerExternalOES";
/* followed by some whitespace */
regex_find = g_strdup_printf ("%s(?=\\s)", from_str);
regex = g_regex_new (regex_find, 0, 0, NULL);
tmp = g_regex_replace_literal (regex, str, -1, 0, to_str, 0, NULL);
g_free (regex_find);
g_regex_unref (regex);
if (tmp) {
ret = tmp;
} else {
GST_FIXME ("Couldn't mangle sampler type successfully from %s to %s",
from_str, to_str);
ret = g_strdup (str);
}
return ret;
}
static gchar *
_mangle_extensions (const gchar * str, GstGLTextureTarget from)
{
const gchar *ext_str = NULL;
if (from == GST_GL_TEXTURE_TARGET_EXTERNAL_OES)
ext_str = "#extension GL_OES_EGL_image_external : require\n";
if (ext_str) {
return g_strdup_printf ("%s%s", ext_str, str);
} else {
return g_strdup (str);
}
}
/* free after use */
static gchar *
_get_shader_string (GstGLViewConvert * viewconvert,
GstVideoMultiviewMode in_mode, GstVideoMultiviewMode out_mode)
{
const gchar *input_str, *output_str;
gboolean mono_input = FALSE;
gchar *tmp, *tmp2;
switch (in_mode) {
case GST_VIDEO_MULTIVIEW_MODE_NONE:
case GST_VIDEO_MULTIVIEW_MODE_MONO:
case GST_VIDEO_MULTIVIEW_MODE_LEFT:
case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
mono_input = TRUE;
/* Fall through */
default:
input_str = frag_input;
break;
}
switch (out_mode) {
case GST_VIDEO_MULTIVIEW_MODE_LEFT:
output_str = frag_output_left;
break;
case GST_VIDEO_MULTIVIEW_MODE_RIGHT:
output_str = frag_output_right;
break;
case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX:
/* FIXME: implement properly with sub-sampling */
case GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE:
output_str = frag_output_side_by_side;
break;
case GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM:
output_str = frag_output_top_bottom;
break;
case GST_VIDEO_MULTIVIEW_MODE_COLUMN_INTERLEAVED:
output_str = frag_output_column_interleaved;
break;
case GST_VIDEO_MULTIVIEW_MODE_ROW_INTERLEAVED:
output_str = frag_output_row_interleaved;
break;
case GST_VIDEO_MULTIVIEW_MODE_SEPARATED:
case GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME:
output_str = frag_output_separated;
break;
case GST_VIDEO_MULTIVIEW_MODE_CHECKERBOARD:
output_str = frag_output_checkerboard;
break;
case GST_VIDEO_MULTIVIEW_MODE_NONE:
case GST_VIDEO_MULTIVIEW_MODE_MONO:
default:
if (mono_input)
output_str = frag_output_left;
else
output_str = frag_output_downmix;
break;
}
tmp = g_strdup_printf (fragment_source, input_str, output_str);
tmp2 = _mangle_sampler_type (tmp, GST_GL_TEXTURE_TARGET_2D,
viewconvert->from_texture_target);
g_free (tmp);
tmp = _mangle_texture_access (tmp2, GST_GL_TEXTURE_TARGET_2D,
viewconvert->from_texture_target);
g_free (tmp2);
tmp2 = _mangle_extensions (tmp, viewconvert->from_texture_target);
g_free (tmp);
return tmp2;
}
static void
_bind_buffer (GstGLViewConvert * viewconvert)
{
const GstGLFuncs *gl = viewconvert->context->gl_vtable;
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
/* Load the vertex position */
gl->VertexAttribPointer (viewconvert->priv->attr_position, 3, GL_FLOAT,
GL_FALSE, 5 * sizeof (GLfloat), (void *) 0);
/* Load the texture coordinate */
gl->VertexAttribPointer (viewconvert->priv->attr_texture, 2, GL_FLOAT,
GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat)));
gl->EnableVertexAttribArray (viewconvert->priv->attr_position);
gl->EnableVertexAttribArray (viewconvert->priv->attr_texture);
}
static void
_unbind_buffer (GstGLViewConvert * viewconvert)
{
const GstGLFuncs *gl = viewconvert->context->gl_vtable;
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
gl->DisableVertexAttribArray (viewconvert->priv->attr_position);
gl->DisableVertexAttribArray (viewconvert->priv->attr_texture);
}
/* Called in the gl thread */
static gboolean
_init_view_convert (GstGLViewConvert * viewconvert)
{
GstGLViewConvertPrivate *priv = viewconvert->priv;
GstVideoMultiviewMode in_mode = priv->input_mode;
GstVideoMultiviewMode out_mode = priv->output_mode;
GstVideoMultiviewFlags in_flags = priv->input_flags;
GstVideoMultiviewFlags out_flags = priv->output_flags;
gfloat tex_scale[2][2] = {
{1., 1.},
{1., 1.}
};
gfloat offsets[2][2] = {
{0., 0.},
{0., 0.}
};
gchar *fragment_source_str;
GstGLFuncs *gl;
gboolean res;
gint l_index, r_index;
gl = viewconvert->context->gl_vtable;
if (viewconvert->reconfigure)
gst_gl_view_convert_reset (viewconvert);
if (viewconvert->initted)
return TRUE;
GST_LOG_OBJECT (viewconvert,
"Initializing multiview conversion from %s mode %d flags 0x%x w %u h %u to "
"%s mode %d flags 0x%x w %u h %u",
gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
(&viewconvert->in_info)), in_mode, in_flags,
viewconvert->in_info.width, viewconvert->in_info.height,
gst_video_format_to_string (GST_VIDEO_INFO_FORMAT
(&viewconvert->out_info)), out_mode, out_flags,
viewconvert->out_info.width, viewconvert->out_info.height);
if (!gl->CreateProgramObject && !gl->CreateProgram) {
gst_gl_context_set_error (viewconvert->context,
"Cannot perform multiview conversion without OpenGL shaders");
goto error;
}
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED
|| out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (!gl->DrawBuffers) {
GST_ERROR_OBJECT (viewconvert,
"Separate texture output mode requested however the current "
"OpenGL API does not support drawing to multiple buffers");
goto error;
}
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
l_index = 0;
r_index = 1;
} else {
GST_LOG_OBJECT (viewconvert, "Switching left/right views");
/* Swap the views */
l_index = 1;
r_index = 0;
}
if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
} else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
/* Side-by-side input */
offsets[r_index][0] += 0.5 * tex_scale[r_index][0];
tex_scale[0][0] *= 0.5f; /* Half horizontal scale */
tex_scale[1][0] *= 0.5f;
} else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */
offsets[r_index][1] += 0.5 * tex_scale[r_index][1];
tex_scale[0][1] *= 0.5f; /* Half vertical scale */
tex_scale[1][1] *= 0.5f;
}
/* Flipped is vertical, flopped is horizontal.
* Adjust and offset per-view scaling. This needs to be done
* after the input scaling already splits the views, before
* adding any output scaling. */
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
offsets[l_index][1] += tex_scale[l_index][1];
tex_scale[l_index][1] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
offsets[l_index][0] += tex_scale[l_index][0];
tex_scale[l_index][0] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
offsets[r_index][1] += tex_scale[r_index][1];
tex_scale[r_index][1] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
offsets[r_index][0] += tex_scale[r_index][0];
tex_scale[r_index][0] *= -1.0;
}
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
/* Side-by-Side */
offsets[1][0] -= tex_scale[1][0];
tex_scale[0][0] *= 2.0f;
tex_scale[1][0] *= 2.0f;
} else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
offsets[1][1] -= tex_scale[1][1];
tex_scale[0][1] *= 2.0f;
tex_scale[1][1] *= 2.0f;
}
GST_DEBUG_OBJECT (viewconvert,
"Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
tex_scale[0][0], tex_scale[0][1],
tex_scale[1][0], tex_scale[1][1],
offsets[0][0], offsets[0][1], offsets[1][0], offsets[1][1]);
fragment_source_str = _get_shader_string (viewconvert, in_mode, out_mode);
// g_print ("%s\n", fragment_source_str);
res = gst_gl_context_gen_shader (viewconvert->context,
gst_gl_shader_string_vertex_mat4_texture_transform, fragment_source_str,
&viewconvert->shader);
g_free (fragment_source_str);
if (!res)
goto error;
viewconvert->priv->attr_position =
gst_gl_shader_get_attribute_location (viewconvert->shader, "a_position");
viewconvert->priv->attr_texture =
gst_gl_shader_get_attribute_location (viewconvert->shader, "a_texcoord");
gst_gl_shader_use (viewconvert->shader);
gst_gl_shader_set_uniform_2fv (viewconvert->shader, "tex_scale",
2, tex_scale[0]);
gst_gl_shader_set_uniform_2fv (viewconvert->shader, "offsets", 2, offsets[0]);
gst_gl_shader_set_uniform_1f (viewconvert->shader, "width",
GST_VIDEO_INFO_WIDTH (&viewconvert->out_info));
gst_gl_shader_set_uniform_1f (viewconvert->shader, "height",
GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info));
gst_gl_shader_set_uniform_matrix_3fv (viewconvert->shader, "downmix",
2, FALSE, &downmix_matrices[viewconvert->downmix_mode][0][0]);
gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader, "u_transformation",
1, FALSE, identity_matrix);
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", l_index);
gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", r_index);
} else {
gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_l", 0);
gst_gl_shader_set_uniform_1i (viewconvert->shader, "tex_r", 0);
}
gst_gl_context_clear_shader (viewconvert->context);
if (!_init_view_convert_fbo (viewconvert)) {
goto error;
}
if (!viewconvert->priv->vertex_buffer) {
if (gl->GenVertexArrays) {
gl->GenVertexArrays (1, &viewconvert->priv->vao);
gl->BindVertexArray (viewconvert->priv->vao);
}
gl->GenBuffers (1, &viewconvert->priv->vertex_buffer);
gl->BindBuffer (GL_ARRAY_BUFFER, viewconvert->priv->vertex_buffer);
gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), vertices,
GL_STATIC_DRAW);
gl->GenBuffers (1, &viewconvert->priv->vbo_indices);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, viewconvert->priv->vbo_indices);
gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices,
GL_STATIC_DRAW);
if (gl->GenVertexArrays) {
_bind_buffer (viewconvert);
gl->BindVertexArray (0);
}
gl->BindBuffer (GL_ARRAY_BUFFER, 0);
gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0);
}
viewconvert->initted = TRUE;
return TRUE;
error:
return FALSE;
}
static gboolean
_do_view_convert_draw (GstGLContext * context, GstGLViewConvert * viewconvert)
{
GstGLViewConvertPrivate *priv = viewconvert->priv;
GstGLFuncs *gl;
guint out_width, out_height;
gint out_views, i;
GLint viewport_dim[4] = { 0 };
GLenum multipleRT[] = {
GL_COLOR_ATTACHMENT0,
GL_COLOR_ATTACHMENT1,
GL_COLOR_ATTACHMENT2
};
GstVideoMultiviewMode in_mode = priv->input_mode;
GstVideoMultiviewMode out_mode = priv->output_mode;
guint from_gl_target =
gst_gl_texture_target_to_gl (viewconvert->from_texture_target);
GstVideoAffineTransformationMeta *af_meta;
gl = context->gl_vtable;
out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
gl->BindFramebuffer (GL_FRAMEBUFFER, viewconvert->fbo);
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
out_views = viewconvert->out_info.views;
} else {
out_views = 1;
}
/* FIXME: the auxillary buffer could have a different transform matrix */
af_meta = gst_buffer_get_video_affine_transformation_meta (priv->primary_in);
if (af_meta)
gst_gl_shader_set_uniform_matrix_4fv (viewconvert->shader,
"u_transformation", 1, FALSE, af_meta->matrix);
/* attach the texture to the FBO to renderer to */
for (i = 0; i < out_views; i++) {
guint gl_target =
gst_gl_texture_target_to_gl (viewconvert->to_texture_target);
/* needed? */
gl->BindTexture (gl_target, priv->out_tex[i]->tex_id);
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0 + i,
gl_target, priv->out_tex[i]->tex_id, 0);
}
if (gl->DrawBuffers)
gl->DrawBuffers (out_views, multipleRT);
else if (gl->DrawBuffer)
gl->DrawBuffer (GL_COLOR_ATTACHMENT0);
gl->GetIntegerv (GL_VIEWPORT, viewport_dim);
gl->Viewport (0, 0, out_width, out_height);
gst_gl_shader_use (viewconvert->shader);
if (gl->BindVertexArray)
gl->BindVertexArray (priv->vao);
else
_bind_buffer (viewconvert);
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (priv->in_tex[1] == NULL) {
GST_ERROR_OBJECT (viewconvert,
"No 2nd view available during conversion!");
return FALSE;
}
gl->ActiveTexture (GL_TEXTURE1);
gl->BindTexture (from_gl_target, priv->in_tex[1]->tex_id);
}
gl->ActiveTexture (GL_TEXTURE0);
gl->BindTexture (from_gl_target, priv->in_tex[0]->tex_id);
gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, NULL);
if (gl->BindVertexArray)
gl->BindVertexArray (0);
else
_unbind_buffer (viewconvert);
if (gl->DrawBuffer)
gl->DrawBuffer (GL_NONE);
/* we are done with the shader */
gst_gl_context_clear_shader (context);
gl->Viewport (viewport_dim[0], viewport_dim[1], viewport_dim[2],
viewport_dim[3]);
gst_gl_context_check_framebuffer_status (context);
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
return TRUE;
}
static gboolean
_gen_buffer (GstGLViewConvert * viewconvert, GstBuffer ** target)
{
GstGLVideoAllocationParams *params;
GstGLMemoryAllocator *mem_allocator;
GstAllocator *allocator;
*target = gst_buffer_new ();
allocator =
GST_ALLOCATOR (gst_gl_memory_allocator_get_default
(viewconvert->context));
mem_allocator = GST_GL_MEMORY_ALLOCATOR (allocator);
params = gst_gl_video_allocation_params_new (viewconvert->context, NULL,
&viewconvert->out_info, 0, NULL, viewconvert->to_texture_target);
if (!gst_gl_memory_setup_buffer (mem_allocator, *target, params)) {
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
gst_object_unref (allocator);
return FALSE;
}
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
gst_object_unref (allocator);
gst_buffer_add_video_meta_full (*target, 0,
GST_VIDEO_INFO_FORMAT (&viewconvert->out_info),
GST_VIDEO_INFO_WIDTH (&viewconvert->out_info),
GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info),
GST_VIDEO_INFO_N_PLANES (&viewconvert->out_info),
viewconvert->out_info.offset, viewconvert->out_info.stride);
return TRUE;
}
static void
_do_view_convert (GstGLContext * context, GstGLViewConvert * viewconvert)
{
GstGLViewConvertPrivate *priv = viewconvert->priv;
guint in_width, in_height, out_width, out_height;
GstMapInfo out_info[GST_VIDEO_MAX_PLANES], in_info[GST_VIDEO_MAX_PLANES];
GstGLMemory *dest_tex[GST_VIDEO_MAX_PLANES];
gboolean res = TRUE;
gint i = 0, j = 0;
gint in_views, out_views;
GstVideoMultiviewMode in_mode;
GstVideoMultiviewMode out_mode;
out_width = GST_VIDEO_INFO_WIDTH (&viewconvert->out_info);
out_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->out_info);
in_width = GST_VIDEO_INFO_WIDTH (&viewconvert->in_info);
in_height = GST_VIDEO_INFO_HEIGHT (&viewconvert->in_info);
g_return_if_fail (priv->primary_out == NULL);
g_return_if_fail (priv->auxilliary_out == NULL);
in_mode = priv->input_mode;
out_mode = priv->output_mode;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
in_views = viewconvert->in_info.views;
else
in_views = 1;
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
out_views = viewconvert->out_info.views;
else
out_views = 1;
if (!_init_view_convert (viewconvert)) {
priv->result = FALSE;
return;
}
if (!_gen_buffer (viewconvert, &priv->primary_out)) {
GST_ERROR_OBJECT (viewconvert,
"Failed to setup memory for primary output buffer");
priv->result = FALSE;
return;
}
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (!_gen_buffer (viewconvert, &priv->auxilliary_out)) {
GST_ERROR_OBJECT (viewconvert,
"Failed to setup memory for second view output buffer");
priv->result = FALSE;
return;
}
}
for (i = 0; i < in_views; i++) {
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME && i > 0) {
priv->in_tex[i] =
(GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_in, 0);
} else {
priv->in_tex[i] =
(GstGLMemory *) gst_buffer_peek_memory (priv->primary_in, i);
}
if (!gst_is_gl_memory ((GstMemory *) priv->in_tex[i])) {
GST_ERROR_OBJECT (viewconvert, "input must be GstGLMemory");
res = FALSE;
goto out;
}
if (!gst_memory_map ((GstMemory *) priv->in_tex[i],
&in_info[i], GST_MAP_READ | GST_MAP_GL)) {
GST_ERROR_OBJECT (viewconvert, "failed to map input memory %p",
priv->in_tex[i]);
res = FALSE;
goto out;
}
}
for (j = 0; j < out_views; j++) {
GstGLMemory *out_tex;
guint width, height;
GstVideoInfo temp_info;
if (j > 0 && out_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
dest_tex[j] = out_tex =
(GstGLMemory *) gst_buffer_peek_memory (priv->auxilliary_out, 0);
} else {
dest_tex[j] = out_tex =
(GstGLMemory *) gst_buffer_peek_memory (priv->primary_out, j);
}
if (!gst_is_gl_memory ((GstMemory *) out_tex)) {
GST_ERROR_OBJECT (viewconvert, "output must be GstGLMemory");
res = FALSE;
goto out;
}
width = gst_gl_memory_get_texture_width (out_tex);
height = gst_gl_memory_get_texture_height (out_tex);
gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, width,
height);
if (out_tex->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE
|| out_tex->tex_type == GST_VIDEO_GL_TEXTURE_TYPE_LUMINANCE_ALPHA
|| out_width != width || out_height != height) {
/* Luminance formats are not color renderable */
/* renderering to a framebuffer only renders the intersection of all
* the attachments i.e. the smallest attachment size */
if (!priv->out_tex[j]) {
GstGLVideoAllocationParams *params;
GstGLBaseMemoryAllocator *base_mem_allocator;
GstAllocator *allocator;
GstVideoInfo temp_info;
gst_video_info_set_format (&temp_info, GST_VIDEO_FORMAT_RGBA, out_width,
out_height);
allocator =
GST_ALLOCATOR (gst_gl_memory_allocator_get_default (context));
base_mem_allocator = GST_GL_BASE_MEMORY_ALLOCATOR (allocator);
params = gst_gl_video_allocation_params_new (context, NULL, &temp_info,
0, NULL, viewconvert->to_texture_target);
priv->out_tex[j] =
(GstGLMemory *) gst_gl_base_memory_alloc (base_mem_allocator,
(GstGLAllocationParams *) params);
gst_gl_allocation_params_free ((GstGLAllocationParams *) params);
gst_object_unref (allocator);
}
} else {
priv->out_tex[j] = out_tex;
}
if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
&out_info[j], GST_MAP_WRITE | GST_MAP_GL)) {
GST_ERROR_OBJECT (viewconvert, "failed to map output memory %p",
priv->out_tex[i]);
res = FALSE;
goto out;
}
}
priv->n_out_tex = out_views;
GST_LOG_OBJECT (viewconvert, "multiview splitting to textures:%p,%p,%p,%p "
"dimensions:%ux%u, from textures:%p,%p,%p,%p dimensions:%ux%u",
priv->out_tex[0], priv->out_tex[1],
priv->out_tex[2], priv->out_tex[3],
out_width, out_height, priv->in_tex[0],
priv->in_tex[1], priv->in_tex[2], priv->in_tex[3], in_width, in_height);
if (!_do_view_convert_draw (context, viewconvert))
res = FALSE;
out:
for (j--; j >= 0; j--) {
GstGLMemory *out_tex;
guint width, height;
out_tex = dest_tex[j];
width = gst_gl_memory_get_texture_width (out_tex);
height = gst_gl_memory_get_texture_height (out_tex);
gst_memory_unmap ((GstMemory *) priv->out_tex[j], &out_info[j]);
if (out_tex != priv->out_tex[j]) {
GstMapInfo to_info, from_info;
if (!gst_memory_map ((GstMemory *) priv->out_tex[j],
&from_info, GST_MAP_READ | GST_MAP_GL)) {
gst_gl_context_set_error (viewconvert->context,
"Failed to map " "intermediate memory");
res = FALSE;
continue;
}
if (!gst_memory_map ((GstMemory *) out_tex, &to_info,
GST_MAP_WRITE | GST_MAP_GL)) {
gst_gl_context_set_error (viewconvert->context, "Failed to map "
"intermediate memory");
res = FALSE;
continue;
}
gst_gl_memory_copy_into (priv->out_tex[j], out_tex->tex_id,
viewconvert->to_texture_target, out_tex->tex_type, width, height);
gst_memory_unmap ((GstMemory *) out_tex, &to_info);
}
priv->out_tex[j] = NULL;
}
for (i--; i >= 0; i--) {
gst_memory_unmap ((GstMemory *) priv->in_tex[i], &in_info[i]);
}
if (!res) {
gst_buffer_replace (&priv->primary_out, NULL);
gst_buffer_replace (&priv->auxilliary_out, NULL);
}
priv->result = res;
return;
}
GstFlowReturn
gst_gl_view_convert_submit_input_buffer (GstGLViewConvert * viewconvert,
gboolean is_discont, GstBuffer * input)
{
GstFlowReturn ret = GST_FLOW_OK;
GstVideoMultiviewMode mode;
GstBuffer **target;
if (is_discont) {
gst_buffer_replace (&viewconvert->priv->primary_in, NULL);
gst_buffer_replace (&viewconvert->priv->auxilliary_in, NULL);
}
mode = viewconvert->input_mode_override;
if (mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
target = &viewconvert->priv->primary_in;
/* For frame-by-frame mode, we need to collect the 2nd eye into
* our auxilliary buffer */
if (mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
if (!GST_BUFFER_FLAG_IS_SET (input, GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE))
target = &viewconvert->priv->auxilliary_in;
}
if (*target)
gst_buffer_unref (*target);
*target = input;
return ret;
}
GstFlowReturn
gst_gl_view_convert_get_output (GstGLViewConvert * viewconvert,
GstBuffer ** outbuf_ptr)
{
GstGLViewConvertPrivate *priv = viewconvert->priv;
GstBuffer *outbuf = NULL;
GstFlowReturn ret = GST_FLOW_OK;
GstVideoMultiviewMode in_mode, out_mode;
GstVideoMultiviewFlags in_flags, out_flags;
g_return_val_if_fail (GST_IS_GL_VIEW_CONVERT (viewconvert), GST_FLOW_ERROR);
g_return_val_if_fail (GST_IS_GL_CONTEXT (viewconvert->context),
GST_FLOW_ERROR);
GST_OBJECT_LOCK (viewconvert);
/* See if a buffer is available already */
if (priv->primary_out) {
outbuf = viewconvert->priv->primary_out;
priv->primary_out = NULL;
goto done;
}
if (viewconvert->priv->auxilliary_out) {
outbuf = priv->auxilliary_out;
priv->auxilliary_out = NULL;
goto done;
}
/* Check prereqs before processing a new input buffer */
if (priv->primary_in == NULL)
goto done;
in_mode = viewconvert->input_mode_override;
in_flags = viewconvert->input_flags_override;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->in_info);
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->in_info);
}
/* Configured output mode already takes any override
* into account */
out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&viewconvert->out_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&viewconvert->out_info);
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) {
/* For frame-by-frame, we need 2 input buffers */
if (priv->auxilliary_in == NULL) {
GST_LOG_OBJECT (viewconvert,
"Can't generate output yet - frame-by-frame mode");
goto done;
}
}
/* Store the current conversion in the priv vars */
priv->input_mode = in_mode;
priv->input_flags = in_flags;
priv->output_mode = out_mode;
priv->output_flags = out_flags;
if (priv->input_mode == priv->output_mode &&
priv->input_flags == priv->output_flags &&
viewconvert->in_info.width == viewconvert->out_info.width &&
viewconvert->in_info.height == viewconvert->out_info.height) {
/* passthrough - just pass input buffers */
outbuf = gst_buffer_ref (priv->primary_in);
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
priv->auxilliary_out = gst_buffer_ref (priv->auxilliary_in);
goto done_clear_input;
}
/* Generate new output buffer(s) */
gst_gl_context_thread_add (viewconvert->context,
(GstGLContextThreadFunc) _do_view_convert, viewconvert);
if (!priv->result) {
if (priv->primary_out)
gst_object_unref (priv->primary_out);
if (priv->auxilliary_out)
gst_object_unref (priv->auxilliary_out);
priv->primary_out = NULL;
priv->auxilliary_out = NULL;
ret = GST_FLOW_ERROR;
goto done_clear_input;
}
outbuf = priv->primary_out;
if (outbuf) {
gst_buffer_copy_into (outbuf, priv->primary_in,
GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1);
GST_BUFFER_FLAG_SET (outbuf,
GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE |
GST_VIDEO_BUFFER_FLAG_MULTIPLE_VIEW);
}
if (priv->auxilliary_out) {
gst_buffer_copy_into (priv->auxilliary_out,
priv->primary_out, GST_BUFFER_COPY_FLAGS, 0, -1);
GST_BUFFER_FLAG_UNSET (priv->auxilliary_out,
GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE);
}
priv->primary_out = NULL;
done_clear_input:
/* Invalidate input buffers now they've been used */
gst_buffer_replace (&priv->primary_in, NULL);
gst_buffer_replace (&priv->auxilliary_in, NULL);
done:
GST_OBJECT_UNLOCK (viewconvert);
*outbuf_ptr = outbuf;
return ret;
}
Reference in a new issue View git blame Copy permalink