mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-12-15 04:46:32 +00:00
Code Issues 1 Projects Releases Wiki Activity
gstreamer/subprojects/gst-plugins-bad/ext/vulkan/vkviewconvert.c
Víctor Manuel Jáquez Leal e177080bec vulkan: number of memories in buffer rather than number of planes 
New vulkan formats don't match the number of planes with the number of memories
attached to the buffer. This patch changes the pattern of using planes for
traverse the memories with the number of attached memories.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4351>
2023-05-19 04:26:29 +00:00

2028 lines
72 KiB
C
Raw Blame History

/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-vulkanimageidentity
* @title: vulkanimgeidentity
*
* vulkanimageidentity produces a vulkan image that is a copy of the input image.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <string.h>
#include "gstvulkanelements.h"
#include "vkviewconvert.h"
#include "shaders/identity.vert.h"
#include "shaders/view_convert.frag.h"
#include "gstvulkan-plugins-enumtypes.h"
GST_DEBUG_CATEGORY (gst_debug_vulkan_view_convert);
#define GST_CAT_DEFAULT gst_debug_vulkan_view_convert
/* *INDENT-OFF* */
/* These match the order and number of DOWNMIX_ANAGLYPH_* modes */
static float downmix_matrices[][2][12] = {
{ /* Green-Magenta Dubois */
{-0.062f, 0.284f, -0.015f, 0.0, -0.158f, 0.668f, -0.027f, 0.0, -0.039f, 0.143f, 0.021f, 0.0},
{0.529f, -0.016f, 0.009f, 0.0, 0.705f, -0.015f, 0.075f, 0.0, 0.024f, -0.065f, 0.937f, 0.0}
},
{ /* Red-Cyan Dubois */
/* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */
{0.437f, -0.062f, -0.048f, 0.0, 0.449f, -0.062f, -0.050f, 0.0, 0.164f, -0.024f, -0.017f},
{-0.011f, 0.377f, -0.026f, 0.0, -0.032f, 0.761f, -0.093f, 0.0, -0.007f, 0.009f, 1.234f}
},
{ /* Amber-blue Dubois */
{1.062f, -0.026f, -0.038f, 0.0, -0.205f, 0.908f, -0.173f, 0.0, 0.299f, 0.068f, 0.022f},
{-0.016f, 0.006f, 0.094f, 0.0, -0.123f, 0.062f, 0.185f, 0.0, -0.017f, -0.017f, 0.911f}
}
};
/* *INDENT-ON* */
struct ViewUpdate
{
int in_reorder_idx[4];
int out_reorder_idx[4];
float tex_offset[2][2];
float tex_scale[2][2];
int tex_size[2];
int output_type;
int _padding;
float downmix[2][12];
};
static void
get_rgb_format_swizzle_order (GstVideoFormat format,
gint swizzle[GST_VIDEO_MAX_COMPONENTS])
{
const GstVideoFormatInfo *finfo = gst_video_format_get_info (format);
int c_i = 0, i;
g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB
|| format == GST_VIDEO_FORMAT_AYUV);
for (i = 0; i < finfo->n_components; i++) {
swizzle[c_i++] = finfo->poffset[i];
}
/* special case spaced RGB formats as the space does not contain a poffset
* value and we need all four components to be valid in order to swizzle
* correctly */
if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) {
swizzle[c_i++] = 0;
} else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx) {
swizzle[c_i++] = 3;
} else {
for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) {
swizzle[c_i++] = -1;
}
}
}
static void
get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle,
guint swizzle_count, guint offset)
{
const GstVulkanFormatInfo *finfo = gst_vulkan_format_get_info (format);
int i;
g_return_if_fail (finfo->flags & GST_VULKAN_FORMAT_FLAG_RGB);
g_return_if_fail (finfo->n_components <= swizzle_count);
for (i = 0; i < finfo->n_components; i++) {
swizzle[i] = offset + finfo->poffset[i];
}
for (i = finfo->n_components; i < swizzle_count; i++) {
swizzle[i] = -1;
}
}
/* given a swizzle index, produce an index such that:
*
* swizzle[idx[i]] == identity[i] where:
* - swizzle is the original swizzle
* - idx is the result
* - identity = {0, 1, 2,...}
* - unset fields are marked by -1
*/
static void
swizzle_identity_order (gint * swizzle, gint * idx)
{
int i;
for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
idx[i] = -1;
}
for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) {
if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) {
idx[swizzle[i]] = i;
}
}
}
static void
video_format_to_reorder (GstVideoFormat v_format, gint * reorder,
gboolean input)
{
switch (v_format) {
case GST_VIDEO_FORMAT_RGBA:
case GST_VIDEO_FORMAT_RGBx:
case GST_VIDEO_FORMAT_BGRA:
case GST_VIDEO_FORMAT_BGRx:
case GST_VIDEO_FORMAT_ARGB:
case GST_VIDEO_FORMAT_xRGB:
case GST_VIDEO_FORMAT_ABGR:
case GST_VIDEO_FORMAT_xBGR:
case GST_VIDEO_FORMAT_AYUV:
get_rgb_format_swizzle_order (v_format, reorder);
break;
case GST_VIDEO_FORMAT_UYVY:
reorder[0] = 1;
reorder[1] = 0;
reorder[2] = input ? 3 : 2;
reorder[3] = 0;
break;
case GST_VIDEO_FORMAT_YUY2:
reorder[0] = 0;
reorder[1] = 1;
reorder[2] = 0;
reorder[3] = input ? 3 : 2;
break;
case GST_VIDEO_FORMAT_NV12:
reorder[0] = 0;
reorder[1] = 1;
reorder[2] = 2;
reorder[3] = 0;
break;
default:
g_assert_not_reached ();
break;
}
GST_TRACE ("swizzle: %u, %u, %u, %u", reorder[0], reorder[1], reorder[2],
reorder[3]);
}
static guint
finfo_get_plane_n_components (const GstVideoFormatInfo * finfo, guint plane)
{
guint n_components = 0, i;
switch (finfo->format) {
case GST_VIDEO_FORMAT_RGBx:
case GST_VIDEO_FORMAT_xRGB:
case GST_VIDEO_FORMAT_BGRx:
case GST_VIDEO_FORMAT_xBGR:
/* fixup spaced RGB formats as we treat the space as a normal alpha
* component */
return plane == 0 ? 4 : 0;
default:
break;
}
for (i = 0; i < finfo->n_components; i++) {
if (finfo->plane[i] == plane)
n_components++;
}
return n_components;
}
static void
get_vulkan_format_swizzle_order (GstVideoFormat v_format,
VkFormat vk_format[GST_VIDEO_MAX_PLANES],
gint swizzle[GST_VIDEO_MAX_COMPONENTS])
{
const GstVideoFormatInfo *finfo;
int i, prev_in_i = 0;
finfo = gst_video_format_get_info (v_format);
for (i = 0; i < finfo->n_planes; i++) {
guint plane_components = finfo_get_plane_n_components (finfo, i);
get_vulkan_rgb_format_swizzle_order (vk_format[i],
&swizzle[prev_in_i], plane_components, prev_in_i);
prev_in_i += plane_components;
}
if (v_format == GST_VIDEO_FORMAT_YUY2 || v_format == GST_VIDEO_FORMAT_UYVY) {
/* Fixup these packed YUV formats as we use a two component format for
* a 4-component pixel and access two samples in the shader */
g_assert (swizzle[0] == 0);
g_assert (swizzle[1] == 1);
swizzle[2] = 2;
swizzle[3] = 3;
}
GST_TRACE ("%s: %i, %i, %i, %i", finfo->name, swizzle[0], swizzle[1],
swizzle[2], swizzle[3]);
}
static void
calculate_reorder_indexes (GstVideoFormat in_format,
GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS],
GstVideoFormat out_format,
GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS],
int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS])
{
const GstVideoFormatInfo *in_finfo, *out_finfo;
VkFormat in_vk_formats[GST_VIDEO_MAX_COMPONENTS];
VkFormat out_vk_formats[GST_VIDEO_MAX_COMPONENTS];
int in_vk_order[GST_VIDEO_MAX_COMPONENTS],
in_reorder[GST_VIDEO_MAX_COMPONENTS];
int out_vk_order[GST_VIDEO_MAX_COMPONENTS],
out_reorder[GST_VIDEO_MAX_COMPONENTS];
int tmp[GST_VIDEO_MAX_PLANES];
int i;
in_finfo = gst_video_format_get_info (in_format);
out_finfo = gst_video_format_get_info (out_format);
for (i = 0; i < in_finfo->n_planes; i++)
in_vk_formats[i] = in_views[i]->image->create_info.format;
for (i = 0; i < out_finfo->n_planes; i++)
out_vk_formats[i] = out_views[i]->image->create_info.format;
get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order);
video_format_to_reorder (in_format, in_reorder, TRUE);
video_format_to_reorder (out_format, out_reorder, FALSE);
get_vulkan_format_swizzle_order (out_format, out_vk_formats, out_vk_order);
for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
tmp[i] = out_vk_order[out_reorder[i]];
/* find the identity order for RGBA->$format */
GST_TRACE ("pre-invert: %u, %u, %u, %u", tmp[0], tmp[1], tmp[2], tmp[3]);
if (out_format == GST_VIDEO_FORMAT_YUY2
|| out_format == GST_VIDEO_FORMAT_UYVY) {
for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
ret_out[i] = tmp[i];
} else {
swizzle_identity_order (tmp, ret_out);
}
for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++)
ret_in[i] = in_reorder[in_vk_order[i]];
GST_TRACE ("in reorder: %u, %u, %u, %u", ret_in[0], ret_in[1], ret_in[2],
ret_in[3]);
GST_TRACE ("out reorder: %u, %u, %u, %u", ret_out[0], ret_out[1], ret_out[2],
ret_out[3]);
}
static void
update_descriptor_set (GstVulkanViewConvert * conv,
GstVulkanImageView ** in_views, guint n_mems)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES];
VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES];
GstVideoMultiviewMode in_mode;
GstVideoMultiviewFlags in_flags, out_flags;
VkImageView views[GST_VIDEO_MAX_PLANES];
guint i = 0;
in_mode = conv->input_mode_override;
in_flags = conv->input_flags_override;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
for (i = 0; i < n_mems; i++) {
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
views[2 * i] = in_views[i]->view;
views[2 * i + 1] = in_views[i]->view;
} else {
views[2 * i] = in_views[i]->view;
views[2 * i + 1] = in_views[i]->view;
}
}
for (i = 0; i < n_mems * 2; i++) {
/* *INDENT-OFF* */
image_info[i] = (VkDescriptorImageInfo) {
.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
.imageView = views[i],
.sampler = (VkSampler) conv->quad->sampler->handle
};
g_assert (i < GST_VIDEO_MAX_PLANES);
writes[i] = (VkWriteDescriptorSet) {
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
.pNext = NULL,
.dstSet = conv->quad->descriptor_set->set,
.dstBinding = i+1,
.dstArrayElement = 0,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.descriptorCount = 1,
.pImageInfo = &image_info[i]
};
/* *INDENT-ON* */
}
g_assert (i <= G_N_ELEMENTS (writes));
vkUpdateDescriptorSets (vfilter->device->device, i, writes, 0, NULL);
}
static gboolean
create_uniform_buffer (GstVulkanViewConvert * conv)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
conv->uniform =
gst_vulkan_buffer_memory_alloc (vfilter->device,
sizeof (struct ViewUpdate),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
return TRUE;
}
static gboolean
update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
GstVulkanImageView ** out_views)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
GstVideoMultiviewMode in_mode, out_mode;
GstVideoMultiviewFlags in_flags, out_flags;
struct ViewUpdate data;
GstMapInfo map_info;
guint l_index, r_index;
gboolean mono_input = FALSE;
calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&vfilter->in_info),
in_views, GST_VIDEO_INFO_FORMAT (&vfilter->out_info),
out_views, data.in_reorder_idx, data.out_reorder_idx);
data.tex_scale[0][0] = data.tex_scale[0][1] = 1.;
data.tex_scale[1][0] = data.tex_scale[1][1] = 1.;
data.tex_offset[0][0] = data.tex_offset[0][1] = 0.;
data.tex_offset[1][0] = data.tex_offset[1][1] = 0.;
in_mode = conv->input_mode_override;
in_flags = conv->input_flags_override;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->in_info);
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
}
/* Configured output mode already takes any override
* into account */
out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->out_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
l_index = 0;
r_index = 1;
} else {
GST_LOG_OBJECT (conv, "Switching left/right views");
l_index = 1;
r_index = 0;
}
if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
} else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
/* Side-by-side input */
data.tex_offset[r_index][0] += 0.5 * data.tex_scale[r_index][0];
data.tex_scale[0][0] *= 0.5f; /* Half horizontal scale */
data.tex_scale[1][0] *= 0.5f;
} else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */
data.tex_offset[r_index][1] += 0.5 * data.tex_scale[r_index][1];
data.tex_scale[0][1] *= 0.5f; /* Half vertical scale */
data.tex_scale[1][1] *= 0.5f;
}
/* Flipped is vertical, flopped is horizontal.
* Adjust and offset per-view scaling. This needs to be done
* after the input scaling already splits the views, before
* adding any output scaling. */
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) {
data.tex_offset[l_index][1] += data.tex_scale[l_index][1];
data.tex_scale[l_index][1] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) {
data.tex_offset[l_index][0] += data.tex_scale[l_index][0];
data.tex_scale[l_index][0] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) {
data.tex_offset[r_index][1] += data.tex_scale[r_index][1];
data.tex_scale[r_index][1] *= -1.0;
}
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) !=
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) {
data.tex_offset[r_index][0] += data.tex_scale[r_index][0];
data.tex_scale[r_index][0] *= -1.0;
}
if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE ||
out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
/* Side-by-Side */
data.tex_offset[1][0] -= data.tex_scale[1][0];
data.tex_scale[0][0] *= 2.0f;
data.tex_scale[1][0] *= 2.0f;
} else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) {
data.tex_offset[1][1] -= data.tex_scale[1][1];
data.tex_scale[0][1] *= 2.0f;
data.tex_scale[1][1] *= 2.0f;
}
GST_DEBUG_OBJECT (conv,
"Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]",
data.tex_scale[0][0], data.tex_scale[0][1],
data.tex_scale[1][0], data.tex_scale[1][1],
data.tex_offset[0][0], data.tex_offset[0][1], data.tex_offset[1][0],
data.tex_offset[1][1]);
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_MONO ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT ||
in_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT)
mono_input = TRUE;
data.output_type = out_mode;
if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_NONE ||
data.output_type == GST_VIDEO_MULTIVIEW_MODE_MONO) {
if (mono_input)
data.output_type = GST_VIDEO_MULTIVIEW_MODE_LEFT;
else
data.output_type = GST_VIDEO_MULTIVIEW_MODE_MONO;
} else if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) {
data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
}
data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&vfilter->out_info);
data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&vfilter->out_info);
memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0],
sizeof (data.downmix[0]));
memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1],
sizeof (data.downmix[1]));
if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) {
return FALSE;
}
memcpy (map_info.data, &data, sizeof (data));
gst_memory_unmap (conv->uniform, &map_info);
return TRUE;
}
static GstMemory *
get_uniforms (GstVulkanViewConvert * conv,
GstVulkanImageView ** in_views, GstVulkanImageView ** out_views)
{
if (!conv->uniform) {
if (!create_uniform_buffer (conv))
return NULL;
if (!update_uniform (conv, in_views, out_views)) {
gst_memory_unref (conv->uniform);
conv->uniform = NULL;
return FALSE;
}
}
return gst_memory_ref (conv->uniform);
}
static void gst_vulkan_view_convert_set_property (GObject * object,
guint prop_id, const GValue * value, GParamSpec * pspec);
static void gst_vulkan_view_convert_get_property (GObject * object,
guint prop_id, GValue * value, GParamSpec * pspec);
static gboolean gst_vulkan_view_convert_start (GstBaseTransform * bt);
static gboolean gst_vulkan_view_convert_stop (GstBaseTransform * bt);
static GstCaps *gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter);
static GstCaps *gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * othercaps);
static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt,
GstBuffer * inbuf, GstBuffer * outbuf);
static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps);
static GstStaticPadTemplate gst_vulkan_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
"{ BGRA, RGBA }")));
static GstStaticPadTemplate gst_vulkan_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
"{ BGRA, RGBA }")));
enum
{
PROP_0,
PROP_INPUT_LAYOUT,
PROP_INPUT_FLAGS,
PROP_OUTPUT_LAYOUT,
PROP_OUTPUT_FLAGS,
PROP_OUTPUT_DOWNMIX_MODE
};
#define DEFAULT_DOWNMIX GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS
#define gst_vulkan_view_convert_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert,
GST_TYPE_VULKAN_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert,
"vulkanviewconvert", 0, "Vulkan View Convert"));
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanviewconvert, "vulkanviewconvert",
GST_RANK_NONE, GST_TYPE_VULKAN_VIEW_CONVERT, vulkan_element_init (plugin));
static void
gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
gobject_class->set_property = gst_vulkan_view_convert_set_property;
gobject_class->get_property = gst_vulkan_view_convert_get_property;
g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT,
g_param_spec_enum ("input-mode-override",
"Input Multiview Mode Override",
"Override any input information about multiview layout",
GST_TYPE_VIDEO_MULTIVIEW_MODE,
GST_VIDEO_MULTIVIEW_MODE_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS,
g_param_spec_flags ("input-flags-override",
"Input Multiview Flags Override",
"Override any input information about multiview layout flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT,
g_param_spec_enum ("output-mode-override",
"Output Multiview Mode Override",
"Override automatic output mode selection for multiview layout",
GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS,
g_param_spec_flags ("output-flags-override",
"Output Multiview Flags Override",
"Override automatic negotiation for output multiview layout flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE,
g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output",
"Output anaglyph type to generate when downmixing to mono",
GST_TYPE_VULKAN_STEREO_DOWNMIX, DEFAULT_DOWNMIX,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
gst_element_class_set_metadata (gstelement_class, "Vulkan View Convert",
"Filter/Video/Convert", "A Vulkan View Convert",
"Matthew Waters <matthew@centricular.com>");
gst_type_mark_as_plugin_api (GST_TYPE_VULKAN_STEREO_DOWNMIX, 0);
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_sink_template);
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_src_template);
gstbasetransform_class->start =
GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_start);
gstbasetransform_class->stop =
GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_stop);
gstbasetransform_class->transform_caps =
gst_vulkan_view_convert_transform_caps;
gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps;
gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps;
gstbasetransform_class->transform = gst_vulkan_view_convert_transform;
}
static void
gst_vulkan_view_convert_init (GstVulkanViewConvert * conv)
{
conv->downmix_mode = DEFAULT_DOWNMIX;
conv->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
conv->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
conv->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE;
conv->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
}
static void
gst_vulkan_view_convert_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
switch (prop_id) {
case PROP_INPUT_LAYOUT:
conv->input_mode_override = g_value_get_enum (value);
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
break;
case PROP_INPUT_FLAGS:
conv->input_flags_override = g_value_get_flags (value);
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
break;
case PROP_OUTPUT_LAYOUT:
conv->output_mode_override = g_value_get_enum (value);
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
break;
case PROP_OUTPUT_FLAGS:
conv->output_flags_override = g_value_get_flags (value);
gst_base_transform_reconfigure_src (GST_BASE_TRANSFORM (conv));
break;
case PROP_OUTPUT_DOWNMIX_MODE:
conv->downmix_mode = g_value_get_enum (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_vulkan_view_convert_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object);
switch (prop_id) {
case PROP_INPUT_LAYOUT:
g_value_set_enum (value, conv->input_mode_override);
break;
case PROP_INPUT_FLAGS:
g_value_set_flags (value, conv->input_flags_override);
break;
case PROP_OUTPUT_LAYOUT:
g_value_set_enum (value, conv->output_mode_override);
break;
case PROP_OUTPUT_FLAGS:
g_value_set_flags (value, conv->output_flags_override);
break;
case PROP_OUTPUT_DOWNMIX_MODE:
g_value_set_enum (value, conv->downmix_mode);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
/* Function that can halve the value
* of ints, fractions, int/fraction ranges and lists of ints/fractions */
static gboolean
_halve_value (GValue * out, const GValue * in_value)
{
/* Fundamental fixed types first */
if (G_VALUE_HOLDS_INT (in_value)) {
g_value_init (out, G_TYPE_INT);
g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1));
} else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
gint num, den;
num = gst_value_get_fraction_numerator (in_value);
den = gst_value_get_fraction_denominator (in_value);
g_value_init (out, GST_TYPE_FRACTION);
/* Don't adjust 'infinite' fractions */
if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
/* FIXME - could do better approximation when den > G_MAXINT/2? */
den = den > G_MAXINT / 2 ? G_MAXINT : den * 2;
}
gst_value_set_fraction (out, num, den);
} else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
gint range_min = gst_value_get_int_range_min (in_value);
gint range_max = gst_value_get_int_range_max (in_value);
gint range_step = gst_value_get_int_range_step (in_value);
g_value_init (out, GST_TYPE_INT_RANGE);
if (range_min != 1)
range_min = MAX (1, range_min / 2);
if (range_max != G_MAXINT)
range_max = MAX (1, range_max / 2);
gst_value_set_int_range_step (out, range_min,
range_max, MAX (1, range_step / 2));
} else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
GValue min_out = G_VALUE_INIT;
GValue max_out = G_VALUE_INIT;
const GValue *range_min = gst_value_get_fraction_range_min (in_value);
const GValue *range_max = gst_value_get_fraction_range_max (in_value);
_halve_value (&min_out, range_min);
_halve_value (&max_out, range_max);
g_value_init (out, GST_TYPE_FRACTION_RANGE);
gst_value_set_fraction_range (out, &min_out, &max_out);
g_value_unset (&min_out);
g_value_unset (&max_out);
} else if (GST_VALUE_HOLDS_LIST (in_value)) {
gint i;
g_value_init (out, GST_TYPE_LIST);
for (i = 0; i < gst_value_list_get_size (in_value); i++) {
const GValue *entry;
GValue tmp = G_VALUE_INIT;
entry = gst_value_list_get_value (in_value, i);
/* Random list values might not be the right type */
if (!_halve_value (&tmp, entry))
goto fail;
gst_value_list_append_and_take_value (out, &tmp);
}
} else {
return FALSE;
}
return TRUE;
fail:
g_value_unset (out);
return FALSE;
}
static GstStructure *
_halve_structure_field (const GstStructure * in, const gchar * field_name)
{
GstStructure *out;
const GValue *in_value = gst_structure_get_value (in, field_name);
GValue tmp = G_VALUE_INIT;
if (G_UNLIKELY (in_value == NULL))
return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
if (!_halve_value (&tmp, in_value))
return NULL;
out = gst_structure_copy (in);
gst_structure_set_value (out, field_name, &tmp);
g_value_unset (&tmp);
return out;
}
/* Function that can double the value
* of ints, fractions, int/fraction ranges and lists of ints/fractions */
static gboolean
_double_value (GValue * out, const GValue * in_value)
{
/* Fundamental fixed types first */
if (G_VALUE_HOLDS_INT (in_value)) {
gint n = g_value_get_int (in_value);
g_value_init (out, G_TYPE_INT);
if (n <= G_MAXINT / 2)
g_value_set_int (out, n * 2);
else
g_value_set_int (out, G_MAXINT);
} else if (GST_VALUE_HOLDS_FRACTION (in_value)) {
gint num, den;
num = gst_value_get_fraction_numerator (in_value);
den = gst_value_get_fraction_denominator (in_value);
g_value_init (out, GST_TYPE_FRACTION);
/* Don't adjust 'infinite' fractions */
if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) {
/* FIXME - could do better approximation when num > G_MAXINT/2? */
num = num > G_MAXINT / 2 ? G_MAXINT : num * 2;
}
gst_value_set_fraction (out, num, den);
} else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) {
gint range_min = gst_value_get_int_range_min (in_value);
gint range_max = gst_value_get_int_range_max (in_value);
gint range_step = gst_value_get_int_range_step (in_value);
if (range_min != 1) {
range_min = MIN (G_MAXINT / 2, range_min);
range_min *= 2;
}
if (range_max != G_MAXINT) {
range_max = MIN (G_MAXINT / 2, range_max);
range_max *= 2;
}
range_step = MIN (G_MAXINT / 2, range_step);
g_value_init (out, GST_TYPE_INT_RANGE);
gst_value_set_int_range_step (out, range_min, range_max, range_step);
} else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) {
GValue min_out = G_VALUE_INIT;
GValue max_out = G_VALUE_INIT;
const GValue *range_min = gst_value_get_fraction_range_min (in_value);
const GValue *range_max = gst_value_get_fraction_range_max (in_value);
_double_value (&min_out, range_min);
_double_value (&max_out, range_max);
g_value_init (out, GST_TYPE_FRACTION_RANGE);
gst_value_set_fraction_range (out, &min_out, &max_out);
g_value_unset (&min_out);
g_value_unset (&max_out);
} else if (GST_VALUE_HOLDS_LIST (in_value)) {
gint i;
g_value_init (out, GST_TYPE_LIST);
for (i = 0; i < gst_value_list_get_size (in_value); i++) {
const GValue *entry;
GValue tmp = G_VALUE_INIT;
entry = gst_value_list_get_value (in_value, i);
/* Random list values might not be the right type */
if (!_double_value (&tmp, entry))
goto fail;
gst_value_list_append_and_take_value (out, &tmp);
}
} else {
return FALSE;
}
return TRUE;
fail:
g_value_unset (out);
return FALSE;
}
static GstStructure *
_double_structure_field (const GstStructure * in, const gchar * field_name)
{
GstStructure *out;
const GValue *in_value = gst_structure_get_value (in, field_name);
GValue tmp = G_VALUE_INIT;
if (G_UNLIKELY (in_value == NULL))
return gst_structure_copy (in); /* Field doesn't exist, leave it as is */
if (!_double_value (&tmp, in_value))
return NULL;
out = gst_structure_copy (in);
gst_structure_set_value (out, field_name, &tmp);
g_value_unset (&tmp);
return out;
}
/* Return a copy of the caps with the requested field doubled in value/range */
static GstCaps *
_double_caps_field (const GstCaps * in, const gchar * field_name)
{
gint i;
GstCaps *out = gst_caps_new_empty ();
for (i = 0; i < gst_caps_get_size (in); i++) {
const GstStructure *cur = gst_caps_get_structure (in, i);
GstCapsFeatures *f = gst_caps_get_features (in, i);
GstStructure *res = _double_structure_field (cur, field_name);
out =
gst_caps_merge_structure_full (out, res,
f ? gst_caps_features_copy (f) : NULL);
}
return out;
}
/* Takes ownership of the input caps */
static GstCaps *
_expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect)
{
guint mview_flags, mview_flags_mask;
GstCaps *out;
GstStructure *tmp;
out = gst_caps_new_empty ();
while (gst_caps_get_size (in) > 0) {
GstStructure *s;
GstCapsFeatures *features;
features = gst_caps_get_features (in, 0);
if (features)
features = gst_caps_features_copy (features);
s = gst_caps_steal_structure (in, 0);
if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
gst_caps_append_structure_full (out, s, features);
continue;
}
/* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */
if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) {
gst_caps_append_structure_full (out, s, features);
continue;
}
if (!gst_structure_has_field (s, "pixel-aspect-ratio")) {
/* No par field, dont-care the half-aspect flag */
gst_structure_set (s, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_caps_append_structure_full (out, s, features);
continue;
}
/* Halve or double PAR base on inputs input specified. */
/* Append a copy with the half-aspect flag as-is */
tmp = gst_structure_copy (s);
out = gst_caps_merge_structure_full (out, tmp,
features ? gst_caps_features_copy (features) : NULL);
/* and then a copy inverted */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
/* Input is half-aspect. Double/halve the PAR, clear the flag */
if (vertical_half_aspect)
tmp = _halve_structure_field (s, "pixel-aspect-ratio");
else
tmp = _double_structure_field (s, "pixel-aspect-ratio");
/* Clear the flag */
gst_structure_set (tmp, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
} else {
if (vertical_half_aspect)
tmp = _double_structure_field (s, "pixel-aspect-ratio");
else
tmp = _halve_structure_field (s, "pixel-aspect-ratio");
/* Set the flag */
gst_structure_set (tmp, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
out = gst_caps_merge_structure_full (out, tmp,
features ? gst_caps_features_copy (features) : NULL);
gst_structure_free (s);
if (features)
gst_caps_features_free (features);
}
gst_caps_unref (in);
return out;
}
/* If input supports top-bottom or row-interleaved, we may halve height to mono frames.
* If input supports left-right, checkerboard, quincunx or column-interleaved,
* we may halve width to mono frames.
* For output of top-bottom or row-interleaved, we may double the mono height
* For output of left-right, checkerboard, quincunx or column-interleaved,
* we may double the mono width.
* In all cases, if input has half-aspect and output does not, we may double the PAR
* And if input does *not* have half-aspect flag and output does not, we may halve the PAR
*/
static GstCaps *
_expand_structure (GstVulkanViewConvert * viewconvert,
GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features)
{
GstCaps *expanded_caps, *tmp;
GstCaps *mono_caps;
const gchar *default_mview_mode_str = NULL;
guint mview_flags, mview_flags_mask;
const GValue *in_modes;
gint i;
/* Empty caps to accumulate into */
expanded_caps = gst_caps_new_empty ();
/* First, set defaults if multiview flags are missing */
default_mview_mode_str =
gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO);
mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE;
mview_flags_mask = GST_FLAG_SET_MASK_EXACT;
if (!gst_structure_has_field (structure, "multiview-mode")) {
gst_structure_set (structure,
"multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL);
}
if (!gst_structure_has_field (structure, "multiview-flags")) {
gst_structure_set (structure,
"multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags,
mview_flags_mask, NULL);
} else {
gst_structure_get_flagset (structure, "multiview-flags",
&mview_flags, &mview_flags_mask);
}
in_modes = gst_structure_get_value (structure, "multiview-mode");
mono_caps = gst_caps_new_empty ();
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_mono_modes ())) {
GstStructure *new_struct = gst_structure_copy (structure);
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Half-aspect makes no sense for mono or unpacked, get rid of it */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
gst_structure_set (new_struct, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
gst_caps_append_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_unpacked_modes ())) {
GstStructure *new_struct = gst_structure_copy (structure);
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Half-aspect makes no sense for mono or unpacked, get rid of it */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
gst_structure_set (new_struct, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
}
gst_caps_append_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_height_modes ())) {
/* Append mono formats with height halved */
GstStructure *new_struct = _halve_structure_field (structure, "height");
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Normalise the half-aspect flag away */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
GstStructure *s =
_halve_structure_field (new_struct, "pixel-aspect-ratio");
gst_structure_set (structure, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_structure_free (new_struct);
new_struct = s;
}
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_width_modes ())) {
/* Append mono formats with width halved */
GstStructure *new_struct = _halve_structure_field (structure, "width");
gst_structure_set_value (new_struct, "multiview-mode",
gst_video_multiview_get_mono_modes ());
/* Normalise the half-aspect flag away */
if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) {
GstStructure *s =
_double_structure_field (new_struct, "pixel-aspect-ratio");
gst_structure_set (structure, "multiview-flags",
GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT,
mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL);
gst_structure_free (new_struct);
new_struct = s;
}
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct,
features ? gst_caps_features_copy (features) : NULL);
}
if (gst_value_intersect (NULL, in_modes,
gst_video_multiview_get_doubled_size_modes ())) {
/* Append checkerboard/doubled size formats with width & height halved */
GstStructure *new_struct_w = _halve_structure_field (structure, "width");
GstStructure *new_struct_wh =
_halve_structure_field (new_struct_w, "height");
gst_structure_free (new_struct_w);
gst_structure_set_value (new_struct_wh, "multiview-mode",
gst_video_multiview_get_mono_modes ());
mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh,
features ? gst_caps_features_copy (features) : NULL);
}
/* Everything is normalised now, unset the flags we can change */
/* Remove the views field, as these are all 'mono' modes
* Need to do this before we expand caps back out to frame packed modes */
for (i = 0; i < gst_caps_get_size (mono_caps); i++) {
GstStructure *s = gst_caps_get_structure (mono_caps, i);
gst_structure_remove_fields (s, "views", NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* Preserve only the half-aspect and mixed-mono flags, for now.
* The rest we can change */
mview_flags_mask &=
(GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT |
GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO);
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
GST_TRACE_OBJECT (viewconvert,
"Collected single-view caps %" GST_PTR_FORMAT, mono_caps);
/* Put unpacked and mono modes first. We don't care about flags. Clear them */
tmp = gst_caps_copy (mono_caps);
for (i = 0; i < gst_caps_get_size (tmp); i++) {
GstStructure *s = gst_caps_get_structure (tmp, i);
gst_structure_remove_fields (s, "views", NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */
mview_flags_mask = 0;
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Unpacked output modes have 2 views, for now */
tmp = gst_caps_copy (mono_caps);
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_unpacked_modes ());
for (i = 0; i < gst_caps_get_size (tmp); i++) {
GstStructure *s = gst_caps_get_structure (tmp, i);
gst_structure_set (s, "views", G_TYPE_INT, 2, NULL);
if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags,
&mview_flags_mask)) {
/* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */
mview_flags_mask = 0;
gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET,
mview_flags, mview_flags_mask, NULL);
}
}
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double height output modes */
tmp = _double_caps_field (mono_caps, "height");
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_height_modes ());
tmp = _expand_par_for_half_aspect (tmp, TRUE);
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double width output modes */
tmp = _double_caps_field (mono_caps, "width");
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_width_modes ());
tmp = _expand_par_for_half_aspect (tmp, FALSE);
expanded_caps = gst_caps_merge (expanded_caps, tmp);
/* Double size output modes */
{
GstCaps *tmp_w = _double_caps_field (mono_caps, "width");
tmp = _double_caps_field (tmp_w, "height");
gst_caps_unref (tmp_w);
gst_caps_set_value (tmp, "multiview-mode",
gst_video_multiview_get_doubled_size_modes ());
expanded_caps = gst_caps_merge (expanded_caps, tmp);
}
/* We're done with the mono caps now */
gst_caps_unref (mono_caps);
GST_TRACE_OBJECT (viewconvert,
"expanded transform caps now %" GST_PTR_FORMAT, expanded_caps);
if (gst_caps_is_empty (expanded_caps)) {
gst_caps_unref (expanded_caps);
return out_caps;
}
/* Really, we can rescale - so at this point we can append full-range
* height/width/PAR as an unpreferred final option. */
/* tmp = gst_caps_copy (expanded_caps);
gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT,
"height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL);
*/
out_caps = gst_caps_merge (out_caps, expanded_caps);
/* out_caps = gst_caps_merge (out_caps, tmp);*/
return out_caps;
}
static GstCaps *
_intersect_with_mview_mode (GstCaps * caps,
GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags)
{
GstCaps *filter, *result;
const gchar *caps_str;
caps_str = gst_video_multiview_mode_to_caps_string (mode);
filter = gst_caps_new_simple ("video/x-raw",
"multiview-mode", G_TYPE_STRING,
caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
GST_FLAG_SET_MASK_EXACT, NULL);
if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED ||
mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME)
gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL);
gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
" with caps %" GST_PTR_FORMAT, caps, filter);
result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (filter);
return result;
}
static GstCaps *
_intersect_with_mview_modes (GstCaps * caps, const GValue * modes)
{
GstCaps *filter, *result;
filter = gst_caps_new_empty_simple ("video/x-raw");
gst_caps_set_value (filter, "multiview-mode", modes);
gst_caps_set_features (filter, 0, gst_caps_features_new_any ());
GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT
" with caps %" GST_PTR_FORMAT, caps, filter);
result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (filter);
return result;
}
static GstCaps *
gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
GstCaps *base_caps =
gst_static_pad_template_get_caps (&gst_vulkan_sink_template);
GstCaps *out_caps, *tmp_caps;
gint i;
GST_DEBUG_OBJECT (viewconvert, "Direction %s "
"input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", caps, filter);
/* We can only process VulkanImage caps, start from that */
caps = gst_caps_intersect (caps, base_caps);
gst_caps_unref (base_caps);
/* Change input/output to the formats we can convert to/from,
* but keep the original caps at the start - we will always prefer
* passthrough */
if (direction == GST_PAD_SINK) {
out_caps = gst_caps_copy (caps);
if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
GstVideoMultiviewMode mode = viewconvert->input_mode_override;
GstVideoMultiviewFlags flags = viewconvert->input_flags_override;
const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode);
/* Coerce the input caps before transforming, so the sizes come out right */
gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING,
caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags,
GST_FLAG_SET_MASK_EXACT, NULL);
}
} else {
out_caps = gst_caps_new_empty ();
}
for (i = 0; i < gst_caps_get_size (caps); i++) {
GstStructure *structure = gst_caps_get_structure (caps, i);
GstCapsFeatures *features = gst_caps_get_features (caps, i);
out_caps = _expand_structure (viewconvert, out_caps, structure, features);
}
if (gst_caps_is_empty (out_caps))
goto out;
/* If we have an output mode override, limit things to that */
if (direction == GST_PAD_SINK &&
viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
tmp_caps = _intersect_with_mview_mode (out_caps,
viewconvert->output_mode_override, viewconvert->output_flags_override);
gst_caps_unref (out_caps);
out_caps = tmp_caps;
} else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* Prepend a copy of our preferred input caps in case the peer
* can handle them */
tmp_caps = _intersect_with_mview_mode (out_caps,
viewconvert->input_mode_override, viewconvert->input_flags_override);
out_caps = gst_caps_merge (out_caps, tmp_caps);
}
if (direction == GST_PAD_SRC) {
GstStructure *s;
/* When generating input caps, we also need a copy of the mono caps
* without multiview-mode or flags for backwards compat, at the end */
tmp_caps = _intersect_with_mview_mode (caps,
GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE);
if (!gst_caps_is_empty (tmp_caps)) {
s = gst_caps_get_structure (tmp_caps, 0);
gst_structure_remove_fields (s, "multiview-mode", "multiview-flags",
NULL);
out_caps = gst_caps_merge (out_caps, tmp_caps);
} else
gst_caps_unref (tmp_caps);
}
out:
gst_caps_unref (caps);
GST_DEBUG_OBJECT (viewconvert, "Have caps %" GST_PTR_FORMAT
" filtering with caps %" GST_PTR_FORMAT, out_caps, filter);
if (filter) {
GstCaps *tmp =
gst_caps_intersect_full (filter, out_caps, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (out_caps);
out_caps = tmp;
}
GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps);
return out_caps;
}
static GstCaps *
fixate_size (GstVulkanViewConvert * viewconvert,
GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
{
GstStructure *ins, *outs;
const GValue *from_par, *to_par;
GValue fpar = { 0, }, tpar = {
0,
};
othercaps = gst_caps_make_writable (othercaps);
othercaps = gst_caps_truncate (othercaps);
GST_DEBUG_OBJECT (viewconvert, "trying to fixate othercaps %" GST_PTR_FORMAT
" based on caps %" GST_PTR_FORMAT, othercaps, caps);
ins = gst_caps_get_structure (caps, 0);
outs = gst_caps_get_structure (othercaps, 0);
from_par = gst_structure_get_value (ins, "pixel-aspect-ratio");
to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
/* If we're fixating from the sinkpad we always set the PAR and
* assume that missing PAR on the sinkpad means 1/1 and
* missing PAR on the srcpad means undefined
*/
if (direction == GST_PAD_SINK) {
if (!from_par) {
g_value_init (&fpar, GST_TYPE_FRACTION);
gst_value_set_fraction (&fpar, 1, 1);
from_par = &fpar;
}
if (!to_par) {
g_value_init (&tpar, GST_TYPE_FRACTION);
gst_value_set_fraction (&tpar, 1, 1);
to_par = &tpar;
}
} else {
if (!to_par) {
g_value_init (&tpar, GST_TYPE_FRACTION);
gst_value_set_fraction (&tpar, 1, 1);
to_par = &tpar;
gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1,
NULL);
}
if (!from_par) {
g_value_init (&fpar, GST_TYPE_FRACTION);
gst_value_set_fraction (&fpar, 1, 1);
from_par = &fpar;
}
}
/* we have both PAR but they might not be fixated */
{
gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d;
gint w = 0, h = 0;
gint from_dar_n, from_dar_d;
gint num, den;
/* from_par should be fixed */
g_return_val_if_fail (gst_value_is_fixed (from_par), othercaps);
from_par_n = gst_value_get_fraction_numerator (from_par);
from_par_d = gst_value_get_fraction_denominator (from_par);
gst_structure_get_int (ins, "width", &from_w);
gst_structure_get_int (ins, "height", &from_h);
gst_structure_get_int (outs, "width", &w);
gst_structure_get_int (outs, "height", &h);
/* if both width and height are already fixed, we can't do anything
* about it anymore */
if (w && h) {
GST_DEBUG_OBJECT (viewconvert,
"dimensions already set to %dx%d, not fixating", w, h);
if (!gst_value_is_fixed (to_par)) {
GST_DEBUG_OBJECT (viewconvert, "fixating to_par to %dx%d", 1, 1);
if (gst_structure_has_field (outs, "pixel-aspect-ratio"))
gst_structure_fixate_field_nearest_fraction (outs,
"pixel-aspect-ratio", 1, 1);
}
goto done;
}
/* Calculate input DAR */
if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d,
&from_dar_n, &from_dar_d)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
goto done;
}
GST_DEBUG_OBJECT (viewconvert, "Input DAR is %d/%d", from_dar_n,
from_dar_d);
/* If either width or height are fixed there's not much we
* can do either except choosing a height or width and PAR
* that matches the DAR as good as possible
*/
if (h) {
gint num, den;
GST_DEBUG_OBJECT (viewconvert, "height is fixed (%d)", h);
if (!gst_value_is_fixed (to_par)) {
/* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
* so let's make it so ...
* especially if following code assumes fixed */
GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
gst_structure_fixate_field_nearest_fraction (outs,
"pixel-aspect-ratio", 1, 1);
to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
}
/* PAR is fixed, choose the height that is nearest to the
* height with the same DAR */
to_par_n = gst_value_get_fraction_numerator (to_par);
to_par_d = gst_value_get_fraction_denominator (to_par);
GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
to_par_n, &num, &den)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
goto done;
}
w = (guint) gst_util_uint64_scale_int (h, num, den);
gst_structure_fixate_field_nearest_int (outs, "width", w);
goto done;
} else if (w) {
gint num, den;
GST_DEBUG_OBJECT (viewconvert, "width is fixed (%d)", w);
if (!gst_value_is_fixed (to_par)) {
/* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1,
* so let's make it so ...
* especially if following code assumes fixed */
GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1");
gst_structure_fixate_field_nearest_fraction (outs,
"pixel-aspect-ratio", 1, 1);
to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
}
/* PAR is fixed, choose the height that is nearest to the
* height with the same DAR */
to_par_n = gst_value_get_fraction_numerator (to_par);
to_par_d = gst_value_get_fraction_denominator (to_par);
GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d);
if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d,
to_par_n, &num, &den)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
goto done;
}
h = (guint) gst_util_uint64_scale_int (w, den, num);
gst_structure_fixate_field_nearest_int (outs, "height", h);
goto done;
} else if (gst_value_is_fixed (to_par)) {
GstStructure *tmp;
gint set_h, set_w, f_h, f_w;
to_par_n = gst_value_get_fraction_numerator (to_par);
to_par_d = gst_value_get_fraction_denominator (to_par);
/* Calculate scale factor for the PAR change */
if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n,
to_par_d, &num, &den)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
goto done;
}
/* Try to keep the input height */
tmp = gst_structure_copy (outs);
gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
gst_structure_get_int (tmp, "height", &set_h);
/* This might have failed but try to scale the width
* to keep the DAR nonetheless */
w = (guint) gst_util_uint64_scale_int (set_h, num, den);
gst_structure_fixate_field_nearest_int (tmp, "width", w);
gst_structure_get_int (tmp, "width", &set_w);
gst_structure_free (tmp);
/* We kept the DAR and the height is nearest to the original height */
if (set_w == w) {
gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
G_TYPE_INT, set_h, NULL);
goto done;
}
f_h = set_h;
f_w = set_w;
/* If the former failed, try to keep the input width at least */
tmp = gst_structure_copy (outs);
gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
gst_structure_get_int (tmp, "width", &set_w);
/* This might have failed but try to scale the width
* to keep the DAR nonetheless */
h = (guint) gst_util_uint64_scale_int (set_w, den, num);
gst_structure_fixate_field_nearest_int (tmp, "height", h);
gst_structure_get_int (tmp, "height", &set_h);
gst_structure_free (tmp);
/* We kept the DAR and the width is nearest to the original width */
if (set_h == h) {
gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
G_TYPE_INT, set_h, NULL);
goto done;
}
/* If all this failed, keep the height that was nearest to the original
* height and the nearest possible width. This changes the DAR but
* there's not much else to do here.
*/
gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT,
f_h, NULL);
goto done;
} else {
GstStructure *tmp;
gint set_h, set_w, set_par_n, set_par_d, tmp2;
/* width, height and PAR are not fixed */
/* First try to keep the height and width as good as possible
* and scale PAR */
tmp = gst_structure_copy (outs);
gst_structure_fixate_field_nearest_int (tmp, "height", from_h);
gst_structure_get_int (tmp, "height", &set_h);
gst_structure_fixate_field_nearest_int (tmp, "width", from_w);
gst_structure_get_int (tmp, "width", &set_w);
if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w,
&to_par_n, &to_par_d)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
gst_structure_free (tmp);
goto done;
}
if (!gst_structure_has_field (tmp, "pixel-aspect-ratio"))
gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par);
gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio",
to_par_n, to_par_d);
gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n,
&set_par_d);
gst_structure_free (tmp);
if (set_par_n == to_par_n && set_par_d == to_par_d) {
gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
G_TYPE_INT, set_h, NULL);
if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
set_par_n != set_par_d)
gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
set_par_n, set_par_d, NULL);
goto done;
}
/* Otherwise try to scale width to keep the DAR with the set
* PAR and height */
if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d,
set_par_n, &num, &den)) {
GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL),
("Error calculating the output scaled size - integer overflow"));
goto done;
}
w = (guint) gst_util_uint64_scale_int (set_h, num, den);
tmp = gst_structure_copy (outs);
gst_structure_fixate_field_nearest_int (tmp, "width", w);
gst_structure_get_int (tmp, "width", &tmp2);
gst_structure_free (tmp);
if (tmp2 == w) {
gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height",
G_TYPE_INT, set_h, NULL);
if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
set_par_n != set_par_d)
gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
set_par_n, set_par_d, NULL);
goto done;
}
/* ... or try the same with the height */
h = (guint) gst_util_uint64_scale_int (set_w, den, num);
tmp = gst_structure_copy (outs);
gst_structure_fixate_field_nearest_int (tmp, "height", h);
gst_structure_get_int (tmp, "height", &tmp2);
gst_structure_free (tmp);
if (tmp2 == h) {
gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
G_TYPE_INT, tmp2, NULL);
if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
set_par_n != set_par_d)
gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
set_par_n, set_par_d, NULL);
goto done;
}
/* If all fails we can't keep the DAR and take the nearest values
* for everything from the first try */
gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height",
G_TYPE_INT, set_h, NULL);
if (gst_structure_has_field (outs, "pixel-aspect-ratio") ||
set_par_n != set_par_d)
gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION,
set_par_n, set_par_d, NULL);
}
}
done:
othercaps = gst_caps_fixate (othercaps);
GST_DEBUG_OBJECT (viewconvert, "fixated othercaps to %" GST_PTR_FORMAT,
othercaps);
if (from_par == &fpar)
g_value_unset (&fpar);
if (to_par == &tpar)
g_value_unset (&tpar);
return othercaps;
}
static GstCaps *
gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
{
GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt);
GstVideoMultiviewMode mode = viewconvert->output_mode_override;
GstVideoMultiviewFlags flags = viewconvert->output_flags_override;
GstCaps *tmp;
othercaps = gst_caps_make_writable (othercaps);
GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT
" against caps %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
if (direction == GST_PAD_SINK) {
if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* We have a requested output mode and are fixating source caps, try and enforce it */
GST_DEBUG_OBJECT (bt, "fixating multiview mode using the configured "
"output override mode 0x%x and flags 0x%x", mode, flags);
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
gst_caps_unref (othercaps);
othercaps = tmp;
} else {
/* See if we can do passthrough */
GstVideoInfo info;
if (gst_video_info_from_caps (&info, caps)) {
GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info);
GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info);
if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
mode = viewconvert->input_mode_override;
flags = viewconvert->input_flags_override;
}
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
if (gst_caps_is_empty (tmp)) {
/* Nope, we can't pass our input caps downstream */
gst_caps_unref (tmp);
} else {
GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
"using the input override mode 0x%x and flags 0x%x", mode, flags);
gst_caps_unref (othercaps);
othercaps = tmp;
goto done;
}
}
/* Prefer an unpacked mode for output */
tmp =
_intersect_with_mview_modes (othercaps,
gst_video_multiview_get_unpacked_modes ());
if (!gst_caps_is_empty (tmp)) {
GST_DEBUG_OBJECT (bt, "preferring an unpacked multiview mode");
gst_caps_unref (othercaps);
othercaps = tmp;
} else {
gst_caps_unref (tmp);
}
}
} else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) {
/* See if we can coerce the caps into matching input mode/flags,
* in case it doesn't care at all, but allow it not to too */
mode = viewconvert->input_mode_override;
flags = viewconvert->input_flags_override;
tmp = _intersect_with_mview_mode (othercaps, mode, flags);
if (gst_caps_is_empty (tmp)) {
/* Nope, we can pass our input caps downstream */
gst_caps_unref (tmp);
} else {
GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode "
"using the input override mode 0x%x and flags 0x%x", mode, flags);
gst_caps_unref (othercaps);
othercaps = tmp;
}
}
done:
othercaps = fixate_size (viewconvert, direction, caps, othercaps);
othercaps = gst_caps_fixate (othercaps);
GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT
" against caps %" GST_PTR_FORMAT,
direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps);
return othercaps;
}
static gboolean
create_descriptor_set_layout (GstVulkanViewConvert * conv, guint n_mems,
GError ** error)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES * 2 + 1] =
{ {0,}, };
VkDescriptorSetLayoutCreateInfo layout_info;
VkDescriptorSetLayout descriptor_set_layout;
int descriptor_n = 0;
VkResult err;
int i;
/* *INDENT-OFF* */
bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
.binding = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT
};
for (i = 0; i < n_mems * 2; i++) {
bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
.binding = i + 1,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
};
};
layout_info = (VkDescriptorSetLayoutCreateInfo) {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = NULL,
.bindingCount = descriptor_n,
.pBindings = bindings
};
/* *INDENT-ON* */
err =
vkCreateDescriptorSetLayout (vfilter->device->device, &layout_info,
NULL, &descriptor_set_layout);
if (gst_vulkan_error_to_g_error (err, error,
"vkCreateDescriptorSetLayout") < 0) {
return FALSE;
}
conv->quad->descriptor_set_layout =
gst_vulkan_handle_new_wrapped (vfilter->device,
GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT,
(GstVulkanHandleTypedef) descriptor_set_layout,
gst_vulkan_handle_free_descriptor_set_layout, NULL);
return TRUE;
}
static gboolean
gst_vulkan_view_convert_start (GstBaseTransform * bt)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanHandle *vert, *frag;
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
return FALSE;
conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);
if (!(vert =
gst_vulkan_create_shader (vfilter->device, identity_vert,
identity_vert_size, NULL))) {
return FALSE;
}
if (!(frag =
gst_vulkan_create_shader (vfilter->device, view_convert_frag,
view_convert_frag_size, NULL))) {
gst_vulkan_handle_unref (vert);
return FALSE;
}
if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) {
gst_vulkan_handle_unref (vert);
gst_vulkan_handle_unref (frag);
return FALSE;
}
gst_vulkan_handle_unref (vert);
gst_vulkan_handle_unref (frag);
return TRUE;
}
static gboolean
gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
GstCaps * out_caps)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
out_caps))
return FALSE;
if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info,
&vfilter->out_info))
return FALSE;
if (conv->uniform)
gst_memory_unref (conv->uniform);
conv->uniform = NULL;
return TRUE;
}
static gboolean
gst_vulkan_view_convert_stop (GstBaseTransform * bt)
{
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
gst_clear_object (&conv->quad);
if (conv->uniform)
gst_memory_unref (conv->uniform);
conv->uniform = NULL;
return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
}
static GstFlowReturn
gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf,
GstBuffer * outbuf)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanCommandBuffer *cmd_buf = NULL;
GstVulkanFence *fence = NULL;
GError *error = NULL;
VkResult err;
guint in_n_mems, out_n_mems;
int i;
if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error))
goto error;
if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, outbuf,
&error))
goto error;
fence = gst_vulkan_device_create_fence (vfilter->device, &error);
if (!fence)
goto error;
in_n_mems = gst_buffer_n_memory (inbuf);
for (i = 0; i < in_n_mems; i++) {
GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i);
if (!gst_is_vulkan_image_memory (img_mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Input memory must be a GstVulkanImageMemory");
goto error;
}
in_img_views[i] =
gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) img_mem);
gst_vulkan_trash_list_add (conv->quad->trash_list,
gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
gst_vulkan_trash_mini_object_unref,
(GstMiniObject *) in_img_views[i]));
}
out_n_mems = gst_buffer_n_memory (outbuf);
for (i = 0; i < out_n_mems; i++) {
GstMemory *mem = gst_buffer_peek_memory (outbuf, i);
if (!gst_is_vulkan_image_memory (mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Output memory must be a GstVulkanImageMemory");
goto error;
}
out_img_views[i] =
gst_vulkan_get_or_create_image_view ((GstVulkanImageMemory *) mem);
gst_vulkan_trash_list_add (conv->quad->trash_list,
gst_vulkan_trash_list_acquire (conv->quad->trash_list, fence,
gst_vulkan_trash_mini_object_unref,
(GstMiniObject *) out_img_views[i]));
}
{
GstMemory *uniforms = get_uniforms (conv, in_img_views, out_img_views);
if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms,
&error))
goto error;
gst_memory_unref (uniforms);
}
if (!conv->quad->descriptor_set_layout)
if (!create_descriptor_set_layout (conv, in_n_mems, &error))
goto error;
if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error))
goto error;
if (!(cmd_buf =
gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error)))
goto error;
{
VkCommandBufferBeginInfo cmd_buf_info = { 0, };
/* *INDENT-OFF* */
cmd_buf_info = (VkCommandBufferBeginInfo) {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.pNext = NULL,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
.pInheritanceInfo = NULL
};
/* *INDENT-ON* */
gst_vulkan_command_buffer_lock (cmd_buf);
err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0)
goto error;
}
update_descriptor_set (conv, in_img_views, in_n_mems);
if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf,
fence, &error))
goto unlock_error;
err = vkEndCommandBuffer (cmd_buf->cmd);
gst_vulkan_command_buffer_unlock (cmd_buf);
if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0)
goto error;
if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error))
goto error;
gst_vulkan_fence_unref (fence);
return GST_FLOW_OK;
unlock_error:
if (cmd_buf) {
gst_vulkan_command_buffer_unlock (cmd_buf);
gst_vulkan_command_buffer_unref (cmd_buf);
}
error:
gst_clear_mini_object ((GstMiniObject **) & fence);
GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
g_clear_error (&error);
return GST_FLOW_ERROR;
}
Reference in a new issue View git blame Copy permalink