diff --git a/ext/vulkan/gstvulkan.c b/ext/vulkan/gstvulkan.c index 6dea43b36f..c7441e2487 100644 --- a/ext/vulkan/gstvulkan.c +++ b/ext/vulkan/gstvulkan.c @@ -34,6 +34,7 @@ #include "vkimageidentity.h" #include "vkcolorconvert.h" #include "vkdownload.h" +#include "vkviewconvert.h" #define GST_CAT_DEFAULT gst_vulkan_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -68,6 +69,11 @@ plugin_init (GstPlugin * plugin) return FALSE; } + if (!gst_element_register (plugin, "vulkanviewconvert", + GST_RANK_NONE, GST_TYPE_VULKAN_VIEW_CONVERT)) { + return FALSE; + } + return TRUE; } diff --git a/ext/vulkan/meson.build b/ext/vulkan/meson.build index 93cae94793..34276a3126 100644 --- a/ext/vulkan/meson.build +++ b/ext/vulkan/meson.build @@ -17,10 +17,23 @@ vulkan_sources = [ 'vksink.c', 'vkshader.c', 'vkupload.c', + 'vkviewconvert.c', ] +vulkan_plugin_enum_headers = [ + 'vkviewconvert.h', +] + +vulkan_plugin_enums = gnome.mkenums_simple('gstvulkan-plugins-enumtypes', + sources : vulkan_plugin_enum_headers, + body_prefix : '#ifdef HAVE_CONFIG_H\n#include "config.h"\n#endif', + header_prefix : '#include ') +vulkan_plugin_enumtypes_c = vulkan_plugin_enums[0] +vulkan_plugin_enumtypes_h = vulkan_plugin_enums[1] +vulkan_plugin_gen_sources = [vulkan_plugin_enumtypes_h] + gstvulkan_plugin = library('gstvulkan', - vulkan_sources, vulkan_compiled_shader_sources, + vulkan_sources, vulkan_compiled_shader_sources, vulkan_plugin_enumtypes_c, vulkan_plugin_enumtypes_h, c_args : gst_plugins_bad_args, objc_args : gst_plugins_bad_args, link_args : noseh_link_args, diff --git a/ext/vulkan/shaders/meson.build b/ext/vulkan/shaders/meson.build index fdae040301..f822dd0eb5 100644 --- a/ext/vulkan/shaders/meson.build +++ b/ext/vulkan/shaders/meson.build @@ -10,6 +10,7 @@ gst_vulkan_shader_sources = [ 'rgb_to_ayuv.frag', 'nv12_to_rgb.frag', 'rgb_to_nv12.frag', + 'view_convert.frag', ] bin2array = find_program('bin2array.py') diff --git a/ext/vulkan/shaders/view_convert.frag b/ext/vulkan/shaders/view_convert.frag new file mode 100644 index 0000000000..1ac1e71872 --- /dev/null +++ b/ext/vulkan/shaders/view_convert.frag @@ -0,0 +1,79 @@ +#version 450 core + +#include "view_defines.h" +#include "swizzle.glsl" + +layout(location = 0) in vec2 inTexCoord; + +layout(set = 0, binding = 0) uniform sampler2D l_tex; +layout(set = 0, binding = 1) uniform sampler2D r_tex; +layout(set = 0, binding = 2) uniform ViewConvert { + ivec4 in_reorder_idx; + ivec4 out_reorder_idx; + vec4 tex_offset; + vec4 tex_scale; + ivec2 tex_size; + int output_type; + mat3 downmix[2]; +}; + +layout(location = 0) out vec4 outColor0; + +void main() +{ + vec2 l_coord = inTexCoord * tex_scale.xy + tex_offset.xy; + vec2 r_coord = inTexCoord * tex_scale.zw + tex_offset.zw; + vec4 l = swizzle(texture(l_tex, l_coord), in_reorder_idx); + vec4 r = swizzle(texture(r_tex, r_coord), in_reorder_idx); + + if (output_type == VIEW_MONO_DOWNMIX) { + vec3 lcol = l.rgb * l.a + vec3(1.0-l.a); + vec3 rcol = r.rgb * r.a + vec3(1.0-r.a); + if (l.a + r.a > 0.0) { + lcol = clamp (downmix[0] * lcol, 0.0, 1.0); + rcol = clamp (downmix[1] * rcol, 0.0, 1.0); + outColor0 = vec4 (lcol + rcol, 1.0); + } else { + outColor0 = vec4 (0.0); + } + } else if (output_type == VIEW_MONO_LEFT) { + outColor0 = swizzle(l, out_reorder_idx); + outColor0 = vec4(1.0, 0.0, 1.0, 1.0); + } else if (output_type == VIEW_MONO_RIGHT) { + outColor0 = swizzle(r, out_reorder_idx); + outColor0 = vec4(0.0, 0.0, 1.0, 1.0); + } else if (output_type == VIEW_SIDE_BY_SIDE) { + if (inTexCoord.x < 0.5) { + outColor0 = swizzle(l, out_reorder_idx); + } else { + outColor0 = swizzle(r, out_reorder_idx); + } + } else if (output_type == VIEW_TOP_BOTTOM) { + if (inTexCoord.y < 0.5) { + outColor0 = swizzle(l, out_reorder_idx); + } else { + outColor0 = swizzle(r, out_reorder_idx); + } + } else if (output_type == VIEW_COLUMN_INTERLEAVED) { + if (int(mod(l_coord.x * tex_size.x, 2.0)) == 0) { + outColor0 = swizzle(l, out_reorder_idx); + } else { + outColor0 = swizzle(r, out_reorder_idx); + } + } else if (output_type == VIEW_ROW_INTERLEAVED) { + if (int(mod(l_coord.y * tex_size.y, 2.0)) == 0) { + outColor0 = swizzle(l, out_reorder_idx); + } else { + outColor0 = swizzle(r, out_reorder_idx); + } + } else if (output_type == VIEW_CHECKERBOARD) { + if (int(mod(l_coord.x * tex_size.x, 2.0)) == + int(mod(l_coord.y * tex_size.y, 2.0))) { + outColor0 = swizzle(l, out_reorder_idx); + } else { + outColor0 = swizzle(r, out_reorder_idx); + } + } else { + outColor0 = vec4(1.0, 0.0, 1.0, 1.0); + } +} diff --git a/ext/vulkan/shaders/view_defines.h b/ext/vulkan/shaders/view_defines.h new file mode 100644 index 0000000000..434fc86aae --- /dev/null +++ b/ext/vulkan/shaders/view_defines.h @@ -0,0 +1,10 @@ +/* values match GstVideoMultiviewMode */ +#define VIEW_MONO_DOWNMIX 0 +#define VIEW_MONO_LEFT 1 +#define VIEW_MONO_RIGHT 2 +#define VIEW_SIDE_BY_SIDE 3 +/* GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX,*/ +#define VIEW_COLUMN_INTERLEAVED 5 +#define VIEW_ROW_INTERLEAVED 6 +#define VIEW_TOP_BOTTOM 7 +#define VIEW_CHECKERBOARD 8 diff --git a/ext/vulkan/vkviewconvert.c b/ext/vulkan/vkviewconvert.c new file mode 100644 index 0000000000..cac13e9166 --- /dev/null +++ b/ext/vulkan/vkviewconvert.c @@ -0,0 +1,2388 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-vulkanimageidentity + * @title: vulkanimgeidentity + * + * vulkanimageidentity produces a vulkan image that is a copy of the input image. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "vkviewconvert.h" +#include "vkshader.h" +#include "shaders/identity.vert.h" +#include "shaders/view_convert.frag.h" +#include "gstvulkan-plugins-enumtypes.h" + +GST_DEBUG_CATEGORY (gst_debug_vulkan_view_convert); +#define GST_CAT_DEFAULT gst_debug_vulkan_view_convert + +/* *INDENT-OFF* */ +/* These match the order and number of DOWNMIX_ANAGLYPH_* modes */ +static float downmix_matrices[][2][12] = { + { /* Green-Magenta Dubois */ + {-0.062f, 0.284f, -0.015f, 0.0, -0.158f, 0.668f, -0.027f, 0.0, -0.039f, 0.143f, 0.021f, 0.0}, + {0.529f, -0.016f, 0.009f, 0.0, 0.705f, -0.015f, 0.075f, 0.0, 0.024f, -0.065f, 0.937f, 0.0} + }, + { /* Red-Cyan Dubois */ + /* Source of this matrix: http://www.site.uottawa.ca/~edubois/anaglyph/LeastSquaresHowToPhotoshop.pdf */ + {0.437f, -0.062f, -0.048f, 0.0, 0.449f, -0.062f, -0.050f, 0.0, 0.164f, -0.024f, -0.017f}, + {-0.011f, 0.377f, -0.026f, 0.0, -0.032f, 0.761f, -0.093f, 0.0, -0.007f, 0.009f, 1.234f} + }, + { /* Amber-blue Dubois */ + {1.062f, -0.026f, -0.038f, 0.0, -0.205f, 0.908f, -0.173f, 0.0, 0.299f, 0.068f, 0.022f}, + {-0.016f, 0.006f, 0.094f, 0.0, -0.123f, 0.062f, 0.185f, 0.0, -0.017f, -0.017f, 0.911f} + } +}; +/* *INDENT-ON* */ + +struct ViewUpdate +{ + int in_reorder_idx[4]; + int out_reorder_idx[4]; + float tex_offset[2][2]; + float tex_scale[2][2]; + int tex_size[2]; + int output_type; + int _padding; + float downmix[2][12]; +}; + +static void +get_rgb_format_swizzle_order (GstVideoFormat format, + gint swizzle[GST_VIDEO_MAX_COMPONENTS]) +{ + const GstVideoFormatInfo *finfo = gst_video_format_get_info (format); + int c_i = 0, i; + + g_return_if_fail (finfo->flags & GST_VIDEO_FORMAT_FLAG_RGB + || format == GST_VIDEO_FORMAT_AYUV); + + for (i = 0; i < finfo->n_components; i++) { + swizzle[c_i++] = finfo->poffset[i]; + } + + /* special case spaced RGB formats as the space does not contain a poffset + * value and we need all four components to be valid in order to swizzle + * correctly */ + if (format == GST_VIDEO_FORMAT_xRGB || format == GST_VIDEO_FORMAT_xBGR) { + swizzle[c_i++] = 0; + } else if (format == GST_VIDEO_FORMAT_RGBx || format == GST_VIDEO_FORMAT_BGRx) { + swizzle[c_i++] = 3; + } else { + for (i = finfo->n_components; i < GST_VIDEO_MAX_COMPONENTS; i++) { + swizzle[c_i++] = -1; + } + } +} + +static void +get_vulkan_rgb_format_swizzle_order (VkFormat format, gint * swizzle, + guint swizzle_count, guint offset) +{ + const GstVulkanFormatInfo *finfo = gst_vulkan_format_get_info (format); + int i; + + g_return_if_fail (finfo->flags & GST_VULKAN_FORMAT_FLAG_RGB); + g_return_if_fail (finfo->n_components <= swizzle_count); + + for (i = 0; i < finfo->n_components; i++) { + swizzle[i] = offset + finfo->poffset[i]; + } + for (i = finfo->n_components; i < swizzle_count; i++) { + swizzle[i] = -1; + } +} + +/* given a swizzle index, produce an index such that: + * + * swizzle[idx[i]] == identity[i] where: + * - swizzle is the original swizzle + * - idx is the result + * - identity = {0, 1, 2,...} + * - unset fields are marked by -1 + */ +static void +swizzle_identity_order (gint * swizzle, gint * idx) +{ + int i; + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { + idx[i] = -1; + } + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) { + if (swizzle[i] >= 0 && swizzle[i] < 4 && idx[swizzle[i]] == -1) { + idx[swizzle[i]] = i; + } + } +} + +static void +video_format_to_reorder (GstVideoFormat v_format, gint * reorder, + gboolean input) +{ + switch (v_format) { + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_BGRA: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_ARGB: + case GST_VIDEO_FORMAT_xRGB: + case GST_VIDEO_FORMAT_ABGR: + case GST_VIDEO_FORMAT_xBGR: + case GST_VIDEO_FORMAT_AYUV: + get_rgb_format_swizzle_order (v_format, reorder); + break; + case GST_VIDEO_FORMAT_UYVY: + reorder[0] = 1; + reorder[1] = 0; + reorder[2] = input ? 3 : 2; + reorder[3] = 0; + break; + case GST_VIDEO_FORMAT_YUY2: + reorder[0] = 0; + reorder[1] = 1; + reorder[2] = 0; + reorder[3] = input ? 3 : 2; + break; + case GST_VIDEO_FORMAT_NV12: + reorder[0] = 0; + reorder[1] = 1; + reorder[2] = 2; + reorder[3] = 0; + break; + default: + g_assert_not_reached (); + break; + } + + GST_TRACE ("swizzle: %u, %u, %u, %u", reorder[0], reorder[1], reorder[2], + reorder[3]); +} + +static guint +finfo_get_plane_n_components (const GstVideoFormatInfo * finfo, guint plane) +{ + guint n_components = 0, i; + + switch (finfo->format) { + case GST_VIDEO_FORMAT_RGBx: + case GST_VIDEO_FORMAT_xRGB: + case GST_VIDEO_FORMAT_BGRx: + case GST_VIDEO_FORMAT_xBGR: + /* fixup spaced RGB formats as we treat the space as a normal alpha + * component */ + return plane == 0 ? 4 : 0; + default: + break; + } + + for (i = 0; i < finfo->n_components; i++) { + if (finfo->plane[i] == plane) + n_components++; + } + + return n_components; +} + +static void +get_vulkan_format_swizzle_order (GstVideoFormat v_format, + VkFormat vk_format[GST_VIDEO_MAX_PLANES], + gint swizzle[GST_VIDEO_MAX_COMPONENTS]) +{ + const GstVideoFormatInfo *finfo; + int i, prev_in_i = 0; + + finfo = gst_video_format_get_info (v_format); + for (i = 0; i < finfo->n_planes; i++) { + guint plane_components = finfo_get_plane_n_components (finfo, i); + get_vulkan_rgb_format_swizzle_order (vk_format[i], + &swizzle[prev_in_i], plane_components, prev_in_i); + prev_in_i += plane_components; + } + + if (v_format == GST_VIDEO_FORMAT_YUY2 || v_format == GST_VIDEO_FORMAT_UYVY) { + /* Fixup these packed YUV formats as we use a two component format for + * a 4-component pixel and access two samples in the shader */ + g_assert (swizzle[0] == 0); + g_assert (swizzle[1] == 1); + swizzle[2] = 2; + swizzle[3] = 3; + } + + GST_TRACE ("%s: %i, %i, %i, %i", finfo->name, swizzle[0], swizzle[1], + swizzle[2], swizzle[3]); +} + +static void +calculate_reorder_indexes (GstVideoFormat in_format, + GstVulkanImageMemory * in_mems[GST_VIDEO_MAX_COMPONENTS], + GstVideoFormat out_format, + GstVulkanImageMemory * out_mems[GST_VIDEO_MAX_COMPONENTS], + int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS]) +{ + const GstVideoFormatInfo *in_finfo, *out_finfo; + VkFormat in_vk_formats[GST_VIDEO_MAX_COMPONENTS]; + VkFormat out_vk_formats[GST_VIDEO_MAX_COMPONENTS]; + int in_vk_order[GST_VIDEO_MAX_COMPONENTS], + in_reorder[GST_VIDEO_MAX_COMPONENTS]; + int out_vk_order[GST_VIDEO_MAX_COMPONENTS], + out_reorder[GST_VIDEO_MAX_COMPONENTS]; + int tmp[GST_VIDEO_MAX_PLANES]; + int i; + + in_finfo = gst_video_format_get_info (in_format); + out_finfo = gst_video_format_get_info (out_format); + + for (i = 0; i < in_finfo->n_planes; i++) + in_vk_formats[i] = in_mems[i]->create_info.format; + for (i = 0; i < out_finfo->n_planes; i++) + out_vk_formats[i] = out_mems[i]->create_info.format; + + get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order); + video_format_to_reorder (in_format, in_reorder, TRUE); + + video_format_to_reorder (out_format, out_reorder, FALSE); + get_vulkan_format_swizzle_order (out_format, out_vk_formats, out_vk_order); + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) + tmp[i] = out_vk_order[out_reorder[i]]; + /* find the identity order for RGBA->$format */ + GST_TRACE ("pre-invert: %u, %u, %u, %u", tmp[0], tmp[1], tmp[2], tmp[3]); + if (out_format == GST_VIDEO_FORMAT_YUY2 + || out_format == GST_VIDEO_FORMAT_UYVY) { + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) + ret_out[i] = tmp[i]; + } else { + swizzle_identity_order (tmp, ret_out); + } + + for (i = 0; i < GST_VIDEO_MAX_COMPONENTS; i++) + ret_in[i] = in_reorder[in_vk_order[i]]; + GST_TRACE ("in reorder: %u, %u, %u, %u", ret_in[0], ret_in[1], ret_in[2], + ret_in[3]); + GST_TRACE ("out reorder: %u, %u, %u, %u", ret_out[0], ret_out[1], ret_out[2], + ret_out[3]); +} + +static void +update_descriptor_set (GstVulkanViewConvert * conv, VkImageView * views, + guint n_views) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + VkDescriptorBufferInfo buffer_info; + VkDescriptorImageInfo image_info[2 * GST_VIDEO_MAX_PLANES]; + VkWriteDescriptorSet writes[10]; + guint i = 0; + + for (; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) { + /* *INDENT-OFF* */ + image_info[i] = (VkDescriptorImageInfo) { + .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + .imageView = views[i], + .sampler = conv->sampler + }; + + g_assert (i < n_views); + g_assert (i < GST_VIDEO_MAX_PLANES); + + writes[i] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = conv->descriptor_set, + .dstBinding = i, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 1, + .pImageInfo = &image_info[i] + }; + /* *INDENT-ON* */ + } + /* *INDENT-OFF* */ + buffer_info = (VkDescriptorBufferInfo) { + .buffer = ((GstVulkanBufferMemory *) conv->uniform)->buffer, + .offset = 0, + .range = sizeof (struct ViewUpdate), + }; + writes[i] = (VkWriteDescriptorSet) { + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = NULL, + .dstSet = conv->descriptor_set, + .dstBinding = i, + .dstArrayElement = 0, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = 1, + .pBufferInfo = &buffer_info + }; + /* *INDENT-ON* */ + i++; + g_assert (i <= G_N_ELEMENTS (writes)); + + vkUpdateDescriptorSets (render->device->device, i, writes, 0, NULL); +} + +static gboolean +_update_uniform (GstVulkanViewConvert * conv, GstVulkanImageMemory ** in_mems, + GstVulkanImageMemory ** out_mems, VkImageView views[GST_VIDEO_MAX_PLANES]) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + GstVideoMultiviewMode in_mode, out_mode; + GstVideoMultiviewFlags in_flags, out_flags; + struct ViewUpdate data; + GstMapInfo map_info; + guint l_index, r_index; + gboolean mono_input = FALSE; + + calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), + in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_mems, data.in_reorder_idx, data.out_reorder_idx); + + data.tex_scale[0][0] = data.tex_scale[0][1] = 1.; + data.tex_scale[1][0] = data.tex_scale[1][1] = 1.; + data.tex_offset[0][0] = data.tex_offset[0][1] = 0.; + data.tex_offset[1][0] = data.tex_offset[1][1] = 0.; + + in_mode = conv->input_mode_override; + in_flags = conv->input_flags_override; + if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) { + in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->in_info); + in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->in_info); + } + + /* Configured output mode already takes any override + * into account */ + out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->out_info); + out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->out_info); + + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) == + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) { + l_index = 0; + r_index = 1; + } else { + VkImageView scratch_view; + + GST_LOG_OBJECT (conv, "Switching left/right views"); + + l_index = 1; + r_index = 0; + scratch_view = views[0]; + views[0] = views[1]; + views[1] = scratch_view; + } + + if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */ + } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE || + in_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) { + /* Side-by-side input */ + data.tex_offset[r_index][0] += 0.5 * data.tex_scale[r_index][0]; + data.tex_scale[0][0] *= 0.5f; /* Half horizontal scale */ + data.tex_scale[1][0] *= 0.5f; + } else if (in_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { /* top-bottom */ + data.tex_offset[r_index][1] += 0.5 * data.tex_scale[r_index][1]; + data.tex_scale[0][1] *= 0.5f; /* Half vertical scale */ + data.tex_scale[1][1] *= 0.5f; + } + + /* Flipped is vertical, flopped is horizontal. + * Adjust and offset per-view scaling. This needs to be done + * after the input scaling already splits the views, before + * adding any output scaling. */ + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED) != + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLIPPED)) { + data.tex_offset[l_index][1] += data.tex_scale[l_index][1]; + data.tex_scale[l_index][1] *= -1.0; + } + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED) != + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_LEFT_FLOPPED)) { + data.tex_offset[l_index][0] += data.tex_scale[l_index][0]; + data.tex_scale[l_index][0] *= -1.0; + } + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED) != + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLIPPED)) { + data.tex_offset[r_index][1] += data.tex_scale[r_index][1]; + data.tex_scale[r_index][1] *= -1.0; + } + if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED) != + (out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_FLOPPED)) { + data.tex_offset[r_index][0] += data.tex_scale[r_index][0]; + data.tex_scale[r_index][0] *= -1.0; + } + + if (out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE || + out_mode == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) { + /* Side-by-Side */ + data.tex_offset[1][0] -= data.tex_scale[1][0]; + data.tex_scale[0][0] *= 2.0f; + data.tex_scale[1][0] *= 2.0f; + } else if (out_mode == GST_VIDEO_MULTIVIEW_MODE_TOP_BOTTOM) { + data.tex_offset[1][1] -= data.tex_scale[1][1]; + data.tex_scale[0][1] *= 2.0f; + data.tex_scale[1][1] *= 2.0f; + } + + GST_DEBUG_OBJECT (conv, + "Scaling matrix [ %f, %f ] [ %f %f]. Offsets [ %f, %f ] [ %f, %f ]", + data.tex_scale[0][0], data.tex_scale[0][1], + data.tex_scale[1][0], data.tex_scale[1][1], + data.tex_offset[0][0], data.tex_offset[0][1], data.tex_offset[1][0], + data.tex_offset[1][1]); + + if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE || + in_mode == GST_VIDEO_MULTIVIEW_MODE_MONO || + in_mode == GST_VIDEO_MULTIVIEW_MODE_LEFT || + in_mode == GST_VIDEO_MULTIVIEW_MODE_RIGHT) + mono_input = TRUE; + + data.output_type = out_mode; + if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_NONE || + data.output_type == GST_VIDEO_MULTIVIEW_MODE_MONO) { + if (mono_input) + data.output_type = GST_VIDEO_MULTIVIEW_MODE_LEFT; + else + data.output_type = GST_VIDEO_MULTIVIEW_MODE_MONO; + } else if (data.output_type == GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX) { + data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE; + } + + data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&render->out_info); + data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&render->out_info); + memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0], + sizeof (data.downmix[0])); + memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1], + sizeof (data.downmix[1])); + + if (!gst_memory_map (conv->uniform, &map_info, GST_MAP_WRITE)) { + return FALSE; + } + memcpy (map_info.data, &data, sizeof (data)); + gst_memory_unmap (conv->uniform, &map_info); + conv->descriptor_up_to_date = TRUE; + + return TRUE; +} + +static gboolean +view_convert_update_command_state (GstVulkanViewConvert * conv, + VkCommandBuffer cmd, GstVulkanImageMemory ** in_mems, + GstVulkanImageMemory ** out_mems) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + VkImageView views[GST_VIDEO_MAX_PLANES]; + int i; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { + views[2 * i] = in_mems[i]->view; + views[2 * i + 1] = in_mems[i]->view; + } + + if (!conv->descriptor_up_to_date) { + if (!_update_uniform (conv, in_mems, out_mems, views)) + return FALSE; + update_descriptor_set (conv, views, + GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2); + } + + vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + render->pipeline_layout, 0, 1, &conv->descriptor_set, 0, NULL); + + return TRUE; +} + +static void gst_vulkan_view_convert_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_vulkan_view_convert_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +static gboolean gst_vulkan_view_convert_start (GstBaseTransform * bt); +static gboolean gst_vulkan_view_convert_stop (GstBaseTransform * bt); + +static GstCaps *gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * filter); +static GstCaps *gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); +static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt, + GstBuffer * inbuf, GstBuffer * outbuf); +static GstFlowReturn gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, + GstCaps * in_caps, GstCaps * out_caps); + +static VkAttachmentReference + * gst_vulkan_view_convert_render_pass_attachment_references + (GstVulkanFullScreenRender * render, guint * n_attachments); +static VkAttachmentDescription + * gst_vulkan_view_convert_render_pass_attachment_descriptions + (GstVulkanFullScreenRender * render, guint * n_descriptions); +static VkDescriptorSetLayoutBinding + * gst_vulkan_view_convert_descriptor_set_layout_bindings + (GstVulkanFullScreenRender * render, guint * n_bindings); +static void +gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render); +static VkPushConstantRange + * gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender * + render, guint * n_constants); + +static GstStaticPadTemplate gst_vulkan_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, + "{ BGRA, RGBA }"))); + +static GstStaticPadTemplate gst_vulkan_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, + "{ BGRA, RGBA }"))); + +enum +{ + PROP_0, + PROP_INPUT_LAYOUT, + PROP_INPUT_FLAGS, + PROP_OUTPUT_LAYOUT, + PROP_OUTPUT_FLAGS, + PROP_OUTPUT_DOWNMIX_MODE +}; + +#define DEFAULT_DOWNMIX GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS + +#define gst_vulkan_view_convert_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert, + GST_TYPE_VULKAN_FULL_SCREEN_RENDER, + GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert, + "vulkanviewconvert", 0, "Vulkan View Convert")); + +static void +gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *gstbasetransform_class; + GstVulkanFullScreenRenderClass *fullscreenrender_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasetransform_class = (GstBaseTransformClass *) klass; + fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass; + + gobject_class->set_property = gst_vulkan_view_convert_set_property; + gobject_class->get_property = gst_vulkan_view_convert_get_property; + + g_object_class_install_property (gobject_class, PROP_INPUT_LAYOUT, + g_param_spec_enum ("input-mode-override", + "Input Multiview Mode Override", + "Override any input information about multiview layout", + GST_TYPE_VIDEO_MULTIVIEW_MODE, + GST_VIDEO_MULTIVIEW_MODE_NONE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_FLAGS, + g_param_spec_flags ("input-flags-override", + "Input Multiview Flags Override", + "Override any input information about multiview layout flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT_LAYOUT, + g_param_spec_enum ("output-mode-override", + "Output Multiview Mode Override", + "Override automatic output mode selection for multiview layout", + GST_TYPE_VIDEO_MULTIVIEW_MODE, GST_VIDEO_MULTIVIEW_MODE_NONE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT_FLAGS, + g_param_spec_flags ("output-flags-override", + "Output Multiview Flags Override", + "Override automatic negotiation for output multiview layout flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGS, GST_VIDEO_MULTIVIEW_FLAGS_NONE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_OUTPUT_DOWNMIX_MODE, + g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output", + "Output anaglyph type to generate when downmixing to mono", + GST_TYPE_VULKAN_STEREO_DOWNMIX, DEFAULT_DOWNMIX, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", + "Filter/Video/Convert", "A Vulkan View Convert", + "Matthew Waters "); + + gst_element_class_add_static_pad_template (gstelement_class, + &gst_vulkan_sink_template); + gst_element_class_add_static_pad_template (gstelement_class, + &gst_vulkan_src_template); + + gstbasetransform_class->start = + GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_start); + gstbasetransform_class->stop = + GST_DEBUG_FUNCPTR (gst_vulkan_view_convert_stop); + gstbasetransform_class->transform_caps = + gst_vulkan_view_convert_transform_caps; + gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps; + gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps; + gstbasetransform_class->transform = gst_vulkan_view_convert_transform; + + fullscreenrender_class->render_pass_attachment_references = + gst_vulkan_view_convert_render_pass_attachment_references; + fullscreenrender_class->render_pass_attachment_descriptions = + gst_vulkan_view_convert_render_pass_attachment_descriptions; + fullscreenrender_class->descriptor_set_layout_bindings = + gst_vulkan_view_convert_descriptor_set_layout_bindings; + fullscreenrender_class->shader_create_info = + gst_vulkan_view_convert_shader_create_info; + fullscreenrender_class->push_constant_ranges = + gst_vulkan_view_convert_push_constant_ranges; +} + +static void +gst_vulkan_view_convert_init (GstVulkanViewConvert * conv) +{ + conv->downmix_mode = DEFAULT_DOWNMIX; + + conv->input_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE; + conv->input_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE; + conv->output_mode_override = GST_VIDEO_MULTIVIEW_MODE_NONE; + conv->output_flags_override = GST_VIDEO_MULTIVIEW_FLAGS_NONE; +} + +static void +gst_vulkan_view_convert_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object); + + switch (prop_id) { + case PROP_INPUT_LAYOUT: + conv->input_mode_override = g_value_get_enum (value); + break; + case PROP_INPUT_FLAGS: + conv->input_flags_override = g_value_get_flags (value); + break; + case PROP_OUTPUT_LAYOUT: + conv->output_mode_override = g_value_get_enum (value); + break; + case PROP_OUTPUT_FLAGS: + conv->output_flags_override = g_value_get_flags (value); + break; + case PROP_OUTPUT_DOWNMIX_MODE: + conv->downmix_mode = g_value_get_enum (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + GST_OBJECT_LOCK (conv); + conv->descriptor_up_to_date = FALSE; + GST_OBJECT_UNLOCK (conv); +} + +static void +gst_vulkan_view_convert_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (object); + + switch (prop_id) { + case PROP_INPUT_LAYOUT: + g_value_set_enum (value, conv->input_mode_override); + break; + case PROP_INPUT_FLAGS: + g_value_set_flags (value, conv->input_flags_override); + break; + case PROP_OUTPUT_LAYOUT: + g_value_set_enum (value, conv->output_mode_override); + break; + case PROP_OUTPUT_FLAGS: + g_value_set_flags (value, conv->output_flags_override); + break; + case PROP_OUTPUT_DOWNMIX_MODE: + g_value_set_enum (value, conv->downmix_mode); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* Function that can halve the value + * of ints, fractions, int/fraction ranges and lists of ints/fractions */ +static gboolean +_halve_value (GValue * out, const GValue * in_value) +{ + /* Fundamental fixed types first */ + if (G_VALUE_HOLDS_INT (in_value)) { + g_value_init (out, G_TYPE_INT); + g_value_set_int (out, MAX (g_value_get_int (in_value) / 2, 1)); + } else if (GST_VALUE_HOLDS_FRACTION (in_value)) { + gint num, den; + num = gst_value_get_fraction_numerator (in_value); + den = gst_value_get_fraction_denominator (in_value); + g_value_init (out, GST_TYPE_FRACTION); + /* Don't adjust 'infinite' fractions */ + if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) { + /* FIXME - could do better approximation when den > G_MAXINT/2? */ + den = den > G_MAXINT / 2 ? G_MAXINT : den * 2; + } + gst_value_set_fraction (out, num, den); + } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) { + gint range_min = gst_value_get_int_range_min (in_value); + gint range_max = gst_value_get_int_range_max (in_value); + gint range_step = gst_value_get_int_range_step (in_value); + g_value_init (out, GST_TYPE_INT_RANGE); + if (range_min != 1) + range_min = MAX (1, range_min / 2); + if (range_max != G_MAXINT) + range_max = MAX (1, range_max / 2); + gst_value_set_int_range_step (out, range_min, + range_max, MAX (1, range_step / 2)); + } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) { + GValue min_out = G_VALUE_INIT; + GValue max_out = G_VALUE_INIT; + const GValue *range_min = gst_value_get_fraction_range_min (in_value); + const GValue *range_max = gst_value_get_fraction_range_max (in_value); + _halve_value (&min_out, range_min); + _halve_value (&max_out, range_max); + g_value_init (out, GST_TYPE_FRACTION_RANGE); + gst_value_set_fraction_range (out, &min_out, &max_out); + g_value_unset (&min_out); + g_value_unset (&max_out); + } else if (GST_VALUE_HOLDS_LIST (in_value)) { + gint i; + g_value_init (out, GST_TYPE_LIST); + for (i = 0; i < gst_value_list_get_size (in_value); i++) { + const GValue *entry; + GValue tmp = G_VALUE_INIT; + + entry = gst_value_list_get_value (in_value, i); + /* Random list values might not be the right type */ + if (!_halve_value (&tmp, entry)) + goto fail; + gst_value_list_append_and_take_value (out, &tmp); + } + } else { + return FALSE; + } + + return TRUE; +fail: + g_value_unset (out); + return FALSE; +} + +static GstStructure * +_halve_structure_field (const GstStructure * in, const gchar * field_name) +{ + GstStructure *out; + const GValue *in_value = gst_structure_get_value (in, field_name); + GValue tmp = G_VALUE_INIT; + + if (G_UNLIKELY (in_value == NULL)) + return gst_structure_copy (in); /* Field doesn't exist, leave it as is */ + + if (!_halve_value (&tmp, in_value)) + return NULL; + + out = gst_structure_copy (in); + gst_structure_set_value (out, field_name, &tmp); + g_value_unset (&tmp); + + return out; +} + +/* Function that can double the value + * of ints, fractions, int/fraction ranges and lists of ints/fractions */ +static gboolean +_double_value (GValue * out, const GValue * in_value) +{ + /* Fundamental fixed types first */ + if (G_VALUE_HOLDS_INT (in_value)) { + gint n = g_value_get_int (in_value); + g_value_init (out, G_TYPE_INT); + if (n <= G_MAXINT / 2) + g_value_set_int (out, n * 2); + else + g_value_set_int (out, G_MAXINT); + } else if (GST_VALUE_HOLDS_FRACTION (in_value)) { + gint num, den; + num = gst_value_get_fraction_numerator (in_value); + den = gst_value_get_fraction_denominator (in_value); + g_value_init (out, GST_TYPE_FRACTION); + /* Don't adjust 'infinite' fractions */ + if ((num != 1 || den != 2147483647) && (num != 2147483647 || den != 1)) { + /* FIXME - could do better approximation when num > G_MAXINT/2? */ + num = num > G_MAXINT / 2 ? G_MAXINT : num * 2; + } + gst_value_set_fraction (out, num, den); + } else if (GST_VALUE_HOLDS_INT_RANGE (in_value)) { + gint range_min = gst_value_get_int_range_min (in_value); + gint range_max = gst_value_get_int_range_max (in_value); + gint range_step = gst_value_get_int_range_step (in_value); + if (range_min != 1) { + range_min = MIN (G_MAXINT / 2, range_min); + range_min *= 2; + } + if (range_max != G_MAXINT) { + range_max = MIN (G_MAXINT / 2, range_max); + range_max *= 2; + } + range_step = MIN (G_MAXINT / 2, range_step); + g_value_init (out, GST_TYPE_INT_RANGE); + gst_value_set_int_range_step (out, range_min, range_max, range_step); + } else if (GST_VALUE_HOLDS_FRACTION_RANGE (in_value)) { + GValue min_out = G_VALUE_INIT; + GValue max_out = G_VALUE_INIT; + const GValue *range_min = gst_value_get_fraction_range_min (in_value); + const GValue *range_max = gst_value_get_fraction_range_max (in_value); + _double_value (&min_out, range_min); + _double_value (&max_out, range_max); + g_value_init (out, GST_TYPE_FRACTION_RANGE); + gst_value_set_fraction_range (out, &min_out, &max_out); + g_value_unset (&min_out); + g_value_unset (&max_out); + } else if (GST_VALUE_HOLDS_LIST (in_value)) { + gint i; + g_value_init (out, GST_TYPE_LIST); + for (i = 0; i < gst_value_list_get_size (in_value); i++) { + const GValue *entry; + GValue tmp = G_VALUE_INIT; + + entry = gst_value_list_get_value (in_value, i); + /* Random list values might not be the right type */ + if (!_double_value (&tmp, entry)) + goto fail; + gst_value_list_append_and_take_value (out, &tmp); + } + } else { + return FALSE; + } + + return TRUE; +fail: + g_value_unset (out); + return FALSE; +} + +static GstStructure * +_double_structure_field (const GstStructure * in, const gchar * field_name) +{ + GstStructure *out; + const GValue *in_value = gst_structure_get_value (in, field_name); + GValue tmp = G_VALUE_INIT; + + if (G_UNLIKELY (in_value == NULL)) + return gst_structure_copy (in); /* Field doesn't exist, leave it as is */ + + if (!_double_value (&tmp, in_value)) + return NULL; + + out = gst_structure_copy (in); + gst_structure_set_value (out, field_name, &tmp); + g_value_unset (&tmp); + + return out; +} + +/* Return a copy of the caps with the requested field doubled in value/range */ +static GstCaps * +_double_caps_field (const GstCaps * in, const gchar * field_name) +{ + gint i; + GstCaps *out = gst_caps_new_empty (); + + for (i = 0; i < gst_caps_get_size (in); i++) { + const GstStructure *cur = gst_caps_get_structure (in, i); + GstCapsFeatures *f = gst_caps_get_features (in, i); + + GstStructure *res = _double_structure_field (cur, field_name); + out = + gst_caps_merge_structure_full (out, res, + f ? gst_caps_features_copy (f) : NULL); + } + + return out; +} + +/* Takes ownership of the input caps */ +static GstCaps * +_expand_par_for_half_aspect (GstCaps * in, gboolean vertical_half_aspect) +{ + guint mview_flags, mview_flags_mask; + GstCaps *out; + GstStructure *tmp; + + out = gst_caps_new_empty (); + + while (gst_caps_get_size (in) > 0) { + GstStructure *s; + GstCapsFeatures *features; + + features = gst_caps_get_features (in, 0); + if (features) + features = gst_caps_features_copy (features); + + s = gst_caps_steal_structure (in, 0); + + if (!gst_structure_get_flagset (s, "multiview-flags", &mview_flags, + &mview_flags_mask)) { + gst_caps_append_structure_full (out, s, features); + continue; + } + /* If the input doesn't care about the half-aspect flag, allow current PAR in either variant */ + if ((mview_flags_mask & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) == 0) { + gst_caps_append_structure_full (out, s, features); + continue; + } + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) { + /* No par field, dont-care the half-aspect flag */ + gst_structure_set (s, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + gst_caps_append_structure_full (out, s, features); + continue; + } + + /* Halve or double PAR base on inputs input specified. */ + + /* Append a copy with the half-aspect flag as-is */ + tmp = gst_structure_copy (s); + out = gst_caps_merge_structure_full (out, tmp, + features ? gst_caps_features_copy (features) : NULL); + + /* and then a copy inverted */ + if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) { + /* Input is half-aspect. Double/halve the PAR, clear the flag */ + if (vertical_half_aspect) + tmp = _halve_structure_field (s, "pixel-aspect-ratio"); + else + tmp = _double_structure_field (s, "pixel-aspect-ratio"); + /* Clear the flag */ + gst_structure_set (tmp, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + } else { + if (vertical_half_aspect) + tmp = _double_structure_field (s, "pixel-aspect-ratio"); + else + tmp = _halve_structure_field (s, "pixel-aspect-ratio"); + /* Set the flag */ + gst_structure_set (tmp, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + } + + out = gst_caps_merge_structure_full (out, tmp, + features ? gst_caps_features_copy (features) : NULL); + + gst_structure_free (s); + if (features) + gst_caps_features_free (features); + } + + gst_caps_unref (in); + + return out; +} + +/* If input supports top-bottom or row-interleaved, we may halve height to mono frames. + * If input supports left-right, checkerboard, quincunx or column-interleaved, + * we may halve width to mono frames. + * For output of top-bottom or row-interleaved, we may double the mono height + * For output of left-right, checkerboard, quincunx or column-interleaved, + * we may double the mono width. + * In all cases, if input has half-aspect and output does not, we may double the PAR + * And if input does *not* have half-aspect flag and output does not, we may halve the PAR + */ +static GstCaps * +_expand_structure (GstVulkanViewConvert * viewconvert, + GstCaps * out_caps, GstStructure * structure, GstCapsFeatures * features) +{ + GstCaps *expanded_caps, *tmp; + GstCaps *mono_caps; + const gchar *default_mview_mode_str = NULL; + guint mview_flags, mview_flags_mask; + const GValue *in_modes; + gint i; + + /* Empty caps to accumulate into */ + expanded_caps = gst_caps_new_empty (); + + /* First, set defaults if multiview flags are missing */ + default_mview_mode_str = + gst_video_multiview_mode_to_caps_string (GST_VIDEO_MULTIVIEW_MODE_MONO); + + mview_flags = GST_VIDEO_MULTIVIEW_FLAGS_NONE; + mview_flags_mask = GST_FLAG_SET_MASK_EXACT; + + if (!gst_structure_has_field (structure, "multiview-mode")) { + gst_structure_set (structure, + "multiview-mode", G_TYPE_STRING, default_mview_mode_str, NULL); + } + if (!gst_structure_has_field (structure, "multiview-flags")) { + gst_structure_set (structure, + "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mview_flags, + mview_flags_mask, NULL); + } else { + gst_structure_get_flagset (structure, "multiview-flags", + &mview_flags, &mview_flags_mask); + } + + in_modes = gst_structure_get_value (structure, "multiview-mode"); + mono_caps = gst_caps_new_empty (); + if (gst_value_intersect (NULL, in_modes, + gst_video_multiview_get_mono_modes ())) { + GstStructure *new_struct = gst_structure_copy (structure); + gst_structure_set_value (new_struct, "multiview-mode", + gst_video_multiview_get_mono_modes ()); + /* Half-aspect makes no sense for mono or unpacked, get rid of it */ + if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) { + gst_structure_set (new_struct, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + } + gst_caps_append_structure_full (mono_caps, new_struct, + features ? gst_caps_features_copy (features) : NULL); + } + if (gst_value_intersect (NULL, in_modes, + gst_video_multiview_get_unpacked_modes ())) { + GstStructure *new_struct = gst_structure_copy (structure); + + gst_structure_set_value (new_struct, "multiview-mode", + gst_video_multiview_get_mono_modes ()); + + /* Half-aspect makes no sense for mono or unpacked, get rid of it */ + if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) { + gst_structure_set (new_struct, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + } + gst_caps_append_structure_full (mono_caps, new_struct, + features ? gst_caps_features_copy (features) : NULL); + } + + if (gst_value_intersect (NULL, in_modes, + gst_video_multiview_get_doubled_height_modes ())) { + /* Append mono formats with height halved */ + GstStructure *new_struct = _halve_structure_field (structure, "height"); + gst_structure_set_value (new_struct, "multiview-mode", + gst_video_multiview_get_mono_modes ()); + /* Normalise the half-aspect flag away */ + if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) { + GstStructure *s = + _halve_structure_field (new_struct, "pixel-aspect-ratio"); + gst_structure_set (structure, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + gst_structure_free (new_struct); + new_struct = s; + } + mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct, + features ? gst_caps_features_copy (features) : NULL); + } + if (gst_value_intersect (NULL, in_modes, + gst_video_multiview_get_doubled_width_modes ())) { + /* Append mono formats with width halved */ + GstStructure *new_struct = _halve_structure_field (structure, "width"); + gst_structure_set_value (new_struct, "multiview-mode", + gst_video_multiview_get_mono_modes ()); + /* Normalise the half-aspect flag away */ + if (mview_flags & GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT) { + GstStructure *s = + _double_structure_field (new_struct, "pixel-aspect-ratio"); + gst_structure_set (structure, "multiview-flags", + GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags & ~GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, + mview_flags_mask | GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT, NULL); + gst_structure_free (new_struct); + new_struct = s; + } + mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct, + features ? gst_caps_features_copy (features) : NULL); + } + if (gst_value_intersect (NULL, in_modes, + gst_video_multiview_get_doubled_size_modes ())) { + /* Append checkerboard/doubled size formats with width & height halved */ + GstStructure *new_struct_w = _halve_structure_field (structure, "width"); + GstStructure *new_struct_wh = + _halve_structure_field (new_struct_w, "height"); + gst_structure_free (new_struct_w); + gst_structure_set_value (new_struct_wh, "multiview-mode", + gst_video_multiview_get_mono_modes ()); + mono_caps = gst_caps_merge_structure_full (mono_caps, new_struct_wh, + features ? gst_caps_features_copy (features) : NULL); + } + + /* Everything is normalised now, unset the flags we can change */ + /* Remove the views field, as these are all 'mono' modes + * Need to do this before we expand caps back out to frame packed modes */ + for (i = 0; i < gst_caps_get_size (mono_caps); i++) { + GstStructure *s = gst_caps_get_structure (mono_caps, i); + gst_structure_remove_fields (s, "views", NULL); + if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags, + &mview_flags_mask)) { + /* Preserve only the half-aspect and mixed-mono flags, for now. + * The rest we can change */ + mview_flags_mask &= + (GST_VIDEO_MULTIVIEW_FLAGS_HALF_ASPECT | + GST_VIDEO_MULTIVIEW_FLAGS_MIXED_MONO); + gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags, mview_flags_mask, NULL); + } + } + + GST_TRACE_OBJECT (viewconvert, + "Collected single-view caps %" GST_PTR_FORMAT, mono_caps); + /* Put unpacked and mono modes first. We don't care about flags. Clear them */ + tmp = gst_caps_copy (mono_caps); + for (i = 0; i < gst_caps_get_size (tmp); i++) { + GstStructure *s = gst_caps_get_structure (tmp, i); + gst_structure_remove_fields (s, "views", NULL); + if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags, + &mview_flags_mask)) { + /* We can change any flags for mono modes - half-aspect and mixed-mono have no meaning */ + mview_flags_mask = 0; + gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags, mview_flags_mask, NULL); + } + } + expanded_caps = gst_caps_merge (expanded_caps, tmp); + + /* Unpacked output modes have 2 views, for now */ + tmp = gst_caps_copy (mono_caps); + gst_caps_set_value (tmp, "multiview-mode", + gst_video_multiview_get_unpacked_modes ()); + for (i = 0; i < gst_caps_get_size (tmp); i++) { + GstStructure *s = gst_caps_get_structure (tmp, i); + gst_structure_set (s, "views", G_TYPE_INT, 2, NULL); + if (gst_structure_get_flagset (s, "multiview-flags", &mview_flags, + &mview_flags_mask)) { + /* We can change any flags for unpacked modes - half-aspect and mixed-mono have no meaning */ + mview_flags_mask = 0; + gst_structure_set (s, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, + mview_flags, mview_flags_mask, NULL); + } + } + expanded_caps = gst_caps_merge (expanded_caps, tmp); + + /* Double height output modes */ + tmp = _double_caps_field (mono_caps, "height"); + gst_caps_set_value (tmp, "multiview-mode", + gst_video_multiview_get_doubled_height_modes ()); + tmp = _expand_par_for_half_aspect (tmp, TRUE); + + expanded_caps = gst_caps_merge (expanded_caps, tmp); + + /* Double width output modes */ + tmp = _double_caps_field (mono_caps, "width"); + gst_caps_set_value (tmp, "multiview-mode", + gst_video_multiview_get_doubled_width_modes ()); + tmp = _expand_par_for_half_aspect (tmp, FALSE); + + expanded_caps = gst_caps_merge (expanded_caps, tmp); + + /* Double size output modes */ + { + GstCaps *tmp_w = _double_caps_field (mono_caps, "width"); + tmp = _double_caps_field (tmp_w, "height"); + gst_caps_unref (tmp_w); + gst_caps_set_value (tmp, "multiview-mode", + gst_video_multiview_get_doubled_size_modes ()); + expanded_caps = gst_caps_merge (expanded_caps, tmp); + } + + /* We're done with the mono caps now */ + gst_caps_unref (mono_caps); + + GST_TRACE_OBJECT (viewconvert, + "expanded transform caps now %" GST_PTR_FORMAT, expanded_caps); + + if (gst_caps_is_empty (expanded_caps)) { + gst_caps_unref (expanded_caps); + return out_caps; + } + /* Really, we can rescale - so at this point we can append full-range + * height/width/PAR as an unpreferred final option. */ +/* tmp = gst_caps_copy (expanded_caps); + gst_caps_set_simple (tmp, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); +*/ + out_caps = gst_caps_merge (out_caps, expanded_caps); +/* out_caps = gst_caps_merge (out_caps, tmp);*/ + return out_caps; +} + +static GstCaps * +_intersect_with_mview_mode (GstCaps * caps, + GstVideoMultiviewMode mode, GstVideoMultiviewFlags flags) +{ + GstCaps *filter, *result; + const gchar *caps_str; + + caps_str = gst_video_multiview_mode_to_caps_string (mode); + + filter = gst_caps_new_simple ("video/x-raw", + "multiview-mode", G_TYPE_STRING, + caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags, + GST_FLAG_SET_MASK_EXACT, NULL); + + if (mode == GST_VIDEO_MULTIVIEW_MODE_SEPARATED || + mode == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) + gst_caps_set_simple (filter, "views", G_TYPE_INT, 2, NULL); + gst_caps_set_features (filter, 0, gst_caps_features_new_any ()); + + GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT + " with caps %" GST_PTR_FORMAT, caps, filter); + + result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (filter); + return result; +} + +static GstCaps * +_intersect_with_mview_modes (GstCaps * caps, const GValue * modes) +{ + GstCaps *filter, *result; + + filter = gst_caps_new_empty_simple ("video/x-raw"); + + gst_caps_set_value (filter, "multiview-mode", modes); + gst_caps_set_features (filter, 0, gst_caps_features_new_any ()); + + GST_DEBUG ("Intersecting target caps %" GST_PTR_FORMAT + " with caps %" GST_PTR_FORMAT, caps, filter); + + result = gst_caps_intersect_full (caps, filter, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (filter); + return result; +} + +static GstCaps * +gst_vulkan_view_convert_transform_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt); + GstCaps *base_caps = + gst_static_pad_template_get_caps (&gst_vulkan_sink_template); + GstCaps *out_caps, *tmp_caps; + gint i; + + GST_DEBUG_OBJECT (viewconvert, "Direction %s " + "input caps %" GST_PTR_FORMAT " filter %" GST_PTR_FORMAT, + direction == GST_PAD_SINK ? "sink" : "src", caps, filter); + + /* We can only process VulkanImage caps, start from that */ + caps = gst_caps_intersect (caps, base_caps); + gst_caps_unref (base_caps); + + /* Change input/output to the formats we can convert to/from, + * but keep the original caps at the start - we will always prefer + * passthrough */ + if (direction == GST_PAD_SINK) { + out_caps = gst_caps_copy (caps); + if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) { + GstVideoMultiviewMode mode = viewconvert->input_mode_override; + GstVideoMultiviewFlags flags = viewconvert->input_flags_override; + + const gchar *caps_str = gst_video_multiview_mode_to_caps_string (mode); + /* Coerce the input caps before transforming, so the sizes come out right */ + gst_caps_set_simple (out_caps, "multiview-mode", G_TYPE_STRING, + caps_str, "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, flags, + GST_FLAG_SET_MASK_EXACT, NULL); + } + } else { + out_caps = gst_caps_new_empty (); + } + + for (i = 0; i < gst_caps_get_size (caps); i++) { + GstStructure *structure = gst_caps_get_structure (caps, i); + GstCapsFeatures *features = gst_caps_get_features (caps, i); + out_caps = _expand_structure (viewconvert, out_caps, structure, features); + } + + if (gst_caps_is_empty (out_caps)) + goto out; + + /* If we have an output mode override, limit things to that */ + if (direction == GST_PAD_SINK && + viewconvert->output_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) { + + tmp_caps = _intersect_with_mview_mode (out_caps, + viewconvert->output_mode_override, viewconvert->output_flags_override); + + gst_caps_unref (out_caps); + out_caps = tmp_caps; + } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) { + /* Prepend a copy of our preferred input caps in case the peer + * can handle them */ + tmp_caps = _intersect_with_mview_mode (out_caps, + viewconvert->input_mode_override, viewconvert->input_flags_override); + out_caps = gst_caps_merge (out_caps, tmp_caps); + } + if (direction == GST_PAD_SRC) { + GstStructure *s; + /* When generating input caps, we also need a copy of the mono caps + * without multiview-mode or flags for backwards compat, at the end */ + tmp_caps = _intersect_with_mview_mode (caps, + GST_VIDEO_MULTIVIEW_MODE_MONO, GST_VIDEO_MULTIVIEW_FLAGS_NONE); + if (!gst_caps_is_empty (tmp_caps)) { + s = gst_caps_get_structure (tmp_caps, 0); + gst_structure_remove_fields (s, "multiview-mode", "multiview-flags", + NULL); + out_caps = gst_caps_merge (out_caps, tmp_caps); + } else + gst_caps_unref (tmp_caps); + } +out: + gst_caps_unref (caps); + + GST_DEBUG_OBJECT (viewconvert, "Have caps %" GST_PTR_FORMAT + " filtering with caps %" GST_PTR_FORMAT, out_caps, filter); + + if (filter) { + GstCaps *tmp = + gst_caps_intersect_full (filter, out_caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (out_caps); + out_caps = tmp; + } + + GST_DEBUG_OBJECT (viewconvert, "Returning caps %" GST_PTR_FORMAT, out_caps); + return out_caps; +} + +static GstCaps * +fixate_size (GstVulkanViewConvert * viewconvert, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) +{ + GstStructure *ins, *outs; + const GValue *from_par, *to_par; + GValue fpar = { 0, }, tpar = { + 0,}; + + othercaps = gst_caps_make_writable (othercaps); + othercaps = gst_caps_truncate (othercaps); + + GST_DEBUG_OBJECT (viewconvert, "trying to fixate othercaps %" GST_PTR_FORMAT + " based on caps %" GST_PTR_FORMAT, othercaps, caps); + + ins = gst_caps_get_structure (caps, 0); + outs = gst_caps_get_structure (othercaps, 0); + + from_par = gst_structure_get_value (ins, "pixel-aspect-ratio"); + to_par = gst_structure_get_value (outs, "pixel-aspect-ratio"); + + /* If we're fixating from the sinkpad we always set the PAR and + * assume that missing PAR on the sinkpad means 1/1 and + * missing PAR on the srcpad means undefined + */ + if (direction == GST_PAD_SINK) { + if (!from_par) { + g_value_init (&fpar, GST_TYPE_FRACTION); + gst_value_set_fraction (&fpar, 1, 1); + from_par = &fpar; + } + if (!to_par) { + g_value_init (&tpar, GST_TYPE_FRACTION); + gst_value_set_fraction (&tpar, 1, 1); + to_par = &tpar; + } + } else { + if (!to_par) { + g_value_init (&tpar, GST_TYPE_FRACTION); + gst_value_set_fraction (&tpar, 1, 1); + to_par = &tpar; + + gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + } + if (!from_par) { + g_value_init (&fpar, GST_TYPE_FRACTION); + gst_value_set_fraction (&fpar, 1, 1); + from_par = &fpar; + } + } + + /* we have both PAR but they might not be fixated */ + { + gint from_w, from_h, from_par_n, from_par_d, to_par_n, to_par_d; + gint w = 0, h = 0; + gint from_dar_n, from_dar_d; + gint num, den; + + /* from_par should be fixed */ + g_return_val_if_fail (gst_value_is_fixed (from_par), othercaps); + + from_par_n = gst_value_get_fraction_numerator (from_par); + from_par_d = gst_value_get_fraction_denominator (from_par); + + gst_structure_get_int (ins, "width", &from_w); + gst_structure_get_int (ins, "height", &from_h); + + gst_structure_get_int (outs, "width", &w); + gst_structure_get_int (outs, "height", &h); + + /* if both width and height are already fixed, we can't do anything + * about it anymore */ + if (w && h) { + GST_DEBUG_OBJECT (viewconvert, + "dimensions already set to %dx%d, not fixating", w, h); + if (!gst_value_is_fixed (to_par)) { + GST_DEBUG_OBJECT (viewconvert, "fixating to_par to %dx%d", 1, 1); + if (gst_structure_has_field (outs, "pixel-aspect-ratio")) + gst_structure_fixate_field_nearest_fraction (outs, + "pixel-aspect-ratio", 1, 1); + } + goto done; + } + + /* Calculate input DAR */ + if (!gst_util_fraction_multiply (from_w, from_h, from_par_n, from_par_d, + &from_dar_n, &from_dar_d)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + goto done; + } + + GST_DEBUG_OBJECT (viewconvert, "Input DAR is %d/%d", from_dar_n, + from_dar_d); + + /* If either width or height are fixed there's not much we + * can do either except choosing a height or width and PAR + * that matches the DAR as good as possible + */ + if (h) { + gint num, den; + + GST_DEBUG_OBJECT (viewconvert, "height is fixed (%d)", h); + + if (!gst_value_is_fixed (to_par)) { + /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1, + * so let's make it so ... + * especially if following code assumes fixed */ + GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1"); + gst_structure_fixate_field_nearest_fraction (outs, + "pixel-aspect-ratio", 1, 1); + to_par = gst_structure_get_value (outs, "pixel-aspect-ratio"); + } + + /* PAR is fixed, choose the height that is nearest to the + * height with the same DAR */ + to_par_n = gst_value_get_fraction_numerator (to_par); + to_par_d = gst_value_get_fraction_denominator (to_par); + + GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d); + + if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d, + to_par_n, &num, &den)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + goto done; + } + + w = (guint) gst_util_uint64_scale_int (h, num, den); + gst_structure_fixate_field_nearest_int (outs, "width", w); + + goto done; + } else if (w) { + gint num, den; + + GST_DEBUG_OBJECT (viewconvert, "width is fixed (%d)", w); + + if (!gst_value_is_fixed (to_par)) { + /* (shortcut) copy-paste (??) of videoscale seems to aim for 1/1, + * so let's make it so ... + * especially if following code assumes fixed */ + GST_DEBUG_OBJECT (viewconvert, "fixating to_par to 1x1"); + gst_structure_fixate_field_nearest_fraction (outs, + "pixel-aspect-ratio", 1, 1); + to_par = gst_structure_get_value (outs, "pixel-aspect-ratio"); + } + + /* PAR is fixed, choose the height that is nearest to the + * height with the same DAR */ + to_par_n = gst_value_get_fraction_numerator (to_par); + to_par_d = gst_value_get_fraction_denominator (to_par); + + GST_DEBUG_OBJECT (viewconvert, "PAR is fixed %d/%d", to_par_n, to_par_d); + + if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_d, + to_par_n, &num, &den)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + goto done; + } + + h = (guint) gst_util_uint64_scale_int (w, den, num); + gst_structure_fixate_field_nearest_int (outs, "height", h); + + goto done; + } else if (gst_value_is_fixed (to_par)) { + GstStructure *tmp; + gint set_h, set_w, f_h, f_w; + + to_par_n = gst_value_get_fraction_numerator (to_par); + to_par_d = gst_value_get_fraction_denominator (to_par); + + /* Calculate scale factor for the PAR change */ + if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, to_par_n, + to_par_d, &num, &den)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + goto done; + } + + /* Try to keep the input height */ + tmp = gst_structure_copy (outs); + gst_structure_fixate_field_nearest_int (tmp, "height", from_h); + gst_structure_get_int (tmp, "height", &set_h); + + /* This might have failed but try to scale the width + * to keep the DAR nonetheless */ + w = (guint) gst_util_uint64_scale_int (set_h, num, den); + gst_structure_fixate_field_nearest_int (tmp, "width", w); + gst_structure_get_int (tmp, "width", &set_w); + gst_structure_free (tmp); + + /* We kept the DAR and the height is nearest to the original height */ + if (set_w == w) { + gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", + G_TYPE_INT, set_h, NULL); + goto done; + } + + f_h = set_h; + f_w = set_w; + + /* If the former failed, try to keep the input width at least */ + tmp = gst_structure_copy (outs); + gst_structure_fixate_field_nearest_int (tmp, "width", from_w); + gst_structure_get_int (tmp, "width", &set_w); + + /* This might have failed but try to scale the width + * to keep the DAR nonetheless */ + h = (guint) gst_util_uint64_scale_int (set_w, den, num); + gst_structure_fixate_field_nearest_int (tmp, "height", h); + gst_structure_get_int (tmp, "height", &set_h); + gst_structure_free (tmp); + + /* We kept the DAR and the width is nearest to the original width */ + if (set_h == h) { + gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", + G_TYPE_INT, set_h, NULL); + goto done; + } + + /* If all this failed, keep the height that was nearest to the orignal + * height and the nearest possible width. This changes the DAR but + * there's not much else to do here. + */ + gst_structure_set (outs, "width", G_TYPE_INT, f_w, "height", G_TYPE_INT, + f_h, NULL); + goto done; + } else { + GstStructure *tmp; + gint set_h, set_w, set_par_n, set_par_d, tmp2; + + /* width, height and PAR are not fixed */ + + /* First try to keep the height and width as good as possible + * and scale PAR */ + tmp = gst_structure_copy (outs); + gst_structure_fixate_field_nearest_int (tmp, "height", from_h); + gst_structure_get_int (tmp, "height", &set_h); + gst_structure_fixate_field_nearest_int (tmp, "width", from_w); + gst_structure_get_int (tmp, "width", &set_w); + + if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_h, set_w, + &to_par_n, &to_par_d)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + gst_structure_free (tmp); + goto done; + } + + if (!gst_structure_has_field (tmp, "pixel-aspect-ratio")) + gst_structure_set_value (tmp, "pixel-aspect-ratio", to_par); + gst_structure_fixate_field_nearest_fraction (tmp, "pixel-aspect-ratio", + to_par_n, to_par_d); + gst_structure_get_fraction (tmp, "pixel-aspect-ratio", &set_par_n, + &set_par_d); + gst_structure_free (tmp); + + if (set_par_n == to_par_n && set_par_d == to_par_d) { + gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", + G_TYPE_INT, set_h, NULL); + + if (gst_structure_has_field (outs, "pixel-aspect-ratio") || + set_par_n != set_par_d) + gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, + set_par_n, set_par_d, NULL); + goto done; + } + + /* Otherwise try to scale width to keep the DAR with the set + * PAR and height */ + if (!gst_util_fraction_multiply (from_dar_n, from_dar_d, set_par_d, + set_par_n, &num, &den)) { + GST_ELEMENT_ERROR (viewconvert, CORE, NEGOTIATION, (NULL), + ("Error calculating the output scaled size - integer overflow")); + goto done; + } + + w = (guint) gst_util_uint64_scale_int (set_h, num, den); + tmp = gst_structure_copy (outs); + gst_structure_fixate_field_nearest_int (tmp, "width", w); + gst_structure_get_int (tmp, "width", &tmp2); + gst_structure_free (tmp); + + if (tmp2 == w) { + gst_structure_set (outs, "width", G_TYPE_INT, tmp2, "height", + G_TYPE_INT, set_h, NULL); + if (gst_structure_has_field (outs, "pixel-aspect-ratio") || + set_par_n != set_par_d) + gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, + set_par_n, set_par_d, NULL); + goto done; + } + + /* ... or try the same with the height */ + h = (guint) gst_util_uint64_scale_int (set_w, den, num); + tmp = gst_structure_copy (outs); + gst_structure_fixate_field_nearest_int (tmp, "height", h); + gst_structure_get_int (tmp, "height", &tmp2); + gst_structure_free (tmp); + + if (tmp2 == h) { + gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", + G_TYPE_INT, tmp2, NULL); + if (gst_structure_has_field (outs, "pixel-aspect-ratio") || + set_par_n != set_par_d) + gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, + set_par_n, set_par_d, NULL); + goto done; + } + + /* If all fails we can't keep the DAR and take the nearest values + * for everything from the first try */ + gst_structure_set (outs, "width", G_TYPE_INT, set_w, "height", + G_TYPE_INT, set_h, NULL); + if (gst_structure_has_field (outs, "pixel-aspect-ratio") || + set_par_n != set_par_d) + gst_structure_set (outs, "pixel-aspect-ratio", GST_TYPE_FRACTION, + set_par_n, set_par_d, NULL); + } + } + +done: + othercaps = gst_caps_fixate (othercaps); + + GST_DEBUG_OBJECT (viewconvert, "fixated othercaps to %" GST_PTR_FORMAT, + othercaps); + + if (from_par == &fpar) + g_value_unset (&fpar); + if (to_par == &tpar) + g_value_unset (&tpar); + + return othercaps; +} + +static GstCaps * +gst_vulkan_view_convert_fixate_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * othercaps) +{ + GstVulkanViewConvert *viewconvert = GST_VULKAN_VIEW_CONVERT (bt); + GstVideoMultiviewMode mode = viewconvert->output_mode_override; + GstVideoMultiviewFlags flags = viewconvert->output_flags_override; + GstCaps *tmp; + + othercaps = gst_caps_make_writable (othercaps); + GST_LOG_OBJECT (viewconvert, "dir %s fixating %" GST_PTR_FORMAT + " against caps %" GST_PTR_FORMAT, + direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps); + + if (direction == GST_PAD_SINK) { + if (mode != GST_VIDEO_MULTIVIEW_MODE_NONE) { + /* We have a requested output mode and are fixating source caps, try and enforce it */ + GST_DEBUG_OBJECT (bt, "fixating multiview mode using the configured " + "output override mode 0x%x and flags 0x%x", mode, flags); + tmp = _intersect_with_mview_mode (othercaps, mode, flags); + gst_caps_unref (othercaps); + othercaps = tmp; + } else { + /* See if we can do passthrough */ + GstVideoInfo info; + + if (gst_video_info_from_caps (&info, caps)) { + GstVideoMultiviewMode mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&info); + GstVideoMultiviewFlags flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&info); + + if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) { + mode = viewconvert->input_mode_override; + flags = viewconvert->input_flags_override; + } + + tmp = _intersect_with_mview_mode (othercaps, mode, flags); + if (gst_caps_is_empty (tmp)) { + /* Nope, we can't pass our input caps downstream */ + gst_caps_unref (tmp); + } else { + GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode " + "using the input override mode 0x%x and flags 0x%x", mode, flags); + gst_caps_unref (othercaps); + othercaps = tmp; + goto done; + } + } + + /* Prefer an unpacked mode for output */ + tmp = + _intersect_with_mview_modes (othercaps, + gst_video_multiview_get_unpacked_modes ()); + if (!gst_caps_is_empty (tmp)) { + GST_DEBUG_OBJECT (bt, "preferring an unpacked multiview mode"); + gst_caps_unref (othercaps); + othercaps = tmp; + } else { + gst_caps_unref (tmp); + } + } + } else if (viewconvert->input_mode_override != GST_VIDEO_MULTIVIEW_MODE_NONE) { + /* See if we can coerce the caps into matching input mode/flags, + * in case it doesn't care at all, but allow it not to too */ + mode = viewconvert->input_mode_override; + flags = viewconvert->input_flags_override; + tmp = _intersect_with_mview_mode (othercaps, mode, flags); + if (gst_caps_is_empty (tmp)) { + /* Nope, we can pass our input caps downstream */ + gst_caps_unref (tmp); + } else { + GST_DEBUG_OBJECT (bt, "can configure a passthrough multiview mode " + "using the input override mode 0x%x and flags 0x%x", mode, flags); + gst_caps_unref (othercaps); + othercaps = tmp; + } + } + +done: + othercaps = fixate_size (viewconvert, direction, caps, othercaps); + othercaps = gst_caps_fixate (othercaps); + + GST_DEBUG_OBJECT (viewconvert, "dir %s fixated to %" GST_PTR_FORMAT + " against caps %" GST_PTR_FORMAT, + direction == GST_PAD_SINK ? "sink" : "src", othercaps, caps); + return othercaps; +} + +static void +destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data) +{ + VkPipelineShaderStageCreateInfo *info = data; + int i; + + for (i = 0; i < render->n_shader_stages; i++) { + vkDestroyShaderModule (render->device->device, info[i].module, NULL); + } + + g_free (info); +} + +static void +gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render) +{ + VkShaderModule vert_module, frag_module; + + vert_module = + _vk_create_shader (render->device, identity_vert, identity_vert_size, + NULL); + frag_module = + _vk_create_shader (render->device, view_convert_frag, + view_convert_frag_size, NULL); + + render->n_shader_stages = 2; + render->shader_create_info = + g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages); + render->destroy_shader_create_info = destroy_shader_create_info; + + /* *INDENT-OFF* */ + render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = NULL, + .stage = VK_SHADER_STAGE_VERTEX_BIT, + .module = vert_module, + .pName = "main" + }; + + render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, + .pNext = NULL, + .stage = VK_SHADER_STAGE_FRAGMENT_BIT, + .module = frag_module, + .pName = "main" + }; + /* *INDENT-ON* */ +} + +static VkPushConstantRange * +gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender * + render, guint * n_constants) +{ + *n_constants = 0; + return NULL; +} + +static VkDescriptorSetLayoutBinding + * gst_vulkan_view_convert_descriptor_set_layout_bindings + (GstVulkanFullScreenRender * render, guint * n_bindings) +{ + VkDescriptorSetLayoutBinding *bindings; + guint i; + + *n_bindings = 0; + *n_bindings += GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; + *n_bindings += 1; /* uniform binding */ + bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) { + /* *INDENT-OFF* */ + bindings[i] = (VkDescriptorSetLayoutBinding) { + .binding = i, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + i++; + bindings[i] = (VkDescriptorSetLayoutBinding) { + .binding = i, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + /* *INDENT-ON* */ + } + /* *INDENT-OFF* */ + bindings[i] = (VkDescriptorSetLayoutBinding) { + .binding = i, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + }; + /* *INDENT-ON* */ + i++; + + g_assert (i == *n_bindings); + + return bindings; +} + +static VkAttachmentReference + * gst_vulkan_view_convert_render_pass_attachment_references + (GstVulkanFullScreenRender * render, guint * n_attachments) +{ + VkAttachmentReference *attachments; + int i; + + *n_attachments = GST_VIDEO_INFO_N_PLANES (&render->out_info); + attachments = g_new0 (VkAttachmentReference, *n_attachments); + + for (i = 0; i < *n_attachments; i++) { + /* *INDENT-OFF* */ + attachments[i] = (VkAttachmentReference) { + .attachment = i, + .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL + }; + /* *INDENT-ON* */ + } + + return attachments; +} + +static VkAttachmentDescription + * gst_vulkan_view_convert_render_pass_attachment_descriptions + (GstVulkanFullScreenRender * render, guint * n_descriptions) +{ + VkAttachmentDescription *color_attachments; + int i; + + *n_descriptions = GST_VIDEO_INFO_N_PLANES (&render->out_info); + color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions); + for (i = 0; i < *n_descriptions; i++) { + /* *INDENT-OFF* */ + color_attachments[i] = (VkAttachmentDescription) { + .format = gst_vulkan_format_from_video_format (GST_VIDEO_INFO_FORMAT (&render->out_info), i), + .samples = VK_SAMPLE_COUNT_1_BIT, + .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, + .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, + /* FIXME: share this between elements to avoid pipeline barriers */ + .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL + }; + /* *INDENT-ON* */ + } + + return color_attachments; +} + +static VkSampler +_create_sampler (GstVulkanViewConvert * conv) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + /* *INDENT-OFF* */ + VkSamplerCreateInfo samplerInfo = { + .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, + .magFilter = VK_FILTER_NEAREST, + .minFilter = VK_FILTER_NEAREST, + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER, + .anisotropyEnable = VK_FALSE, + .maxAnisotropy = 1, + .borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK, + .unnormalizedCoordinates = VK_FALSE, + .compareEnable = VK_FALSE, + .compareOp = VK_COMPARE_OP_ALWAYS, + .mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR, + .mipLodBias = 0.0f, + .minLod = 0.0f, + .maxLod = 0.0f + }; + /* *INDENT-ON* */ + GError *error = NULL; + VkSampler sampler; + VkResult err; + + err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { + g_clear_error (&error); + return NULL; + } + + return sampler; +} + +static gboolean +gst_vulkan_view_convert_start (GstBaseTransform * bt) +{ + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); + + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) + return FALSE; + + if (!(conv->sampler = _create_sampler (conv))) + return FALSE; + + return TRUE; +} + +static VkDescriptorPool +_create_descriptor_pool (GstVulkanViewConvert * conv) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + /* *INDENT-OFF* */ + VkDescriptorPoolSize pool_sizes[] = { + { + .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2, + }, + { + .type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, + .descriptorCount = 1 + }, + }; + + VkDescriptorPoolCreateInfo pool_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, + .pNext = NULL, + .poolSizeCount = G_N_ELEMENTS (pool_sizes), + .pPoolSizes = pool_sizes, + .maxSets = 1 + }; + /* *INDENT-ON* */ + VkDescriptorPool pool; + GError *error = NULL; + VkResult err; + + err = + vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { + GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return pool; +} + +static VkDescriptorSet +_create_descriptor_set (GstVulkanViewConvert * conv) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + /* *INDENT-OFF* */ + VkDescriptorSetAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, + .pNext = NULL, + .descriptorPool = conv->descriptor_pool, + .descriptorSetCount = 1, + .pSetLayouts = &render->descriptor_set_layout + }; + /* *INDENT-ON* */ + VkDescriptorSet descriptor; + GError *error = NULL; + VkResult err; + + err = + vkAllocateDescriptorSets (render->device->device, &alloc_info, + &descriptor); + if (gst_vulkan_error_to_g_error (err, &error, "vkAllocateDescriptorSets") < 0) { + GST_ERROR_OBJECT (conv, "Failed to allocate descriptor: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return descriptor; +} + +static gboolean +_create_uniform_buffer (GstVulkanViewConvert * conv) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + conv->uniform = + gst_vulkan_buffer_memory_alloc (render->device, VK_FORMAT_R8_UNORM, + sizeof (struct ViewUpdate), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + + return TRUE; +} + +static gboolean +gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps, + GstCaps * out_caps) +{ + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + if (render->last_fence) { + if (conv->descriptor_pool) + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref + (render->last_fence), conv->descriptor_pool)); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->uniform) + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref + (render->last_fence), (GstMiniObject *) conv->uniform)); + conv->uniform = NULL; + } else { + if (conv->descriptor_pool) + vkDestroyDescriptorPool (render->device->device, + conv->descriptor_pool, NULL); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->uniform) + gst_memory_unref (conv->uniform); + conv->uniform = NULL; + } + + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, + out_caps)) + return FALSE; + + if (!(conv->descriptor_pool = _create_descriptor_pool (conv))) + return FALSE; + if (!(conv->descriptor_set = _create_descriptor_set (conv))) + return FALSE; + + if (!_create_uniform_buffer (conv)) + return FALSE; + + return TRUE; +} + +static gboolean +gst_vulkan_view_convert_stop (GstBaseTransform * bt) +{ + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + conv->descriptor_up_to_date = FALSE; + + if (render->device) { + if (render->last_fence) { + if (conv->descriptor_pool) + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref + (render->last_fence), conv->descriptor_pool)); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->sampler) + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_free_sampler (gst_vulkan_fence_ref + (render->last_fence), conv->sampler)); + conv->sampler = NULL; + if (conv->uniform) + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref + (render->last_fence), (GstMiniObject *) conv->uniform)); + conv->uniform = NULL; + } else { + if (conv->descriptor_pool) + vkDestroyDescriptorPool (render->device->device, + conv->descriptor_pool, NULL); + conv->descriptor_set = NULL; + conv->descriptor_pool = NULL; + if (conv->sampler) + vkDestroySampler (render->device->device, conv->sampler, NULL); + conv->sampler = NULL; + if (conv->uniform) + gst_memory_unref (conv->uniform); + conv->uniform = NULL; + } + } + + if (conv->cmd_pool) + gst_object_unref (conv->cmd_pool); + conv->cmd_pool = NULL; + + return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt); +} + +static VkFramebuffer +_create_framebuffer (GstVulkanViewConvert * conv, guint n_views, + VkImageView * views) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); + + /* *INDENT-OFF* */ + VkFramebufferCreateInfo framebuffer_info = { + .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, + .pNext = NULL, + .renderPass = render->render_pass, + .attachmentCount = n_views, + .pAttachments = views, + .width = GST_VIDEO_INFO_WIDTH (&render->out_info), + .height = GST_VIDEO_INFO_HEIGHT (&render->out_info), + .layers = 1 + }; + /* *INDENT-ON* */ + VkFramebuffer framebuffer; + GError *error = NULL; + VkResult err; + + err = + vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL, + &framebuffer); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { + GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return framebuffer; +} + +static GstFlowReturn +gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, + GstBuffer * outbuf) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); + GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageMemory *out_img_mems[4] = { NULL, }; + GstVulkanFence *fence = NULL; + VkFramebuffer framebuffer; + VkCommandBuffer cmd; + GError *error = NULL; + VkResult err; + int i; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { + GstMemory *mem = gst_buffer_peek_memory (inbuf, i); + if (!gst_is_vulkan_image_memory (mem)) { + g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Input memory must be a GstVulkanImageMemory"); + goto error; + } + in_img_mems[i] = (GstVulkanImageMemory *) mem; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { + GstMemory *mem = gst_buffer_peek_memory (outbuf, i); + if (!gst_is_vulkan_image_memory (mem)) { + g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Output memory must be a GstVulkanImageMemory"); + goto error; + } + out_img_mems[i] = (GstVulkanImageMemory *) mem; + } + + if (!conv->cmd_pool) { + if (!(conv->cmd_pool = + gst_vulkan_queue_create_command_pool (render->queue, &error))) + goto error; + } + + if (!(cmd = gst_vulkan_command_pool_create (conv->cmd_pool, &error))) + goto error; + + fence = gst_vulkan_fence_new (render->device, 0, &error); + if (!fence) + goto error; + + { + VkCommandBufferBeginInfo cmd_buf_info = { 0, }; + + /* *INDENT-OFF* */ + cmd_buf_info = (VkCommandBufferBeginInfo) { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, + .pNext = NULL, + .flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT, + .pInheritanceInfo = NULL + }; + /* *INDENT-ON* */ + + err = vkBeginCommandBuffer (cmd, &cmd_buf_info); + if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0) + goto error; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { + /* *INDENT-OFF* */ + VkImageMemoryBarrier in_image_memory_barrier = { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = NULL, + .srcAccessMask = in_img_mems[i]->barrier.parent.access_flags, + .dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT, + .oldLayout = in_img_mems[i]->barrier.image_layout, + .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + /* FIXME: implement exclusive transfers */ + .srcQueueFamilyIndex = 0, + .dstQueueFamilyIndex = 0, + .image = in_img_mems[i]->image, + .subresourceRange = in_img_mems[i]->barrier.subresource_range + }; + /* *INDENT-ON* */ + + vkCmdPipelineBarrier (cmd, in_img_mems[i]->barrier.parent.pipeline_stages, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, + &in_image_memory_barrier); + + in_img_mems[i]->barrier.parent.pipeline_stages = + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + in_img_mems[i]->barrier.parent.access_flags = + in_image_memory_barrier.dstAccessMask; + in_img_mems[i]->barrier.image_layout = in_image_memory_barrier.newLayout; + } + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { + VkImageMemoryBarrier out_image_memory_barrier; + + /* *INDENT-OFF* */ + out_image_memory_barrier = (VkImageMemoryBarrier) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = NULL, + .srcAccessMask = out_img_mems[i]->barrier.parent.access_flags, + .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .oldLayout = out_img_mems[i]->barrier.image_layout, + .newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + /* FIXME: implement exclusive transfers */ + .srcQueueFamilyIndex = 0, + .dstQueueFamilyIndex = 0, + .image = out_img_mems[i]->image, + .subresourceRange = out_img_mems[i]->barrier.subresource_range + }; + /* *INDENT-ON* */ + + vkCmdPipelineBarrier (cmd, + out_img_mems[i]->barrier.parent.pipeline_stages, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1, + &out_image_memory_barrier); + + out_img_mems[i]->barrier.parent.pipeline_stages = + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + out_img_mems[i]->barrier.parent.access_flags = + out_image_memory_barrier.dstAccessMask; + out_img_mems[i]->barrier.image_layout = out_image_memory_barrier.newLayout; + } + + { + VkImageView attachments[4] = { 0, }; + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { + attachments[i] = out_img_mems[i]->view; + } + + if (!(framebuffer = _create_framebuffer (conv, + GST_VIDEO_INFO_N_PLANES (&render->out_info), attachments))) { + g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, + "Failed to create framebuffer"); + goto error; + } + } + + view_convert_update_command_state (conv, cmd, in_img_mems, out_img_mems); + + if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd, + framebuffer)) + return GST_FLOW_ERROR; + + err = vkEndCommandBuffer (cmd); + if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0) + goto error; + + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_free_framebuffer (gst_vulkan_fence_ref (fence), + framebuffer)); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_free_command_buffer (gst_vulkan_fence_ref (fence), + conv->cmd_pool, cmd)); + + if (!gst_vulkan_full_screen_render_submit (render, cmd, fence)) + return GST_FLOW_ERROR; + + return GST_FLOW_OK; + +error: + GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL)); + g_clear_error (&error); + return GST_FLOW_ERROR; +} diff --git a/ext/vulkan/vkviewconvert.h b/ext/vulkan/vkviewconvert.h new file mode 100644 index 0000000000..370d89dfaf --- /dev/null +++ b/ext/vulkan/vkviewconvert.h @@ -0,0 +1,84 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _VK_VIEW_CONVERT_H_ +#define _VK_VIEW_CONVERT_H_ + +#include +#include +#include +#include "vkfullscreenrender.h" + +G_BEGIN_DECLS + +#define GST_TYPE_VULKAN_VIEW_CONVERT (gst_vulkan_view_convert_get_type()) +#define GST_VULKAN_VIEW_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_VIEW_CONVERT,GstVulkanViewConvert)) +#define GST_VULKAN_VIEW_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_VIEW_CONVERT,GstVulkanViewConvertClass)) +#define GST_IS_VULKAN_VIEW_CONVERT(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_VIEW_CONVERT)) +#define GST_IS_VULKAN_VIEW_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_VIEW_CONVERT)) + +typedef struct _GstVulkanViewConvert GstVulkanViewConvert; +typedef struct _GstVulkanViewConvertClass GstVulkanViewConvertClass; + +typedef enum +{ + GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS, + GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_RED_CYAN_DUBOIS, + GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_AMBER_BLUE_DUBOIS, +} GstVulkanStereoDownmix; + +struct _GstVulkanViewConvert +{ + GstVulkanFullScreenRender parent; + + GstVulkanCommandPool *cmd_pool; + + VkSampler sampler; + VkDescriptorPool descriptor_pool; + VkDescriptorSet descriptor_set; + + VkShaderModule vert_module; + VkShaderModule frag_module; + + VkDescriptorSetLayoutBinding sampler_layout_binding; + VkDescriptorSetLayoutCreateInfo layout_info; + + GstMemory *uniform; + gboolean descriptor_up_to_date; + + /* properties */ + GstVideoMultiviewMode input_mode_override; + GstVideoMultiviewFlags input_flags_override; + GstVideoMultiviewMode output_mode_override; + GstVideoMultiviewFlags output_flags_override; + + GstVulkanStereoDownmix downmix_mode; +}; + +struct _GstVulkanViewConvertClass +{ + GstVulkanFullScreenRenderClass parent_class; +}; + +GType gst_vulkan_view_convert_get_type(void); + +G_END_DECLS + +#endif