vulkan: add view converter element

ext/vulkan/gstvulkan.c

@@ -34,6 +34,7 @@
 #include "vkimageidentity.h"
 #include "vkcolorconvert.h"
 #include "vkdownload.h"
+#include "vkviewconvert.h"
 
 #define GST_CAT_DEFAULT gst_vulkan_debug
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
@@ -68,6 +69,11 @@ plugin_init (GstPlugin * plugin)
     return FALSE;
   }
 
+  if (!gst_element_register (plugin, "vulkanviewconvert",
+          GST_RANK_NONE, GST_TYPE_VULKAN_VIEW_CONVERT)) {
+    return FALSE;
+  }
+
   return TRUE;
 }
 

ext/vulkan/meson.build

@@ -17,10 +17,23 @@ vulkan_sources = [
   'vksink.c',
   'vkshader.c',
   'vkupload.c',
+  'vkviewconvert.c',
 ]
 
+vulkan_plugin_enum_headers = [
+  'vkviewconvert.h',
+]
+
+vulkan_plugin_enums = gnome.mkenums_simple('gstvulkan-plugins-enumtypes',
+    sources : vulkan_plugin_enum_headers,
+    body_prefix : '#ifdef HAVE_CONFIG_H\n#include "config.h"\n#endif',
+    header_prefix : '#include <gst/gst.h>')
+vulkan_plugin_enumtypes_c = vulkan_plugin_enums[0]
+vulkan_plugin_enumtypes_h = vulkan_plugin_enums[1]
+vulkan_plugin_gen_sources = [vulkan_plugin_enumtypes_h]
+
 gstvulkan_plugin = library('gstvulkan',
-  vulkan_sources, vulkan_compiled_shader_sources,
+  vulkan_sources, vulkan_compiled_shader_sources, vulkan_plugin_enumtypes_c, vulkan_plugin_enumtypes_h,
   c_args : gst_plugins_bad_args,
   objc_args : gst_plugins_bad_args,
   link_args : noseh_link_args,

ext/vulkan/shaders/meson.build

@@ -10,6 +10,7 @@ gst_vulkan_shader_sources = [
   'rgb_to_ayuv.frag',
   'nv12_to_rgb.frag',
   'rgb_to_nv12.frag',
+  'view_convert.frag',
 ]
 
 bin2array = find_program('bin2array.py')

ext/vulkan/shaders/view_convert.frag

@@ -0,0 +1,79 @@
+#version 450 core
+
+#include "view_defines.h"
+#include "swizzle.glsl"
+
+layout(location = 0) in vec2 inTexCoord;
+
+layout(set = 0, binding = 0) uniform sampler2D l_tex;
+layout(set = 0, binding = 1) uniform sampler2D r_tex;
+layout(set = 0, binding = 2) uniform ViewConvert {
+  ivec4 in_reorder_idx;
+  ivec4 out_reorder_idx;
+  vec4 tex_offset;
+  vec4 tex_scale;
+  ivec2 tex_size;
+  int output_type;
+  mat3 downmix[2];
+};
+
+layout(location = 0) out vec4 outColor0;
+
+void main()
+{
+  vec2 l_coord = inTexCoord * tex_scale.xy + tex_offset.xy;
+  vec2 r_coord = inTexCoord * tex_scale.zw + tex_offset.zw;
+  vec4 l = swizzle(texture(l_tex, l_coord), in_reorder_idx);
+  vec4 r = swizzle(texture(r_tex, r_coord), in_reorder_idx);
+
+  if (output_type == VIEW_MONO_DOWNMIX) {
+    vec3 lcol = l.rgb * l.a + vec3(1.0-l.a);
+    vec3 rcol = r.rgb * r.a + vec3(1.0-r.a);
+    if (l.a + r.a > 0.0) {
+      lcol = clamp (downmix[0] * lcol, 0.0, 1.0);
+      rcol = clamp (downmix[1] * rcol, 0.0, 1.0);
+      outColor0 = vec4 (lcol + rcol, 1.0);
+    } else {
+      outColor0 = vec4 (0.0);
+    }
+  } else if (output_type == VIEW_MONO_LEFT) {
+    outColor0 = swizzle(l, out_reorder_idx);
+    outColor0 = vec4(1.0, 0.0, 1.0, 1.0);
+  } else if (output_type == VIEW_MONO_RIGHT) {
+    outColor0 = swizzle(r, out_reorder_idx);
+    outColor0 = vec4(0.0, 0.0, 1.0, 1.0);
+  } else if (output_type == VIEW_SIDE_BY_SIDE) {
+    if (inTexCoord.x < 0.5) {
+      outColor0 = swizzle(l, out_reorder_idx);
+    } else {
+      outColor0 = swizzle(r, out_reorder_idx);
+    }
+  } else if (output_type == VIEW_TOP_BOTTOM) {
+    if (inTexCoord.y < 0.5) {
+      outColor0 = swizzle(l, out_reorder_idx);
+    } else {
+      outColor0 = swizzle(r, out_reorder_idx);
+    }
+  } else if (output_type == VIEW_COLUMN_INTERLEAVED) {
+    if (int(mod(l_coord.x * tex_size.x, 2.0)) == 0) {
+      outColor0 = swizzle(l, out_reorder_idx);
+    } else {
+      outColor0 = swizzle(r, out_reorder_idx);
+    }
+  } else if (output_type == VIEW_ROW_INTERLEAVED) {
+    if (int(mod(l_coord.y * tex_size.y, 2.0)) == 0) {
+      outColor0 = swizzle(l, out_reorder_idx);
+    } else {
+      outColor0 = swizzle(r, out_reorder_idx);
+    }
+  } else if (output_type == VIEW_CHECKERBOARD) {
+     if (int(mod(l_coord.x * tex_size.x, 2.0)) ==
+         int(mod(l_coord.y * tex_size.y, 2.0))) {
+      outColor0 = swizzle(l, out_reorder_idx);
+    } else {
+      outColor0 = swizzle(r, out_reorder_idx);
+    }
+  } else {
+    outColor0 = vec4(1.0, 0.0, 1.0, 1.0);
+  }
+}

ext/vulkan/shaders/view_defines.h

@@ -0,0 +1,10 @@
+/* values match GstVideoMultiviewMode */
+#define VIEW_MONO_DOWNMIX 0
+#define VIEW_MONO_LEFT 1
+#define VIEW_MONO_RIGHT 2
+#define VIEW_SIDE_BY_SIDE 3
+/*  GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE_QUINCUNX,*/
+#define VIEW_COLUMN_INTERLEAVED 5
+#define VIEW_ROW_INTERLEAVED 6
+#define VIEW_TOP_BOTTOM 7
+#define VIEW_CHECKERBOARD 8

ext/vulkan/vkviewconvert.h

@@ -0,0 +1,84 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _VK_VIEW_CONVERT_H_
+#define _VK_VIEW_CONVERT_H_
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/vulkan/vulkan.h>
+#include "vkfullscreenrender.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VULKAN_VIEW_CONVERT            (gst_vulkan_view_convert_get_type())
+#define GST_VULKAN_VIEW_CONVERT(obj)            (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_VIEW_CONVERT,GstVulkanViewConvert))
+#define GST_VULKAN_VIEW_CONVERT_CLASS(klass)    (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_VIEW_CONVERT,GstVulkanViewConvertClass))
+#define GST_IS_VULKAN_VIEW_CONVERT(obj)         (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_VIEW_CONVERT))
+#define GST_IS_VULKAN_VIEW_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_VIEW_CONVERT))
+
+typedef struct _GstVulkanViewConvert GstVulkanViewConvert;
+typedef struct _GstVulkanViewConvertClass GstVulkanViewConvertClass;
+
+typedef enum
+{
+  GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS,
+  GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_RED_CYAN_DUBOIS,
+  GST_VULKAN_STEREO_DOWNMIX_ANAGLYPH_AMBER_BLUE_DUBOIS,
+} GstVulkanStereoDownmix;
+
+struct _GstVulkanViewConvert
+{
+  GstVulkanFullScreenRender         parent;
+
+  GstVulkanCommandPool             *cmd_pool;
+
+  VkSampler                         sampler;
+  VkDescriptorPool                  descriptor_pool;
+  VkDescriptorSet                   descriptor_set;
+
+  VkShaderModule                    vert_module;
+  VkShaderModule                    frag_module;
+
+  VkDescriptorSetLayoutBinding      sampler_layout_binding;
+  VkDescriptorSetLayoutCreateInfo   layout_info;
+
+  GstMemory                        *uniform;
+  gboolean                          descriptor_up_to_date;
+
+  /* properties */
+  GstVideoMultiviewMode input_mode_override;
+  GstVideoMultiviewFlags input_flags_override;
+  GstVideoMultiviewMode output_mode_override;
+  GstVideoMultiviewFlags output_flags_override;
+
+  GstVulkanStereoDownmix downmix_mode;
+};
+
+struct _GstVulkanViewConvertClass
+{
+  GstVulkanFullScreenRenderClass parent_class;
+};
+
+GType gst_vulkan_view_convert_get_type(void);
+
+G_END_DECLS
+
+#endif