vulkan: add a color conversion element

Currently converts between all 4-component RGBA/RGBx formats.

ext/vulkan/gstvulkan.c

@@ -32,6 +32,7 @@
 #include "vksink.h"
 #include "vkupload.h"
 #include "vkimageidentity.h"
+#include "vkcolorconvert.h"
 #include "vkdownload.h"
 
 #define GST_CAT_DEFAULT gst_vulkan_debug
@@ -57,6 +58,11 @@ plugin_init (GstPlugin * plugin)
     return FALSE;
   }
 
+  if (!gst_element_register (plugin, "vulkancolorconvert",
+          GST_RANK_NONE, GST_TYPE_VULKAN_COLOR_CONVERT)) {
+    return FALSE;
+  }
+
   if (!gst_element_register (plugin, "vulkanimageidentity",
           GST_RANK_NONE, GST_TYPE_VULKAN_IMAGE_IDENTITY)) {
     return FALSE;

ext/vulkan/meson.build

@@ -10,6 +10,7 @@ subdir('shaders')
 
 vulkan_sources = [
   'gstvulkan.c',
+  'vkcolorconvert.c',
   'vkdownload.c',
   'vkfullscreenrender.c',
   'vkimageidentity.c',

ext/vulkan/shaders/meson.build

@@ -2,6 +2,8 @@
 gst_vulkan_shader_sources = [
   'identity.frag',
   'identity.vert',
+  'swizzle.frag',
+  'swizzle_and_clobber_alpha.frag',
 ]
 
 bin2array = find_program('bin2array.py')

ext/vulkan/shaders/swizzle.frag

@@ -0,0 +1,19 @@
+#version 450 core
+
+#include "swizzle.glsl"
+
+layout(location = 0) in vec2 inTexCoord;
+
+layout(push_constant) uniform reorder {
+  ivec4 in_reorder_idx;
+  ivec4 out_reorder_idx;
+};
+layout(set = 0, binding = 0) uniform sampler2D inTexture0;
+
+layout(location = 0) out vec4 outColor0;
+
+void main()
+{
+  vec4 rgba = swizzle (texture(inTexture0, inTexCoord), in_reorder_idx);
+  outColor0 = swizzle (rgba, out_reorder_idx);
+}

ext/vulkan/shaders/swizzle.glsl

@@ -0,0 +1,4 @@
+vec4 swizzle(in vec4 texel, in ivec4 swizzle_idx)
+{
+  return vec4(texel[swizzle_idx[0]], texel[swizzle_idx[1]], texel[swizzle_idx[2]], texel[swizzle_idx[3]]);
+}

ext/vulkan/shaders/swizzle_and_clobber_alpha.frag

@@ -0,0 +1,20 @@
+#version 450 core
+
+#include "swizzle.glsl"
+
+layout(location = 0) in vec2 inTexCoord;
+
+layout(push_constant) uniform reorder {
+  ivec4 in_reorder_idx;
+  ivec4 out_reorder_idx;
+};
+layout(set = 0, binding = 0) uniform sampler2D inTexture0;
+
+layout(location = 0) out vec4 outColor0;
+
+void main()
+{
+  vec4 rgba = swizzle (texture(inTexture0, inTexCoord), in_reorder_idx);
+  rgba.a = 1.0;
+  outColor0 = swizzle (rgba, out_reorder_idx);
+}

ext/vulkan/vkcolorconvert.h

@@ -0,0 +1,79 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _VK_COLOR_CONVERT_H_
+#define _VK_COLOR_CONVERT_H_
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/vulkan/vulkan.h>
+#include "vkfullscreenrender.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VULKAN_COLOR_CONVERT            (gst_vulkan_color_convert_get_type())
+#define GST_VULKAN_COLOR_CONVERT(obj)            (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_COLOR_CONVERT,GstVulkanColorConvert))
+#define GST_VULKAN_COLOR_CONVERT_CLASS(klass)    (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_COLOR_CONVERT,GstVulkanColorConvertClass))
+#define GST_IS_VULKAN_COLOR_CONVERT(obj)         (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_COLOR_CONVERT))
+#define GST_IS_VULKAN_COLOR_CONVERT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_COLOR_CONVERT))
+
+typedef struct _GstVulkanColorConvert GstVulkanColorConvert;
+typedef struct _GstVulkanColorConvertClass GstVulkanColorConvertClass;
+
+typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, GstVulkanImageMemory ** src_mems, GstVulkanImageMemory ** dest_mems);
+
+struct shader_info
+{
+  GstVideoFormat from;
+  GstVideoFormat to;
+  CommandStateUpdate cmd_state_update;
+  gchar *frag_code;
+  gsize frag_size;
+};
+
+struct _GstVulkanColorConvert
+{
+  GstVulkanFullScreenRender         parent;
+
+  GstVulkanCommandPool             *cmd_pool;
+
+  VkSampler                         sampler;
+  VkDescriptorPool                  descriptor_pool;
+  VkDescriptorSet                   descriptor_set;
+
+  VkShaderModule                    vert_module;
+  VkShaderModule                    frag_module;
+
+  VkDescriptorSetLayoutBinding      sampler_layout_binding;
+  VkDescriptorSetLayoutCreateInfo   layout_info;
+
+  struct shader_info               *current_shader;
+};
+
+struct _GstVulkanColorConvertClass
+{
+  GstVulkanFullScreenRenderClass parent_class;
+};
+
+GType gst_vulkan_color_convert_get_type(void);
+
+G_END_DECLS
+
+#endif

tests/check/elements/vkcolorconvert.c

@@ -0,0 +1,159 @@
+/* GStreamer
+ *
+ * unit test for vulkancolorconvert element
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <gst/gst.h>
+#include <gst/check/gstcheck.h>
+#include <gst/check/gstharness.h>
+#include <gst/vulkan/vulkan.h>
+
+static GstVulkanInstance *instance;
+static GstVulkanDevice *device;
+
+static void
+setup (void)
+{
+  instance = gst_vulkan_instance_new ();
+  fail_unless (gst_vulkan_instance_open (instance, NULL));
+  device = gst_vulkan_device_new (instance);
+  fail_unless (gst_vulkan_device_open (device, NULL));
+}
+
+static void
+teardown (void)
+{
+  gst_object_unref (instance);
+  gst_object_unref (device);
+}
+
+typedef struct _TestFrame
+{
+  gint width;
+  gint height;
+  GstVideoFormat v_format;
+  guint8 *data[GST_VIDEO_MAX_PLANES];
+} TestFrame;
+
+#define IGNORE_MAGIC 0x05
+
+static const guint8 rgba_reorder_data[] = { 0x49, 0x24, 0x72, 0xff };
+static const guint8 rgbx_reorder_data[] = { 0x49, 0x24, 0x72, IGNORE_MAGIC };
+static const guint8 argb_reorder_data[] = { 0xff, 0x49, 0x24, 0x72 };
+static const guint8 xrgb_reorder_data[] = { IGNORE_MAGIC, 0x49, 0x24, 0x72 };
+static const guint8 bgra_reorder_data[] = { 0x72, 0x24, 0x49, 0xff };
+static const guint8 bgrx_reorder_data[] = { 0x72, 0x24, 0x49, IGNORE_MAGIC };
+static const guint8 abgr_reorder_data[] = { 0xff, 0x72, 0x24, 0x49 };
+static const guint8 xbgr_reorder_data[] = { IGNORE_MAGIC, 0x72, 0x24, 0x49 };
+
+static TestFrame test_rgba_reorder[] = {
+  {1, 1, GST_VIDEO_FORMAT_RGBA, {(guint8 *) & rgba_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_RGBx, {(guint8 *) & rgbx_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_ARGB, {(guint8 *) & argb_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_xRGB, {(guint8 *) & xrgb_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_BGRA, {(guint8 *) & bgra_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_BGRx, {(guint8 *) & bgrx_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_ABGR, {(guint8 *) & abgr_reorder_data}},
+  {1, 1, GST_VIDEO_FORMAT_xBGR, {(guint8 *) & xbgr_reorder_data}},
+};
+
+GST_START_TEST (test_vulkan_color_convert_rgba_reorder)
+{
+  GstHarness *h =
+      gst_harness_new_parse
+      ("vulkanupload ! vulkancolorconvert ! vulkandownload");
+  int i, j, k;
+
+  for (i = 0; i < G_N_ELEMENTS (test_rgba_reorder); i++) {
+    for (j = 0; j < G_N_ELEMENTS (test_rgba_reorder); j++) {
+      GstCaps *in_caps, *out_caps;
+      GstVideoInfo in_info, out_info;
+      GstBuffer *inbuf, *outbuf;
+      GstMapInfo map_info;
+
+      fail_unless (gst_video_info_set_format (&in_info,
+              test_rgba_reorder[i].v_format, test_rgba_reorder[i].width,
+              test_rgba_reorder[i].height));
+      fail_unless (gst_video_info_set_format (&out_info,
+              test_rgba_reorder[j].v_format, test_rgba_reorder[j].width,
+              test_rgba_reorder[j].height));
+
+      in_caps = gst_video_info_to_caps (&in_info);
+      out_caps = gst_video_info_to_caps (&out_info);
+
+      gst_harness_set_caps (h, in_caps, out_caps);
+
+      GST_INFO ("converting from %s to %s",
+          gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&in_info)),
+          gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&out_info)));
+
+      inbuf =
+          gst_buffer_new_wrapped_full (0, test_rgba_reorder[i].data[0], 4, 0, 4,
+          NULL, NULL);
+      outbuf = gst_harness_push_and_pull (h, inbuf);
+
+      fail_unless (gst_buffer_map (outbuf, &map_info, GST_MAP_READ));
+      /* FIXME: wrong size! */
+      fail_unless (map_info.size >= out_info.size);
+
+      for (k = 0; k < out_info.size; k++) {
+        if (test_rgba_reorder[j].data[0][k] != IGNORE_MAGIC
+            && map_info.data[k] != IGNORE_MAGIC) {
+          guint8 *expected = test_rgba_reorder[j].data[0];
+          GST_DEBUG ("%i 0x%x =? 0x%x", k, expected[k],
+              (guint) map_info.data[k]);
+          fail_unless (expected[k] == map_info.data[k]);
+        }
+      }
+      gst_buffer_unmap (outbuf, &map_info);
+      gst_buffer_unref (outbuf);
+    }
+  }
+
+  gst_harness_teardown (h);
+}
+
+GST_END_TEST;
+
+static Suite *
+vkcolorconvert_suite (void)
+{
+  Suite *s = suite_create ("vkcolorconvert");
+  TCase *tc_basic = tcase_create ("general");
+  gboolean have_instance;
+
+  suite_add_tcase (s, tc_basic);
+  tcase_add_checked_fixture (tc_basic, setup, teardown);
+
+  /* FIXME: CI doesn't have a software vulkan renderer (and none exists currently) */
+  instance = gst_vulkan_instance_new ();
+  have_instance = gst_vulkan_instance_open (instance, NULL);
+  gst_object_unref (instance);
+  if (have_instance) {
+    tcase_add_test (tc_basic, test_vulkan_color_convert_rgba_reorder);
+  }
+
+  return s;
+}
+
+GST_CHECK_MAIN (vkcolorconvert);

tests/check/meson.build

@@ -63,6 +63,7 @@ base_tests = [
   [['libs/vc1parser.c'], false, [gstcodecparsers_dep]],
   [['libs/vp8parser.c'], false, [gstcodecparsers_dep]],
   [['libs/vkmemory.c'], not gstvulkan_dep.found(), [gstvulkan_dep]],
+  [['elements/vkcolorconvert.c'], not gstvulkan_dep.found(), [gstvulkan_dep]],
 ]
 
 # FIXME: unistd dependency, unstable or not tested yet on windows