vulkan: add a new image copying element

Copies using the graphics pipeline, an input image and renders to an
output image.

ext/vulkan/gstvulkan.c

@@ -31,6 +31,7 @@
 
 #include "vksink.h"
 #include "vkupload.h"
+#include "vkimageidentity.h"
 
 #define GST_CAT_DEFAULT gst_vulkan_debug
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
@@ -50,6 +51,11 @@ plugin_init (GstPlugin * plugin)
     return FALSE;
   }
 
+  if (!gst_element_register (plugin, "vulkanimageidentity",
+          GST_RANK_NONE, GST_TYPE_VULKAN_IMAGE_IDENTITY)) {
+    return FALSE;
+  }
+
   return TRUE;
 }
 

ext/vulkan/meson.build

@@ -1,9 +1,16 @@
 if get_option('vulkan').disabled()
   subdir_done()
 endif
+glslc = find_program('glslc', required: get_option('vulkan'))
+if not glslc.found()
+  subdir_done()
+endif
+
+subdir('shaders')
 
 vulkan_sources = [
   'gstvulkan.c',
+  'vkimageidentity.c',
   'vksink.c',
   'vkshader.c',
   'vkswapper.c',
@@ -12,7 +19,7 @@ vulkan_sources = [
 ]
 
 gstvulkan_plugin = library('gstvulkan',
-  vulkan_sources,
+  vulkan_sources, vulkan_compiled_shader_sources,
   c_args : gst_plugins_bad_args,
   objc_args : gst_plugins_bad_args,
   link_args : noseh_link_args,

ext/vulkan/shaders/bin2array.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+import binascii
+import sys
+import os
+import argparse
+
+autogenerated_notice = """/*
+ * This file is autogenerated by bin2array.py
+ */
+"""
+
+def make_sublist_group(lst: list, grp: int) -> list:
+    """
+    Group list elements into sublists.
+    make_sublist_group([1, 2, 3, 4, 5, 6, 7], 3) = [[1, 2, 3], [4, 5, 6], 7]
+    """
+    return [lst[i:i+grp] for i in range(0, len(lst), grp)]
+
+def generate_c_array_data(bindata: bytes, element_prefix: str, element_size: int, element_suffix: str, newline_value: str, newline_after: int, element_separator: str):
+    """
+    Generate the contents of a C array
+    """
+    hexstr = binascii.hexlify(bindata).decode("UTF-8")
+    array = []
+    for i in range(0, len(hexstr), 2 * element_size):
+        array += [element_prefix + hexstr[i:i + 2 * element_size] + element_suffix]
+
+    if newline_after:
+        array = make_sublist_group(array, newline_after)
+    else:
+        array = [array,]
+
+    return newline_value.join([element_separator.join(e) + element_separator for e in array])
+
+def decorate_c_array_data (hexdata: str, var_name: str, var_type: str, newline_value: str):
+    ret = var_type + " " + var_name + "[] = {" + newline_value
+    ret += hexdata + newline_value
+    ret += "};" + newline_value
+    return ret
+
+def main(args):
+    parser = argparse.ArgumentParser(description='Convert binary file to C-style array initializer.')
+    parser.add_argument("-i", "--input", help="the file to be converted")
+    parser.add_argument("--output", help="c source file location")
+    parser.add_argument("--header-output", help="c header file location")
+    parser.add_argument("--linebreak", type=int, help="add linebreak after every N element")
+    parser.add_argument("--linebreak-value", default="\n", help="use what sequence to break lines, defaults to \"\\n\"")
+    parser.add_argument("--separator", default=", ", help="use what to separate elements, defaults to \", \"")
+    parser.add_argument("--array-name", default="array_data", help="name of the resulting array")
+    parser.add_argument("--element-type", default="char", help="C type for the array")
+    parser.add_argument("--element-size", type=int, default=1, help="how many bytes per element")
+    parser.add_argument("--element-prefix", default="0x", help="string to be added to the head of element, defaults to \"0x\"")
+    parser.add_argument("--element-suffix", default="", help="string to be added to the tail of element, defaults to none")
+    parser.add_argument("--c-include", help="header to include")
+    args = parser.parse_args(args)
+
+    with open(args.input, 'rb') as f:
+        file_content = f.read()
+
+    # don't deal with unaligned content
+    assert len(file_content) % args.element_size == 0
+
+    # get a relative path from the source file to the header to use in an #include
+    source_to_header = os.path.relpath (os.path.dirname(args.header_output), os.path.dirname (args.output))
+
+    ret = autogenerated_notice
+    ret += "#include \"" + os.path.join (source_to_header, os.path.basename (args.header_output)) + "\"" + args.linebreak_value
+
+    ret += decorate_c_array_data (generate_c_array_data (file_content, args.element_prefix, args.element_size, args.element_suffix, args.linebreak_value, args.linebreak, args.separator), args.array_name, args.element_type, args.linebreak_value)
+
+    with open (args.output, 'w') as f:
+        f.write (ret)
+
+    # write companion header
+    with open(args.header_output, 'w') as f:
+        f.write (autogenerated_notice)
+        if args.c_include:
+            f.write ("#include <" + args.c_include + ">" + args.linebreak_value)
+        f.write ("extern " + args.element_type + " " + args.array_name + "[];" + args.linebreak_value)
+        f.write ("#define " + args.array_name + "_size " + str(len(file_content) // args.element_size))
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

ext/vulkan/shaders/identity.frag

@@ -0,0 +1,12 @@
+#version 420 core
+
+layout(location = 0) in vec2 inTexCoord;
+
+layout(set = 0, binding = 0) uniform sampler2D inTexture;
+
+layout(location = 0) out vec4 outColor;
+
+void main()
+{
+  outColor = texture (inTexture, inTexCoord);
+}

ext/vulkan/shaders/identity.vert

@@ -0,0 +1,12 @@
+#version 420 core
+
+layout(location = 0) in vec4 inPos;
+layout(location = 1) in vec2 inTexCoord;
+
+layout(location = 0) out vec2 outTexCoord;
+
+void main() {
+   gl_Position = inPos;
+   outTexCoord = inTexCoord;
+}
+

ext/vulkan/shaders/meson.build

@@ -0,0 +1,44 @@
+# Shaders
+gst_vulkan_shader_sources = [
+  'identity.frag',
+  'identity.vert',
+]
+
+bin2array = find_program('bin2array.py')
+
+vulkan_compiled_shader_sources = []
+foreach shader: gst_vulkan_shader_sources
+  basefn = shader.split('.').get(0)
+  suffix = shader.split('.').get(1)
+
+  stage_arg = suffix == 'frag' ? '-fshader-stage=fragment' : '-fshader-stage=vertex'
+  basename = '@0@.@1@'.format(basefn, suffix)
+  spv_shader = basename + '.spv'
+  c_shader_source = basename + '.c'
+  c_shader_header = basename + '.h'
+
+  compiled_shader = custom_target(spv_shader,
+                                  input: shader,
+                                  output: spv_shader,
+                                  command: [
+                                    glslc,
+                                    stage_arg,
+                                    '@INPUT@',
+                                    '-o', '@OUTPUT@'
+                                  ])
+
+  c_shader = custom_target (c_shader_source,
+      input: compiled_shader,
+      output: [c_shader_source, c_shader_header],
+      command: [ bin2array,
+        '--array-name=' + basename.underscorify(),
+        '--c-include=gst/gst.h',
+        '--element-type=gchar',
+        '--element-size=1',
+        '--linebreak=8',
+        '--input', '@INPUT@', 
+        '--output', '@OUTPUT0@',
+        '--header-output', '@OUTPUT1@'])
+
+  vulkan_compiled_shader_sources += [c_shader]
+endforeach

ext/vulkan/vkimageidentity.h

@@ -0,0 +1,77 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _VK_IMAGE_IDENTITY_H_
+#define _VK_IMAGE_IDENTITY_H_
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/vulkan/vulkan.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VULKAN_IMAGE_IDENTITY            (gst_vulkan_image_identity_get_type())
+#define GST_VULKAN_IMAGE_IDENTITY(obj)            (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_IMAGE_IDENTITY,GstVulkanImageIdentity))
+#define GST_VULKAN_IMAGE_IDENTITY_CLASS(klass)    (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_IMAGE_IDENTITY,GstVulkanImageIdentityClass))
+#define GST_IS_VULKAN_IMAGE_IDENTITY(obj)         (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_IMAGE_IDENTITY))
+#define GST_IS_VULKAN_IMAGE_IDENTITY_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_IMAGE_IDENTITY))
+
+typedef struct _GstVulkanImageIdentity GstVulkanImageIdentity;
+typedef struct _GstVulkanImageIdentityClass GstVulkanImageIdentityClass;
+
+struct _GstVulkanImageIdentity
+{
+  GstBaseTransform      parent;
+
+  GstVulkanInstance    *instance;
+  GstVulkanDevice      *device;
+  GstVulkanQueue       *queue;
+  GstVulkanCommandPool *cmd_pool;
+
+  GstCaps              *caps;
+  GstVideoInfo          v_info;
+
+  VkRenderPass          render_pass;
+  VkPipelineLayout      pipeline_layout;
+  VkPipeline            graphics_pipeline;
+  VkSampler             sampler;
+  VkDescriptorSetLayout descriptor_set_layout;
+  VkDescriptorPool      descriptor_pool;
+  VkDescriptorSet       descriptor_set;
+
+  GstMemory            *vertices;
+  GstMemory            *indices;
+
+  GList                *trash_list;
+  GstVulkanFence       *last_fence;
+
+  GArray               *frames;
+};
+
+struct _GstVulkanImageIdentityClass
+{
+  GstBaseTransformClass video_sink_class;
+};
+
+GType gst_vulkan_image_identity_get_type(void);
+
+G_END_DECLS
+
+#endif

ext/vulkan/vkshader.c

@@ -0,0 +1,89 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "vkshader.h"
+
+#define SPIRV_MAGIC_NUMBER_NE 0x07230203
+#define SPIRV_MAGIC_NUMBER_OE 0x03022307
+
+VkShaderModule
+_vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size,
+    GError ** error)
+{
+  VkShaderModule ret;
+  VkResult res;
+
+  /* *INDENT-OFF* */
+  VkShaderModuleCreateInfo info = {
+      .sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+      .pNext = NULL,
+      .flags = 0,
+      .codeSize = size,
+      .pCode = (guint32 *) code
+  };
+  /* *INDENT-ON* */
+  guint32 first_word;
+  guint32 *new_code = NULL;
+
+  g_return_val_if_fail (size >= 4, NULL);
+  g_return_val_if_fail (size % 4 == 0, NULL);
+
+  first_word = ((guint32 *) code)[0];
+  g_return_val_if_fail (first_word == SPIRV_MAGIC_NUMBER_NE
+      || first_word == SPIRV_MAGIC_NUMBER_OE, NULL);
+  if (first_word == SPIRV_MAGIC_NUMBER_OE) {
+    /* endianess swap... */
+    guint32 *old_code = (guint32 *) code;
+    gsize i;
+
+    GST_DEBUG ("performaing endianess conversion on spirv shader of size %"
+        G_GSIZE_FORMAT, size);
+    new_code = g_new0 (guint32, size / 4);
+
+    for (i = 0; i < size / 4; i++) {
+      guint32 old = old_code[i];
+      guint32 new = 0;
+
+      new |= (old & 0xff) << 24;
+      new |= (old & 0xff00) << 8;
+      new |= (old & 0xff0000) >> 8;
+      new |= (old & 0xff000000) >> 24;
+      new_code[i] = new;
+    }
+
+    first_word = ((guint32 *) new_code)[0];
+    g_assert (first_word == SPIRV_MAGIC_NUMBER_NE);
+
+    info.pCode = new_code;
+  }
+
+  res = vkCreateShaderModule (device->device, &info, NULL, &ret);
+  g_free (new_code);
+  if (gst_vulkan_error_to_g_error (res, error, "VkCreateShaderModule") < 0)
+    return NULL;
+
+  g_free (new_code);
+
+  return ret;
+}

ext/vulkan/vkshader.h

@@ -0,0 +1,31 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _VK_SHADER_H_
+#define _VK_SHADER_H_
+
+#include <gst/gst.h>
+#include <gst/vulkan/vulkan.h>
+
+G_BEGIN_DECLS
+
+VkShaderModule _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error);
+
+#endif