gstreamer/subprojects/gst-plugins-bad/ext/vulkan/vkshaderspv.c

/*
 * GStreamer
 * Copyright (C) 2022 Martin Reboredo <yakoyoku@gmail.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-vulkanshaderspv
 * @title: vulkanshaderspv
 *
 * Vulkan image shader filter.
 *
 * ## Examples
 * ```
 * gst-launch-1.0 videotestsrc ! vulkanupload ! vulkanshader fragment-location="myshader.f.spv" ! vulkanimagesink
 * ```
 * The following is a simple Vulkan passthrough shader with the required inputs.
 * Compile it with `glslc --target-env=vulkan1.0 myshader.frag -o myshader.f.spv`.
 * ``` glsl
 * #version 450
 *
 * layout(location = 0) in vec2 inTexCoord;
 *
 * layout(set = 0, binding = 0) uniform ShaderFilter {
 *   float time;
 *   float width;
 *   float height;
 * };
 * layout(set = 0, binding = 1) uniform sampler2D inTexture;
 *
 * layout(location = 0) out vec4 outColor;
 *
 * void main () {
 *   outColor = texture (inTexture, inTexCoord);
 * }
 * ```
 *
 * Since: 1.22
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <string.h>

#include <gio/gio.h>

#include "gstvulkanelements.h"
#include "vkshaderspv.h"

#include "shaders/identity.vert.h"
#include "shaders/identity.frag.h"

GST_DEBUG_CATEGORY (gst_debug_vulkan_shader_spv);
#define GST_CAT_DEFAULT gst_debug_vulkan_shader_spv

static void gst_vulkan_shader_spv_finalize (GObject * object);
static void gst_vulkan_shader_spv_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_vulkan_shader_spv_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

static gboolean gst_vulkan_shader_spv_start (GstBaseTransform * bt);
static gboolean gst_vulkan_shader_spv_stop (GstBaseTransform * bt);

static GstFlowReturn gst_vulkan_shader_spv_transform (GstBaseTransform * bt,
    GstBuffer * inbuf, GstBuffer * outbuf);
static gboolean gst_vulkan_shader_spv_set_caps (GstBaseTransform * bt,
    GstCaps * in_caps, GstCaps * out_caps);

#define IMAGE_FORMATS " { BGRA }"

static GstStaticPadTemplate gst_vulkan_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
    GST_PAD_SINK,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
        (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
            IMAGE_FORMATS)));

static GstStaticPadTemplate gst_vulkan_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
        (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
            IMAGE_FORMATS)));

enum
{
  PROP_0,
  PROP_VERTEX,
  PROP_FRAGMENT,
  PROP_VERTEX_PATH,
  PROP_FRAGMENT_PATH,
};

enum
{
  SIGNAL_0,
  LAST_SIGNAL
};

/* static guint gst_vulkan_shader_spv_signals[LAST_SIGNAL] = { 0 }; */

#define gst_vulkan_shader_spv_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanShaderSpv, gst_vulkan_shader_spv,
    GST_TYPE_VULKAN_VIDEO_FILTER,
    GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_shader_spv,
        "vulkanshaderspv", 0, "Vulkan Image identity"));
GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (vulkanshaderspv,
    "vulkanshaderspv", GST_RANK_NONE, GST_TYPE_VULKAN_SHADER_SPV,
    vulkan_element_init (plugin));

static void
gst_vulkan_shader_spv_class_init (GstVulkanShaderSpvClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *gstelement_class;
  GstBaseTransformClass *gstbasetransform_class;

  gobject_class = G_OBJECT_CLASS (klass);
  gstelement_class = GST_ELEMENT_CLASS (klass);
  gstbasetransform_class = GST_BASE_TRANSFORM_CLASS (klass);

  gobject_class->finalize = gst_vulkan_shader_spv_finalize;
  gobject_class->set_property = gst_vulkan_shader_spv_set_property;
  gobject_class->get_property = gst_vulkan_shader_spv_get_property;

  g_object_class_install_property (gobject_class, PROP_VERTEX,
      g_param_spec_boxed ("vertex", "Vertex Binary",
          "SPIRV vertex binary", G_TYPE_BYTES,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_FRAGMENT,
      g_param_spec_boxed ("fragment", "Fragment Binary",
          "SPIRV fragment binary", G_TYPE_BYTES,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_VERTEX_PATH,
      g_param_spec_string ("vertex-location", "Vertex Source",
          "SPIRV vertex source", NULL,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  g_object_class_install_property (gobject_class, PROP_FRAGMENT_PATH,
      g_param_spec_string ("fragment-location", "Fragment Source",
          "SPIRV fragment source", NULL,
          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));

  gst_element_class_set_metadata (gstelement_class, "Vulkan Shader SPV",
      "Filter/Video", "Performs operations with SPIRV shaders in Vulkan",
      "Martin Reboredo <yakoyoku@gmail.com>");

  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_vulkan_sink_template);
  gst_element_class_add_static_pad_template (gstelement_class,
      &gst_vulkan_src_template);

  gstbasetransform_class->start =
      GST_DEBUG_FUNCPTR (gst_vulkan_shader_spv_start);
  gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_vulkan_shader_spv_stop);
  gstbasetransform_class->set_caps = gst_vulkan_shader_spv_set_caps;
  gstbasetransform_class->transform = gst_vulkan_shader_spv_transform;
}

static void
gst_vulkan_shader_spv_init (GstVulkanShaderSpv * vk_shader)
{
  vk_shader->vert = g_bytes_new (NULL, 0);
  vk_shader->frag = g_bytes_new (NULL, 0);
}

static void
gst_vulkan_shader_spv_finalize (GObject * object)
{
  GstVulkanShaderSpv *filter = GST_VULKAN_SHADER_SPV (object);

  g_bytes_unref (filter->vert);
  filter->vert = NULL;

  g_bytes_unref (filter->frag);
  filter->frag = NULL;

  g_free (filter->vert_path);
  filter->vert_path = NULL;

  g_free (filter->frag_path);
  filter->frag_path = NULL;

  if (filter->uniforms)
    gst_memory_unref (filter->uniforms);
  filter->uniforms = NULL;

  G_OBJECT_CLASS (gst_vulkan_shader_spv_parent_class)->finalize (object);
}

#define SPIRV_MAGIC_NUMBER_NE 0x07230203
#define SPIRV_MAGIC_NUMBER_OE 0x03022307

static GBytes *
gst_vulkan_shader_spv_check_shader_binary (const GValue * value)
{
  GBytes *bytes = NULL;
  gsize len;
  const gchar *data;
  gint32 first_word;

  bytes = g_value_dup_boxed (value);
  if (!bytes)
    return NULL;
  data = g_bytes_get_data (bytes, &len);
  if (len == 0 || len & 0x03) {
    g_bytes_unref (bytes);
    return NULL;
  }
  first_word = data[0] | data[1] << 8 | data[2] << 16 | data[3] << 24;
  if (first_word != SPIRV_MAGIC_NUMBER_NE &&
      first_word != SPIRV_MAGIC_NUMBER_OE) {
    g_bytes_unref (bytes);
    return NULL;
  }
  return bytes;
}

static void
gst_vulkan_shader_spv_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstVulkanShaderSpv *filter = GST_VULKAN_SHADER_SPV (object);
  GBytes *bytes = NULL;

  switch (prop_id) {
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filter);
      if (!(bytes = gst_vulkan_shader_spv_check_shader_binary (value)))
        goto wrong_format;
      g_bytes_unref (filter->vert);
      filter->vert = bytes;
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filter);
      if (!(bytes = gst_vulkan_shader_spv_check_shader_binary (value)))
        goto wrong_format;
      g_bytes_unref (filter->frag);
      filter->frag = bytes;
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_VERTEX_PATH:
      GST_OBJECT_LOCK (filter);
      g_free (filter->vert_path);
      filter->vert_path = g_value_dup_string (value);
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_FRAGMENT_PATH:
      GST_OBJECT_LOCK (filter);
      g_free (filter->frag_path);
      filter->frag_path = g_value_dup_string (value);
      GST_OBJECT_UNLOCK (filter);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }

  return;

wrong_format:
  {
    g_critical ("Badly formatted byte sequence, must have a nonzero length"
        " that is a multiple of four and start with the SPIRV magic number");
    GST_OBJECT_UNLOCK (filter);
    return;
  }
}

static void
gst_vulkan_shader_spv_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  GstVulkanShaderSpv *filter = GST_VULKAN_SHADER_SPV (object);

  switch (prop_id) {
    case PROP_VERTEX:
      GST_OBJECT_LOCK (filter);
      g_value_set_boxed (value, filter->vert);
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_FRAGMENT:
      GST_OBJECT_LOCK (filter);
      g_value_set_boxed (value, filter->frag);
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_VERTEX_PATH:
      GST_OBJECT_LOCK (filter);
      g_value_set_string (value, filter->vert_path);
      GST_OBJECT_UNLOCK (filter);
      break;
    case PROP_FRAGMENT_PATH:
      GST_OBJECT_LOCK (filter);
      g_value_set_string (value, filter->frag_path);
      GST_OBJECT_UNLOCK (filter);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
gst_vulkan_shader_spv_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
    GstCaps * out_caps)
{
  GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
  GstVulkanShaderSpv *vk_identity = GST_VULKAN_SHADER_SPV (bt);

  if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
          out_caps))
    return FALSE;

  if (!gst_vulkan_full_screen_quad_set_info (vk_identity->quad,
          &vfilter->in_info, &vfilter->out_info))
    return FALSE;

  return TRUE;
}

static GstVulkanHandle *
gst_vulkan_shader_spv_create_shader (GstVulkanShaderSpv * shader,
    GBytes * binary, const char *path, const gchar * identity,
    gsize identity_size, GError ** error)
{
  GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (shader);
  const gchar *data;
  gsize len;
  GstVulkanHandle *handle;

  data = g_bytes_get_data (binary, &len);
  if (data) {
    if (!(handle =
            gst_vulkan_create_shader (vfilter->device, data, len, error)))
      return NULL;
  } else if (path) {
    GFile *file;
    GFileInfo *info;
    GFileInputStream *istream;
    GBytes *res;
    const gchar *data;
    gsize len = 35648;

    file = g_file_new_for_path (path);
    if (!(istream = g_file_read (file, NULL, error))) {
      g_object_unref (file);
      return NULL;
    }
    if ((info =
            g_file_query_info (file, G_FILE_ATTRIBUTE_STANDARD_SIZE,
                G_FILE_QUERY_INFO_NONE, NULL, NULL))) {
      len = g_file_info_get_size (info);
      g_object_unref (info);
    }
    if (!(res =
            g_input_stream_read_bytes (G_INPUT_STREAM (istream), len, NULL,
                error))) {
      g_input_stream_close (G_INPUT_STREAM (istream), NULL, NULL);
      g_object_unref (file);
      return NULL;
    }
    data = g_bytes_get_data (res, &len);
    if (!(handle =
            gst_vulkan_create_shader (vfilter->device, data, len, error))) {
      g_bytes_unref (res);
      g_input_stream_close (G_INPUT_STREAM (istream), NULL, NULL);
      g_object_unref (file);
      return NULL;
    }
    g_bytes_unref (res);
    g_input_stream_close (G_INPUT_STREAM (istream), NULL, NULL);
    g_object_unref (file);
  } else {
    if (!(handle = gst_vulkan_create_shader (vfilter->device, identity,
                identity_size, error)))
      return NULL;
  }

  return handle;
}

static gboolean
gst_vulkan_shader_spv_start (GstBaseTransform * bt)
{
  GstVulkanShaderSpv *vk_shader = GST_VULKAN_SHADER_SPV (bt);
  GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (vk_shader);
  GstVulkanHandle *vert, *frag;
  GError *error = NULL;

  if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
    return FALSE;

  GST_OBJECT_LOCK (vfilter);

  vk_shader->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);

  if (!(vert = gst_vulkan_shader_spv_create_shader (vk_shader, vk_shader->vert,
              vk_shader->vert_path, identity_vert, identity_vert_size,
              &error))) {
    goto error;
  }

  if (!(frag = gst_vulkan_shader_spv_create_shader (vk_shader, vk_shader->frag,
              vk_shader->frag_path, identity_frag, identity_frag_size,
              &error))) {
    gst_vulkan_handle_unref (vert);
    goto error;
  }

  if (!gst_vulkan_full_screen_quad_set_shaders (vk_shader->quad, vert, frag)) {
    gst_vulkan_handle_unref (vert);
    gst_vulkan_handle_unref (frag);
    g_set_error (&error, GST_VULKAN_WINDOW_ERROR, FALSE,
        "Failed to set shaders in full screen quad");
    goto error;
  }

  gst_vulkan_handle_unref (vert);
  gst_vulkan_handle_unref (frag);

  GST_OBJECT_UNLOCK (vfilter);

  return TRUE;

error:
  GST_OBJECT_UNLOCK (vfilter);
  if (error->domain == GST_VULKAN_ERROR) {
    GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("Failed to create shader: %s",
            gst_vulkan_result_to_string (error->code)), (NULL));
    GST_DEBUG ("%s", error->message);
  } else {
    GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("Failed to create shader: %s",
            error->message), (NULL));
  }
  return FALSE;
}

static gboolean
gst_vulkan_shader_spv_stop (GstBaseTransform * bt)
{
  GstVulkanShaderSpv *vk_shader = GST_VULKAN_SHADER_SPV (bt);

  gst_clear_object (&vk_shader->quad);

  return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
}

struct ShaderUpdateData
{
  float time;
  float width;
  float height;
};

static inline gboolean
_gst_clock_time_to_double (GstClockTime time, gint64 * result)
{
  if (!GST_CLOCK_TIME_IS_VALID (time))
    return FALSE;

  *result = time;

  return TRUE;
}

static inline gboolean
_gint64_time_val_to_double (gint64 time, gint64 * result)
{
  if (time == -1)
    return FALSE;

  *result = time / GST_SECOND;

  return TRUE;
}

static gboolean
shader_spv_update_time (GstVulkanShaderSpv * shader_spv, GstBuffer * inbuf)
{
  GstMapInfo map_info;
  gint64 time = 0;

  if (!_gst_clock_time_to_double (GST_BUFFER_PTS (inbuf), &time)) {
    if (!_gst_clock_time_to_double (GST_BUFFER_DTS (inbuf), &time))
      _gint64_time_val_to_double (g_get_monotonic_time (), &time);
  }

  if (!gst_memory_map (shader_spv->uniforms, &map_info, GST_MAP_WRITE))
    return FALSE;

  ((struct ShaderUpdateData *) map_info.data)->time = (float) time / GST_SECOND;
  gst_memory_unmap (shader_spv->uniforms, &map_info);

  return TRUE;
}

static GstMemory *
shader_spv_create_uniform (GstVulkanShaderSpv * shader_spv)
{
  GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (shader_spv);

  if (shader_spv->uniforms) {
    return shader_spv->uniforms;
  } else {
    struct ShaderUpdateData data = { 0.0f,
      GST_VIDEO_INFO_WIDTH (&shader_spv->quad->in_info),
      GST_VIDEO_INFO_HEIGHT (&shader_spv->quad->in_info),
    };
    GstMapInfo map_info;
    GstMemory *uniforms;

    uniforms =
        gst_vulkan_buffer_memory_alloc (vfilter->device,
        sizeof (struct ShaderUpdateData),
        VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
        VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
        VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);

    if (!gst_memory_map (uniforms, &map_info, GST_MAP_WRITE))
      return NULL;

    memcpy (map_info.data, &data, sizeof (data));
    gst_memory_unmap (uniforms, &map_info);

    shader_spv->uniforms = uniforms;
    return uniforms;
  }
}

static GstFlowReturn
gst_vulkan_shader_spv_transform (GstBaseTransform * bt, GstBuffer * inbuf,
    GstBuffer * outbuf)
{
  GstVulkanShaderSpv *vk_shader = GST_VULKAN_SHADER_SPV (bt);
  GError *error = NULL;
  GstMemory *uniforms;

  if (!gst_vulkan_full_screen_quad_set_input_buffer (vk_shader->quad, inbuf,
          &error))
    goto error;
  if (!gst_vulkan_full_screen_quad_set_output_buffer (vk_shader->quad, outbuf,
          &error))
    goto error;

  if (!(uniforms = shader_spv_create_uniform (vk_shader)))
    goto error;

  shader_spv_update_time (vk_shader, inbuf);
  if (!gst_vulkan_full_screen_quad_set_uniform_buffer (vk_shader->quad,
          uniforms, &error))
    goto error;

  if (!gst_vulkan_full_screen_quad_draw (vk_shader->quad, &error))
    goto error;

  return GST_FLOW_OK;

error:
  if (error->domain == GST_VULKAN_ERROR) {
    GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("Failed to apply shader: %s",
            gst_vulkan_result_to_string (error->code)), (NULL));
    GST_DEBUG ("%s", error->message);
  } else {
    GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("Failed to apply shader: %s",
            error->message), (NULL));
  }
  g_clear_error (&error);
  return GST_FLOW_ERROR;
}