mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-09-27 14:31:01 +00:00
Code Issues 1 Projects Releases Wiki Activity

vulkan: split vkfullscreenrender into two

Browse source 
Part 1 is a base class (vkvideofilter) that handles instance, device,
queue retrieval and holding that has been moved to the library
Part 2 is a fullscreenrenderquad that is still in the plugin that
performs all of the previous vulkan-specific functionality.
This commit is contained in:
Matthew Waters 2019-11-26 18:11:25 +11:00 committed by GStreamer Merge Bot
parent 756d52ef15
commit 615022ad93
28 changed files with 2323 additions and 2910 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ext/vulkan/meson.build
View file

@ -23,7 +23,7 @@ vulkan_sources = [
'vkdownload.c',
'vkdeviceprovider.c',
'vkelementutils.c',
'vkfullscreenrender.c',
'vkfullscreenquad.c',
'vkimageidentity.c',
'vksink.c',
'vkshader.c',

4
ext/vulkan/shaders/ayuv_to_rgb.frag
View file

@ -6,13 +6,13 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

2
ext/vulkan/shaders/identity.frag
View file

@ -2,7 +2,7 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture;
layout(set = 0, binding = 1) uniform sampler2D inTexture;
layout(location = 0) out vec4 outColor;

6
ext/vulkan/shaders/nv12_to_rgb.frag
View file

@ -6,14 +6,14 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform sampler2D inTexture1;
layout(set = 0, binding = 2) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(set = 0, binding = 2) uniform sampler2D inTexture1;
layout(location = 0) out vec4 outColor0;

4
ext/vulkan/shaders/rgb_to_ayuv.frag
View file

@ -5,13 +5,13 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

4
ext/vulkan/shaders/rgb_to_nv12.frag
View file

@ -5,13 +5,13 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;
layout(location = 1) out vec4 outColor1;

4
ext/vulkan/shaders/rgb_to_yuy2.frag
View file

@ -5,13 +5,13 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

4
ext/vulkan/shaders/swizzle.frag
View file

@ -4,11 +4,11 @@
layout(location = 0) in vec2 inTexCoord;
layout(push_constant) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
};
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

4
ext/vulkan/shaders/swizzle_and_clobber_alpha.frag
View file

@ -4,11 +4,11 @@
layout(location = 0) in vec2 inTexCoord;
layout(push_constant) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
};
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

30
ext/vulkan/shaders/uyvy_to_rgb.frag
View file

@ -1,30 +0,0 @@
#version 450 core
#include "yuy2_uyvy_to_rgb.glsl"
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform YUVCoefficients_ubo { YUVCoefficients coeff; };
layout(set = 0, binding = 2) uniform TexelOrdering
{
vec2 tex_size;
vec2 poffset;
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
} ordering;
layout(location = 0) out vec4 outColor0;
void main()
{
float dx;
if (mod (inTexCoord.x * ordering.tex_size.x, 2.0) < 1.0) {
dx = -ordering.poffset.x;
} else {
dx = 0.0;
}
outColor0 = YUY2_UYVY_to_RGB (inTexture0, inTexCoord, dx, coeff, ordering.in_reorder_idx, ordering.out_reorder_idx);
}

6
ext/vulkan/shaders/view_convert.frag
View file

@ -5,9 +5,7 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D l_tex;
layout(set = 0, binding = 1) uniform sampler2D r_tex;
layout(set = 0, binding = 2) uniform ViewConvert {
layout(set = 0, binding = 0) uniform ViewConvert {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
vec4 tex_offset;
@ -16,6 +14,8 @@ layout(set = 0, binding = 2) uniform ViewConvert {
int output_type;
mat3 downmix[2];
};
layout(set = 0, binding = 1) uniform sampler2D l_tex;
layout(set = 0, binding = 2) uniform sampler2D r_tex;
layout(location = 0) out vec4 outColor0;

4
ext/vulkan/shaders/yuy2_to_rgb.frag
View file

@ -6,13 +6,13 @@
layout(location = 0) in vec2 inTexCoord;
layout(set = 0, binding = 0) uniform sampler2D inTexture0;
layout(set = 0, binding = 1) uniform reorder {
layout(set = 0, binding = 0) uniform reorder {
ivec4 in_reorder_idx;
ivec4 out_reorder_idx;
ivec2 texSize;
ColorMatrices matrices;
};
layout(set = 0, binding = 1) uniform sampler2D inTexture0;
layout(location = 0) out vec4 outColor0;

946
ext/vulkan/vkcolorconvert.c
View file

File diff suppressed because it is too large Load diff

23
ext/vulkan/vkcolorconvert.h
View file

@ -24,7 +24,7 @@
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
#include "vkfullscreenrender.h"
#include "vkfullscreenquad.h"
G_BEGIN_DECLS
@ -41,16 +41,15 @@ typedef struct _GstVulkanColorConvertClass GstVulkanColorConvertClass;
typedef struct _shader_info shader_info;
typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageView ** src_views, GstVulkanImageView ** dest_views, GstVulkanFence * fence);
typedef GstMemory * (*CommandCreateUniformMemory) (GstVulkanColorConvert * conv, shader_info * sinfo, GstVulkanImageView ** src_views, GstVulkanImageView ** dest_views);
struct _shader_info
{
GstVideoFormat from;
GstVideoFormat to;
CommandStateUpdate cmd_state_update;
CommandCreateUniformMemory cmd_create_uniform;
gchar *frag_code;
gsize frag_size;
VkPushConstantRange push_constant_ranges[MAX_PUSH_CONSTANTS];
gsize uniform_size;
GDestroyNotify notify;
gpointer user_data;
@ -58,26 +57,16 @@ struct _shader_info
struct _GstVulkanColorConvert
{
GstVulkanFullScreenRender parent;
GstVulkanVideoFilter parent;
GstVulkanCommandPool *cmd_pool;
VkSampler sampler;
GstVulkanDescriptorCache *descriptor_pool;
VkShaderModule vert_module;
VkShaderModule frag_module;
VkDescriptorSetLayoutBinding sampler_layout_binding;
VkDescriptorSetLayoutCreateInfo layout_info;
GstVulkanFullScreenQuad *quad;
shader_info *current_shader;
GstMemory *uniform;
};
struct _GstVulkanColorConvertClass
{
GstVulkanFullScreenRenderClass parent_class;
GstVulkanVideoFilterClass parent_class;
};
GType gst_vulkan_color_convert_get_type(void);

1325
ext/vulkan/vkfullscreenquad.c Normal file
View file

File diff suppressed because it is too large Load diff

96
ext/vulkan/vkfullscreenquad.h Normal file
View file

@ -0,0 +1,96 @@
/*
* GStreamer Plugins
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_VULKAN_FULL_SCREEN_QUAD_H__
#define __GST_VULKAN_FULL_SCREEN_QUAD_H__
#include <gst/gst.h>
#include <gst/vulkan/vulkan.h>
G_BEGIN_DECLS
GType gst_vulkan_full_screen_quad_get_type (void);
#define GST_TYPE_VULKAN_FULL_SCREEN_QUAD (gst_vulkan_full_screen_quad_get_type ())
#define GST_VULKAN_FULL_SCREEN_QUAD(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuad))
#define GST_VULKAN_FULL_SCREEN_QUAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuadClass))
#define GST_IS_VULKAN_FULL_SCREEN_QUAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD))
#define GST_IS_VULKAN_FULL_SCREEN_QUAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_VULKAN_FULL_SCREEN_QUAD))
#define GST_VULKAN_FULL_SCREEN_QUAD_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_VULKAN_FULL_SCREEN_QUAD, GstVulkanFullScreenQuadClass))
/* XXX: privatise this on moving to lib */
struct Vertex
{
float x, y, z;
float s, t;
};
typedef struct _GstVulkanFullScreenQuad GstVulkanFullScreenQuad;
typedef struct _GstVulkanFullScreenQuadClass GstVulkanFullScreenQuadClass;
typedef struct _GstVulkanFullScreenQuadPrivate GstVulkanFullScreenQuadPrivate;
struct _GstVulkanFullScreenQuad
{
GstObject parent;
GstVideoInfo out_info;
GstVideoInfo in_info;
GstVulkanQueue *queue;
GstVulkanHandle *render_pass;
GstVulkanHandle *pipeline_layout;
GstVulkanHandle *graphics_pipeline;
GstVulkanHandle *descriptor_set_layout;
GstVulkanDescriptorCache *descriptor_cache;
GstVulkanDescriptorSet *descriptor_set;
GstVulkanHandle *framebuffer;
GstVulkanHandle *sampler;
GstVulkanCommandPool *cmd_pool;
GstVulkanTrashList *trash_list;
GstVulkanFence *last_fence;
};
struct _GstVulkanFullScreenQuadClass
{
GstObjectClass parent_class;
};
GstVulkanFullScreenQuad * gst_vulkan_full_screen_quad_new (GstVulkanQueue * queue);
gboolean gst_vulkan_full_screen_quad_set_info (GstVulkanFullScreenQuad * self, GstVideoInfo *in_info, GstVideoInfo * out_info);
gboolean gst_vulkan_full_screen_quad_set_shaders (GstVulkanFullScreenQuad * self, GstVulkanHandle * vert, GstVulkanHandle * frag);
gboolean gst_vulkan_full_screen_quad_set_uniform_buffer (GstVulkanFullScreenQuad * self, GstMemory * uniforms, GError ** error);
gboolean gst_vulkan_full_screen_quad_set_input_buffer (GstVulkanFullScreenQuad * self, GstBuffer * buffer, GError ** error);
gboolean gst_vulkan_full_screen_quad_set_output_buffer (GstVulkanFullScreenQuad * self, GstBuffer * buffer, GError ** error);
gboolean gst_vulkan_full_screen_quad_prepare_draw (GstVulkanFullScreenQuad * self, GstVulkanFence * fence, GError ** error);
gboolean gst_vulkan_full_screen_quad_fill_command_buffer (GstVulkanFullScreenQuad * self, GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error);
gboolean gst_vulkan_full_screen_quad_submit (GstVulkanFullScreenQuad * self, GstVulkanCommandBuffer * cmd, GstVulkanFence * fence, GError ** error);
gboolean gst_vulkan_full_screen_quad_draw (GstVulkanFullScreenQuad * self, GError ** error);
GstVulkanFence * gst_vulkan_full_screen_quad_get_last_fence (GstVulkanFullScreenQuad * self);
G_END_DECLS
#endif /* __GST_VULKAN_FULL_SCREEN_QUAD_H__ */

921
ext/vulkan/vkfullscreenrender.c
View file

@ -1,921 +0,0 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-vulkanimageidentity
* @title: vulkanimgeidentity
*
* vulkanimageidentity produces a vulkan image that is a copy of the input image.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <string.h>
#include "vkimageidentity.h"
#include "vkshader.h"
GST_DEBUG_CATEGORY (gst_debug_vulkan_full_screen_render);
#define GST_CAT_DEFAULT gst_debug_vulkan_full_screen_render
struct Vertex vertices[] = {
{-1.0f, -1.0f, 0.0f, 0.0f, 0.0f},
{1.0f, -1.0f, 0.0f, 1.0f, 0.0f},
{1.0f, 1.0f, 0.0f, 1.0f, 1.0f},
{-1.0f, 1.0f, 0.0f, 0.0f, 1.0f},
};
gushort indices[] = {
0, 1, 2, 0, 2, 3,
};
static void gst_vulkan_full_screen_render_finalize (GObject * object);
static gboolean gst_vulkan_full_screen_render_query (GstBaseTransform * bt,
GstPadDirection direction, GstQuery * query);
static void gst_vulkan_full_screen_render_set_context (GstElement * element,
GstContext * context);
static gboolean gst_vulkan_full_screen_render_start (GstBaseTransform * bt);
static gboolean gst_vulkan_full_screen_render_stop (GstBaseTransform * bt);
static gboolean gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps);
static GstCaps *gst_vulkan_full_screen_render_transform_caps (GstBaseTransform *
bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
static gboolean
gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt,
GstQuery * decide_query, GstQuery * query);
static gboolean
gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt,
GstQuery * query);
#define IMAGE_FORMATS " { BGRA }"
static GstStaticPadTemplate gst_vulkan_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
IMAGE_FORMATS)));
static GstStaticPadTemplate gst_vulkan_src_template =
GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,
IMAGE_FORMATS)));
enum
{
PROP_0,
};
enum
{
SIGNAL_0,
LAST_SIGNAL
};
/* static guint gst_vulkan_full_screen_render_signals[LAST_SIGNAL] = { 0 }; */
#define gst_vulkan_full_screen_render_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanFullScreenRender,
gst_vulkan_full_screen_render, GST_TYPE_BASE_TRANSFORM,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_full_screen_render,
"vulkanimageidentity", 0, "Vulkan Image identity"));
static void
gst_vulkan_full_screen_render_class_init (GstVulkanFullScreenRenderClass *
klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader",
"Filter/Video", "A Vulkan image copier",
"Matthew Waters <matthew@centricular.com>");
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_sink_template);
gst_element_class_add_static_pad_template (gstelement_class,
&gst_vulkan_src_template);
gobject_class->finalize = gst_vulkan_full_screen_render_finalize;
gstelement_class->set_context = gst_vulkan_full_screen_render_set_context;
gstbasetransform_class->start =
GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_start);
gstbasetransform_class->stop =
GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_stop);
gstbasetransform_class->query =
GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_query);
gstbasetransform_class->set_caps = gst_vulkan_full_screen_render_set_caps;
gstbasetransform_class->transform_caps =
gst_vulkan_full_screen_render_transform_caps;
gstbasetransform_class->propose_allocation =
gst_vulkan_full_screen_render_propose_allocation;
gstbasetransform_class->decide_allocation =
gst_vulkan_full_screen_render_decide_allocation;
}
static void
gst_vulkan_full_screen_render_init (GstVulkanFullScreenRender * render)
{
}
static void
gst_vulkan_full_screen_render_finalize (GObject * object)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (object);
gst_caps_replace (&render->in_caps, NULL);
gst_caps_replace (&render->out_caps, NULL);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static gboolean
gst_vulkan_full_screen_render_query (GstBaseTransform * bt,
GstPadDirection direction, GstQuery * query)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:{
if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query,
NULL, render->instance, render->device))
return TRUE;
if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render),
query, render->queue))
return TRUE;
break;
}
default:
break;
}
return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query);
}
static void
gst_vulkan_full_screen_render_set_context (GstElement * element,
GstContext * context)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (element);
gst_vulkan_handle_set_context (element, context, NULL, &render->instance);
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
struct choose_data
{
GstVulkanFullScreenRender *upload;
GstVulkanQueue *queue;
};
static gboolean
_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue,
struct choose_data *data)
{
guint flags =
device->physical_device->queue_family_props[queue->family].queueFlags;
GST_ERROR ("flags 0x%x", flags);
if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) {
if (data->queue)
gst_object_unref (data->queue);
data->queue = gst_object_ref (queue);
return FALSE;
}
return TRUE;
}
static GstVulkanQueue *
_find_graphics_queue (GstVulkanFullScreenRender * upload)
{
struct choose_data data;
data.upload = upload;
data.queue = NULL;
gst_vulkan_device_foreach_queue (upload->device,
(GstVulkanDeviceForEachQueueFunc) _choose_queue, &data);
return data.queue;
}
static GstCaps *
gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
GstCaps *result, *tmp;
tmp = gst_caps_copy (caps);
if (filter) {
result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (tmp);
} else {
result = tmp;
}
return result;
}
static void
clear_shader_create_info (GstVulkanFullScreenRender * render)
{
if (render->shader_create_info) {
if (render->destroy_shader_create_info)
render->destroy_shader_create_info (render, render->shader_create_info);
}
render->n_shader_stages = 0;
render->shader_create_info = NULL;
render->destroy_shader_create_info = NULL;
}
static VkPipeline
_create_pipeline (GstVulkanFullScreenRender * render)
{
GstVulkanFullScreenRenderClass *render_class =
GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
VkVertexInputBindingDescription vertex_binding_description;
VkVertexInputAttributeDescription vertex_attribute_description[2];
VkPipelineVertexInputStateCreateInfo vertex_input_info;
VkPipelineInputAssemblyStateCreateInfo input_assembly;
VkPipelineViewportStateCreateInfo viewport_state;
VkPipelineRasterizationStateCreateInfo rasterizer;
VkPipelineMultisampleStateCreateInfo multisampling;
VkPipelineColorBlendAttachmentState
color_blend_attachments[GST_VIDEO_MAX_PLANES];
VkPipelineColorBlendStateCreateInfo color_blending;
VkGraphicsPipelineCreateInfo pipeline_info;
VkPipeline pipeline;
GError *error = NULL;
VkResult err;
render_class->shader_create_info (render);
/* *INDENT-OFF* */
vertex_binding_description = (VkVertexInputBindingDescription) {
.binding = 0,
.stride = sizeof (struct Vertex),
.inputRate = VK_VERTEX_INPUT_RATE_VERTEX
};
vertex_attribute_description[0] = (VkVertexInputAttributeDescription) {
.binding = 0,
.location = 0,
.format = VK_FORMAT_R32G32B32_SFLOAT,
.offset = G_STRUCT_OFFSET (struct Vertex, x)
};
vertex_attribute_description[1] = (VkVertexInputAttributeDescription) {
.binding = 0,
.location = 1,
.format = VK_FORMAT_R32G32_SFLOAT,
.offset = G_STRUCT_OFFSET (struct Vertex, s)
};
vertex_input_info = (VkPipelineVertexInputStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO,
.pNext = NULL,
.vertexBindingDescriptionCount = 1,
.pVertexBindingDescriptions = &vertex_binding_description,
.vertexAttributeDescriptionCount = G_N_ELEMENTS (vertex_attribute_description),
.pVertexAttributeDescriptions = vertex_attribute_description
};
input_assembly = (VkPipelineInputAssemblyStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO,
.pNext = NULL,
.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST,
.primitiveRestartEnable = VK_FALSE
};
viewport_state = (VkPipelineViewportStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO,
.pNext = NULL,
.viewportCount = 1,
.pViewports = &(VkViewport) {
.x = 0.0f,
.y = 0.0f,
.width = (float) GST_VIDEO_INFO_WIDTH (&render->out_info),
.height = (float) GST_VIDEO_INFO_HEIGHT (&render->out_info),
.minDepth = 0.0f,
.maxDepth = 1.0f
},
.scissorCount = 1,
.pScissors = &(VkRect2D) {
.offset = { 0, 0 },
.extent = {
GST_VIDEO_INFO_WIDTH (&render->out_info),
GST_VIDEO_INFO_HEIGHT (&render->out_info)
}
}
};
rasterizer = (VkPipelineRasterizationStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO,
.pNext = NULL,
.depthClampEnable = VK_FALSE,
.rasterizerDiscardEnable = VK_FALSE,
.polygonMode = VK_POLYGON_MODE_FILL,
.lineWidth = 1.0f,
.cullMode = VK_CULL_MODE_BACK_BIT,
.frontFace = VK_FRONT_FACE_CLOCKWISE,
.depthBiasEnable = VK_FALSE
};
multisampling = (VkPipelineMultisampleStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO,
.pNext = NULL,
.sampleShadingEnable = VK_FALSE,
.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT
};
color_blend_attachments[0] = (VkPipelineColorBlendAttachmentState) {
.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
.blendEnable = VK_FALSE
};
color_blend_attachments[1] = (VkPipelineColorBlendAttachmentState) {
.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
.blendEnable = VK_FALSE
};
color_blend_attachments[2] = (VkPipelineColorBlendAttachmentState) {
.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
.blendEnable = VK_FALSE
};
color_blend_attachments[3] = (VkPipelineColorBlendAttachmentState) {
.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT,
.blendEnable = VK_FALSE
};
color_blending = (VkPipelineColorBlendStateCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO,
.pNext = NULL,
.logicOpEnable = VK_FALSE,
.logicOp = VK_LOGIC_OP_COPY,
.attachmentCount = GST_VIDEO_INFO_N_PLANES (&render->out_info),
.pAttachments = color_blend_attachments,
.blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f }
};
pipeline_info = (VkGraphicsPipelineCreateInfo) {
.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO,
.pNext = NULL,
.stageCount = render->n_shader_stages,
.pStages = render->shader_create_info,
.pVertexInputState = &vertex_input_info,
.pInputAssemblyState = &input_assembly,
.pViewportState = &viewport_state,
.pRasterizationState = &rasterizer,
.pMultisampleState = &multisampling,
.pColorBlendState = &color_blending,
.layout = render->pipeline_layout,
.renderPass = render->render_pass,
.subpass = 0,
.basePipelineHandle = VK_NULL_HANDLE
};
/* *INDENT-ON* */
err =
vkCreateGraphicsPipelines (render->device->device, VK_NULL_HANDLE, 1,
&pipeline_info, NULL, &pipeline);
clear_shader_create_info (render);
if (gst_vulkan_error_to_g_error (err, &error,
"vkCreateGraphicsPipelines") < 0) {
GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return pipeline;
}
static VkPipelineLayout
_create_pipeline_layout (GstVulkanFullScreenRender * render)
{
GstVulkanFullScreenRenderClass *render_class =
GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
VkPipelineLayoutCreateInfo pipeline_layout_info;
VkPipelineLayout pipeline_layout;
VkPushConstantRange *constants = NULL;
guint n_constants = 0;
GError *error = NULL;
VkResult err;
if (render_class->push_constant_ranges)
constants = render_class->push_constant_ranges (render, &n_constants);
/* *INDENT-OFF* */
pipeline_layout_info = (VkPipelineLayoutCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO,
.pNext = NULL,
.setLayoutCount = 1,
.pSetLayouts = (VkDescriptorSetLayout *) &render->descriptor_set_layout->handle,
.pushConstantRangeCount = n_constants,
.pPushConstantRanges = constants,
};
/* *INDENT-ON* */
err =
vkCreatePipelineLayout (render->device->device,
&pipeline_layout_info, NULL, &pipeline_layout);
g_free (constants);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreatePipelineLayout") < 0) {
GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return pipeline_layout;
}
static VkRenderPass
_create_render_pass (GstVulkanFullScreenRender * render)
{
GstVulkanFullScreenRenderClass *render_class =
GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
guint n_descriptions;
VkAttachmentDescription *descriptions =
render_class->render_pass_attachment_descriptions (render,
&n_descriptions);
guint n_refs;
VkAttachmentReference *color_attachment_refs =
render_class->render_pass_attachment_references (render, &n_refs);
/* *INDENT-OFF* */
VkSubpassDescription subpass = {
.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS,
.colorAttachmentCount = n_refs,
.pColorAttachments = color_attachment_refs
};
VkRenderPassCreateInfo render_pass_info = {
.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO,
.pNext = NULL,
.attachmentCount = n_descriptions,
.pAttachments = descriptions,
.subpassCount = 1,
.pSubpasses = &subpass
};
/* *INDENT-ON* */
VkRenderPass render_pass = VK_NULL_HANDLE;
VkResult err;
GError *error = NULL;
err =
vkCreateRenderPass (render->device->device, &render_pass_info, NULL,
&render_pass);
g_free (color_attachment_refs);
g_free (descriptions);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateRenderPass") < 0) {
GST_ERROR_OBJECT (render, "Failed to create renderpass: %s",
error->message);
return VK_NULL_HANDLE;
}
return render_pass;
}
static GstVulkanHandle *
_create_descriptor_set_layout (GstVulkanFullScreenRender * render)
{
GstVulkanFullScreenRenderClass *render_class =
GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render);
guint n_bindings;
VkDescriptorSetLayoutBinding *bindings =
render_class->descriptor_set_layout_bindings (render, &n_bindings);
/* *INDENT-OFF* */
VkDescriptorSetLayoutCreateInfo layout_info = {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = NULL,
.bindingCount = n_bindings,
.pBindings = bindings
};
/* *INDENT-ON* */
VkDescriptorSetLayout descriptor_set_layout;
GstVulkanHandle *ret;
VkResult err;
GError *error = NULL;
err =
vkCreateDescriptorSetLayout (render->device->device, &layout_info,
NULL, &descriptor_set_layout);
g_free (bindings);
if (gst_vulkan_error_to_g_error (err, &error,
"vkCreateDescriptorSetLayout") < 0) {
GST_ERROR_OBJECT (render, "Failed to create renderpass: %s",
error->message);
return VK_NULL_HANDLE;
}
ret = gst_vulkan_handle_new_wrapped (render->device,
GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT,
(GstVulkanHandleTypedef) descriptor_set_layout,
gst_vulkan_handle_free_descriptor_set_layout, NULL);
return ret;
}
static gboolean
gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstVulkanFence *last_fence;
if (!gst_video_info_from_caps (&render->in_info, in_caps))
return FALSE;
if (!gst_video_info_from_caps (&render->out_info, out_caps))
return FALSE;
gst_caps_replace (&render->in_caps, in_caps);
gst_caps_replace (&render->out_caps, out_caps);
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (render->descriptor_set_layout) {
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (last_fence,
(GstMiniObject *) render->descriptor_set_layout));
render->descriptor_set_layout = NULL;
}
if (render->pipeline_layout) {
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_pipeline_layout (last_fence,
render->pipeline_layout));
render->pipeline_layout = VK_NULL_HANDLE;
}
if (render->render_pass) {
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_render_pass (last_fence,
render->render_pass));
render->render_pass = VK_NULL_HANDLE;
}
if (render->graphics_pipeline) {
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_pipeline (last_fence,
render->graphics_pipeline));
render->graphics_pipeline = VK_NULL_HANDLE;
}
gst_vulkan_fence_unref (last_fence);
if (!(render->descriptor_set_layout = _create_descriptor_set_layout (render)))
return FALSE;
if (!(render->pipeline_layout = _create_pipeline_layout (render)))
return FALSE;
if (!(render->render_pass = _create_render_pass (render)))
return FALSE;
if (!(render->graphics_pipeline = _create_pipeline (render)))
return FALSE;
GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps);
return TRUE;
}
static gboolean
gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt,
GstQuery * decide_query, GstQuery * query)
{
/* FIXME: */
return FALSE;
}
static gboolean
gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt,
GstQuery * query)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstBufferPool *pool = NULL;
GstStructure *config;
GstCaps *caps;
guint min, max, size;
gboolean update_pool;
gst_query_parse_allocation (query, &caps, NULL);
if (!caps)
return FALSE;
if (gst_query_get_n_allocation_pools (query) > 0) {
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
update_pool = TRUE;
} else {
GstVideoInfo vinfo;
gst_video_info_init (&vinfo);
gst_video_info_from_caps (&vinfo, caps);
size = vinfo.size;
min = max = 0;
update_pool = FALSE;
}
if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) {
if (pool)
gst_object_unref (pool);
pool = gst_vulkan_image_buffer_pool_new (render->device);
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
gst_buffer_pool_set_config (pool, config);
if (update_pool)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
gst_object_unref (pool);
return TRUE;
}
static gboolean
_create_vertex_buffers (GstVulkanFullScreenRender * render)
{
GstMapInfo map_info;
render->vertices =
gst_vulkan_buffer_memory_alloc (render->device, sizeof (vertices),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (!gst_memory_map (render->vertices, &map_info, GST_MAP_WRITE)) {
gst_memory_unref (render->vertices);
render->vertices = NULL;
return FALSE;
}
memcpy (map_info.data, vertices, sizeof (vertices));
gst_memory_unmap (render->vertices, &map_info);
render->indices =
gst_vulkan_buffer_memory_alloc (render->device, sizeof (indices),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (!gst_memory_map (render->indices, &map_info, GST_MAP_WRITE)) {
gst_memory_unref (render->vertices);
render->vertices = NULL;
gst_memory_unref (render->indices);
render->indices = NULL;
return FALSE;
}
memcpy (map_info.data, indices, sizeof (indices));
gst_memory_unmap (render->indices, &map_info);
return TRUE;
}
static gboolean
gst_vulkan_full_screen_render_start (GstBaseTransform * bt)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL,
&render->instance)) {
GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
("Failed to retrieve vulkan instance"), (NULL));
return FALSE;
}
if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render),
&render->device)) {
GError *error = NULL;
GST_DEBUG_OBJECT (render, "No device retrieved from peer elements");
if (!(render->device =
gst_vulkan_instance_create_device (render->instance, &error))) {
GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
("Failed to create vulkan device"), ("%s", error->message));
g_clear_error (&error);
return FALSE;
}
}
if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render),
&render->queue)) {
GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements");
render->queue = _find_graphics_queue (render);
}
if (!render->queue)
return FALSE;
if (!_create_vertex_buffers (render))
return FALSE;
render->trash_list = gst_vulkan_trash_fence_list_new ();
return TRUE;
}
static gboolean
gst_vulkan_full_screen_render_stop (GstBaseTransform * bt)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
if (render->device) {
GstVulkanFence *last_fence;
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (render->graphics_pipeline)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_pipeline (last_fence,
render->graphics_pipeline));
render->graphics_pipeline = VK_NULL_HANDLE;
if (render->pipeline_layout)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_pipeline_layout (last_fence,
render->pipeline_layout));
render->pipeline_layout = VK_NULL_HANDLE;
if (render->render_pass)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_render_pass (last_fence,
render->render_pass));
render->render_pass = VK_NULL_HANDLE;
if (render->descriptor_set_layout)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (last_fence,
(GstMiniObject *) render->descriptor_set_layout));
render->descriptor_set_layout = NULL;
gst_vulkan_fence_unref (last_fence);
if (render->last_fence)
gst_vulkan_fence_unref (render->last_fence);
render->last_fence = NULL;
if (!gst_vulkan_trash_list_wait (render->trash_list, -1))
GST_WARNING_OBJECT (render,
"Failed to wait for all resources to be freed");
gst_object_unref (render->trash_list);
render->trash_list = NULL;
if (render->vertices)
gst_memory_unref (render->vertices);
render->vertices = NULL;
if (render->indices)
gst_memory_unref (render->indices);
render->indices = NULL;
gst_object_unref (render->device);
}
render->device = NULL;
if (render->queue)
gst_object_unref (render->queue);
render->queue = NULL;
if (render->instance)
gst_object_unref (render->instance);
render->instance = NULL;
return TRUE;
}
/**
* gst_vulkan_full_screen_render_fill_command_buffer:
* @render: a #GstVulkanFullScreenRender
* @cmd: a `VkCommandBuffer`
* @framebuffer: a `VkFramebuffer`
*
* Returns: whether @cmd could be filled with the commands necessary to render
*/
gboolean
gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender *
render, VkCommandBuffer cmd, VkFramebuffer framebuffer)
{
/* *INDENT-OFF* */
VkClearValue clearColor = {{{ 0.0f, 0.0f, 0.0f, 1.0f }}};
VkClearValue clearColors[GST_VIDEO_MAX_PLANES] = {
clearColor, clearColor, clearColor, clearColor,
};
VkRenderPassBeginInfo render_pass_info = {
.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
.renderPass = render->render_pass,
.framebuffer = framebuffer,
.renderArea.offset = { 0, 0 },
.renderArea.extent = {
GST_VIDEO_INFO_WIDTH (&render->out_info),
GST_VIDEO_INFO_HEIGHT (&render->out_info)
},
.clearValueCount = GST_VIDEO_INFO_N_PLANES (&render->out_info),
.pClearValues = clearColors,
};
/* *INDENT-ON* */
VkDeviceSize offsets[] = { 0 };
vkCmdBeginRenderPass (cmd, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE);
vkCmdBindPipeline (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
render->graphics_pipeline);
vkCmdBindVertexBuffers (cmd, 0, 1,
&((GstVulkanBufferMemory *) render->vertices)->buffer, offsets);
vkCmdBindIndexBuffer (cmd,
((GstVulkanBufferMemory *) render->indices)->buffer, 0,
VK_INDEX_TYPE_UINT16);
vkCmdDrawIndexed (cmd, G_N_ELEMENTS (indices), 1, 0, 0, 0);
vkCmdEndRenderPass (cmd);
return TRUE;
}
gboolean
gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render,
VkCommandBuffer cmd, GstVulkanFence * fence)
{
VkSubmitInfo submit_info;
GError *error = NULL;
VkResult err;
if (!fence)
fence = gst_vulkan_fence_new (render->device, 0, &error);
if (!fence)
goto error;
/* *INDENT-OFF* */
submit_info = (VkSubmitInfo) {
.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO,
.pNext = NULL,
.waitSemaphoreCount = 0,
.pWaitSemaphores = NULL,
.pWaitDstStageMask = NULL,
.commandBufferCount = 1,
.pCommandBuffers = &cmd,
.signalSemaphoreCount = 0,
.pSignalSemaphores = NULL,
};
/* *INDENT-ON* */
if (render->last_fence)
gst_vulkan_fence_unref (render->last_fence);
render->last_fence = gst_vulkan_fence_ref (fence);
gst_vulkan_queue_submit_lock (render->queue);
err =
vkQueueSubmit (render->queue->queue, 1, &submit_info,
GST_VULKAN_FENCE_FENCE (fence));
gst_vulkan_queue_submit_unlock (render->queue);
if (gst_vulkan_error_to_g_error (err, &error, "vkQueueSubmit") < 0)
goto error;
gst_vulkan_trash_list_gc (render->trash_list);
gst_vulkan_fence_unref (fence);
return TRUE;
error:
GST_ELEMENT_ERROR (render, LIBRARY, FAILED, ("%s", error->message), (NULL));
g_clear_error (&error);
return FALSE;
}

96
ext/vulkan/vkfullscreenrender.h
View file

@ -1,96 +0,0 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _VK_FULL_SCREEN_RENDER_H_
#define _VK_FULL_SCREEN_RENDER_H_
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
G_BEGIN_DECLS
#define GST_TYPE_VULKAN_FULL_SCREEN_RENDER (gst_vulkan_full_screen_render_get_type())
#define GST_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRender))
#define GST_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass))
#define GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass))
#define GST_IS_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER))
#define GST_IS_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER))
typedef struct _GstVulkanFullScreenRender GstVulkanFullScreenRender;
typedef struct _GstVulkanFullScreenRenderClass GstVulkanFullScreenRenderClass;
typedef void (*GstVulkanFullScreenRenderDestroyInfoNotify)
(GstVulkanFullScreenRender * render, gpointer info);
struct Vertex
{
gfloat x, y, z;
gfloat s, t;
};
struct _GstVulkanFullScreenRender
{
GstBaseTransform parent;
GstVulkanInstance *instance;
GstVulkanDevice *device;
GstVulkanQueue *queue;
GstCaps *in_caps;
GstVideoInfo in_info;
GstCaps *out_caps;
GstVideoInfo out_info;
guint n_shader_stages;
VkPipelineShaderStageCreateInfo *shader_create_info;
GstVulkanFullScreenRenderDestroyInfoNotify destroy_shader_create_info;
VkRenderPass render_pass;
VkPipelineLayout pipeline_layout;
VkPipeline graphics_pipeline;
GstVulkanHandle *descriptor_set_layout;
GstMemory *vertices;
GstMemory *indices;
GstVulkanTrashList *trash_list;
GstVulkanFence *last_fence;
};
struct _GstVulkanFullScreenRenderClass
{
GstBaseTransformClass video_sink_class;
void (*shader_create_info) (GstVulkanFullScreenRender * render);
VkDescriptorSetLayoutBinding * (*descriptor_set_layout_bindings) (GstVulkanFullScreenRender * render, guint * n_bindings);
VkAttachmentReference * (*render_pass_attachment_references) (GstVulkanFullScreenRender * render, guint * n_refs);
VkAttachmentDescription * (*render_pass_attachment_descriptions) (GstVulkanFullScreenRender * render, guint * n_descriptions);
VkPushConstantRange * (*push_constant_ranges) (GstVulkanFullScreenRender * render, guint * n_constants);
};
gboolean gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, VkFramebuffer framebuffer);
gboolean gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, GstVulkanFence *fence);
GType gst_vulkan_full_screen_render_get_type(void);
G_END_DECLS
#endif

543
ext/vulkan/vkimageidentity.c
View file

@ -44,26 +44,11 @@ GST_DEBUG_CATEGORY (gst_debug_vulkan_image_identity);
static gboolean gst_vulkan_image_identity_start (GstBaseTransform * bt);
static gboolean gst_vulkan_image_identity_stop (GstBaseTransform * bt);
static GstCaps *gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter);
static GstFlowReturn gst_vulkan_image_identity_transform (GstBaseTransform * bt,
GstBuffer * inbuf, GstBuffer * outbuf);
static gboolean gst_vulkan_image_identity_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps);
static VkAttachmentReference
* gst_vulkan_image_identity_render_pass_attachment_references
(GstVulkanFullScreenRender * render, guint * n_attachments);
static VkAttachmentDescription
* gst_vulkan_image_identity_render_pass_attachment_descriptions
(GstVulkanFullScreenRender * render, guint * n_descriptions);
static VkDescriptorSetLayoutBinding
* gst_vulkan_image_identity_descriptor_set_layout_bindings
(GstVulkanFullScreenRender * render, guint * n_bindings);
static void
gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender *
render);
#define IMAGE_FORMATS " { BGRA }"
static GstStaticPadTemplate gst_vulkan_sink_template =
@ -97,7 +82,7 @@ enum
#define gst_vulkan_image_identity_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanImageIdentity, gst_vulkan_image_identity,
GST_TYPE_VULKAN_FULL_SCREEN_RENDER,
GST_TYPE_VULKAN_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_image_identity,
"vulkanimageidentity", 0, "Vulkan Image identity"));
@ -106,13 +91,11 @@ gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass)
{
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
GstVulkanFullScreenRenderClass *fullscreenrender_class;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass;
gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader",
gst_element_class_set_metadata (gstelement_class, "Vulkan Image Identity",
"Filter/Video", "A Vulkan image copier",
"Matthew Waters <matthew@centricular.com>");
@ -125,19 +108,8 @@ gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass)
GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_start);
gstbasetransform_class->stop =
GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_stop);
gstbasetransform_class->transform_caps =
gst_vulkan_image_identity_transform_caps;
gstbasetransform_class->set_caps = gst_vulkan_image_identity_set_caps;
gstbasetransform_class->transform = gst_vulkan_image_identity_transform;
fullscreenrender_class->render_pass_attachment_references =
gst_vulkan_image_identity_render_pass_attachment_references;
fullscreenrender_class->render_pass_attachment_descriptions =
gst_vulkan_image_identity_render_pass_attachment_descriptions;
fullscreenrender_class->descriptor_set_layout_bindings =
gst_vulkan_image_identity_descriptor_set_layout_bindings;
fullscreenrender_class->shader_create_info =
gst_vulkan_image_identity_shader_create_info;
}
static void
@ -145,535 +117,86 @@ gst_vulkan_image_identity_init (GstVulkanImageIdentity * vk_identity)
{
}
static GstCaps *
gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
GstCaps *result, *tmp;
tmp = gst_caps_copy (caps);
if (filter) {
result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (tmp);
} else {
result = tmp;
}
return result;
}
static void
destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data)
{
VkPipelineShaderStageCreateInfo *info = data;
int i;
for (i = 0; i < render->n_shader_stages; i++) {
vkDestroyShaderModule (render->device->device, info[i].module, NULL);
}
g_free (info);
}
static void
gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender *
render)
{
VkShaderModule vert_module, frag_module;
vert_module =
_vk_create_shader (render->device, identity_vert, identity_vert_size,
NULL);
frag_module =
_vk_create_shader (render->device, identity_frag, identity_frag_size,
NULL);
render->n_shader_stages = 2;
render->shader_create_info =
g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages);
render->destroy_shader_create_info = destroy_shader_create_info;
/* *INDENT-OFF* */
render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.pNext = NULL,
.stage = VK_SHADER_STAGE_VERTEX_BIT,
.module = vert_module,
.pName = "main"
};
render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.pNext = NULL,
.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
.module = frag_module,
.pName = "main"
};
}
static VkDescriptorSetLayoutBinding *
gst_vulkan_image_identity_descriptor_set_layout_bindings (GstVulkanFullScreenRender * render, guint * n_bindings)
{
VkDescriptorSetLayoutBinding *bindings;
*n_bindings = 1;
bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings);
/* *INDENT-OFF* */
bindings[0] = (VkDescriptorSetLayoutBinding) {
.binding = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
};
/* *INDENT-ON* */
return bindings;
}
static VkAttachmentReference
* gst_vulkan_image_identity_render_pass_attachment_references
(GstVulkanFullScreenRender * render, guint * n_attachments)
{
VkAttachmentReference *attachments;
*n_attachments = 1;
attachments = g_new0 (VkAttachmentReference, *n_attachments);
/* *INDENT-OFF* */
attachments[0] = (VkAttachmentReference) {
.attachment = 0,
.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
};
/* *INDENT-ON* */
return attachments;
}
static VkAttachmentDescription
* gst_vulkan_image_identity_render_pass_attachment_descriptions
(GstVulkanFullScreenRender * render, guint * n_descriptions)
{
VkAttachmentDescription *color_attachments;
*n_descriptions = 1;
color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions);
/* *INDENT-OFF* */
color_attachments[0] = (VkAttachmentDescription) {
.format = gst_vulkan_format_from_video_info (&render->in_info, 0),
.samples = VK_SAMPLE_COUNT_1_BIT,
.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR,
.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE,
/* FIXME: share this between elements to avoid pipeline barriers */
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
};
/* *INDENT-ON* */
return color_attachments;
}
static GstVulkanDescriptorCache *
_create_descriptor_pool (GstVulkanImageIdentity * vk_identity)
{
GstVulkanFullScreenRender *render =
GST_VULKAN_FULL_SCREEN_RENDER (vk_identity);
guint max_sets = 32; /* FIXME: Don't hardcode this! */
/* *INDENT-OFF* */
VkDescriptorPoolSize pool_sizes = {
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.descriptorCount = max_sets
};
VkDescriptorPoolCreateInfo pool_info = {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
.pNext = NULL,
.poolSizeCount = 1,
.pPoolSizes = &pool_sizes,
.maxSets = max_sets
};
/* *INDENT-ON* */
VkDescriptorPool pool;
GstVulkanDescriptorPool *ret;
GstVulkanDescriptorCache *cache;
GError *error = NULL;
VkResult err;
err =
vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) {
GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s",
error->message);
g_clear_error (&error);
return NULL;
}
ret = gst_vulkan_descriptor_pool_new_wrapped (render->device, pool, max_sets);
cache =
gst_vulkan_descriptor_cache_new (ret, 1, &render->descriptor_set_layout);
gst_object_unref (ret);
return cache;
}
static gboolean
gst_vulkan_image_identity_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
GstCaps * out_caps)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt);
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstVulkanFence *last_fence;
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
out_caps))
return FALSE;
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (vk_identity->descriptor_pool)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_object_unref (last_fence,
(GstObject *) vk_identity->descriptor_pool));
vk_identity->descriptor_pool = NULL;
gst_vulkan_fence_unref (last_fence);
if (!(vk_identity->descriptor_pool = _create_descriptor_pool (vk_identity)))
if (!gst_vulkan_full_screen_quad_set_info (vk_identity->quad,
&vfilter->in_info, &vfilter->out_info))
return FALSE;
return TRUE;
}
static VkSampler
_create_sampler (GstVulkanImageIdentity * vk_identity)
{
GstVulkanFullScreenRender *render =
GST_VULKAN_FULL_SCREEN_RENDER (vk_identity);
/* *INDENT-OFF* */
VkSamplerCreateInfo samplerInfo = {
.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
.magFilter = VK_FILTER_LINEAR,
.minFilter = VK_FILTER_LINEAR,
.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.anisotropyEnable = VK_FALSE,
.maxAnisotropy = 1,
.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK,
.unnormalizedCoordinates = VK_FALSE,
.compareEnable = VK_FALSE,
.compareOp = VK_COMPARE_OP_ALWAYS,
.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR,
.mipLodBias = 0.0f,
.minLod = 0.0f,
.maxLod = 0.0f
};
/* *INDENT-ON* */
GError *error = NULL;
VkSampler sampler;
VkResult err;
err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) {
GST_ERROR_OBJECT (vk_identity, "Failed to create sampler: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return sampler;
}
static gboolean
gst_vulkan_image_identity_start (GstBaseTransform * bt)
{
GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt);
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (vk_identity);
GstVulkanHandle *vert, *frag;
GError *error = NULL;
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
return FALSE;
if (!(vk_identity->sampler = _create_sampler (vk_identity)))
return FALSE;
vk_identity->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);
if (!(vert = _vk_create_shader (vfilter->device, identity_vert,
identity_vert_size, &error)))
goto error;
if (!(frag = _vk_create_shader (vfilter->device, identity_frag,
identity_frag_size, &error))) {
gst_vulkan_handle_unref (vert);
goto error;
}
gst_vulkan_full_screen_quad_set_shaders (vk_identity->quad, vert, frag);
gst_vulkan_handle_unref (vert);
gst_vulkan_handle_unref (frag);
return TRUE;
error:
GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL));
return FALSE;
}
static gboolean
gst_vulkan_image_identity_stop (GstBaseTransform * bt)
{
GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt);
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
if (render->device) {
GstVulkanFence *last_fence;
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (vk_identity->descriptor_pool)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_object_unref (last_fence,
(GstObject *) vk_identity->descriptor_pool));
vk_identity->descriptor_pool = NULL;
if (vk_identity->sampler)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_sampler (last_fence, vk_identity->sampler));
vk_identity->sampler = VK_NULL_HANDLE;
gst_vulkan_fence_unref (last_fence);
}
if (vk_identity->cmd_pool)
gst_object_unref (vk_identity->cmd_pool);
vk_identity->cmd_pool = VK_NULL_HANDLE;
gst_clear_object (&vk_identity->quad);
return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
}
static void
update_descriptor_set (GstVulkanImageIdentity * vk_identity,
VkDescriptorSet set, VkImageView view)
{
GstVulkanFullScreenRender *render =
GST_VULKAN_FULL_SCREEN_RENDER (vk_identity);
/* *INDENT-OFF* */
VkDescriptorImageInfo image_info = {
.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
.imageView = view,
.sampler = vk_identity->sampler
};
VkWriteDescriptorSet writes = {
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
.pNext = NULL,
.dstSet = set,
.dstBinding = 0,
.dstArrayElement = 0,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.descriptorCount = 1,
.pImageInfo = &image_info
};
/* *INDENT-ON* */
vkUpdateDescriptorSets (render->device->device, 1, &writes, 0, NULL);
}
static VkFramebuffer
_create_framebuffer (GstVulkanImageIdentity * vk_identity, VkImageView view)
{
GstVulkanFullScreenRender *render =
GST_VULKAN_FULL_SCREEN_RENDER (vk_identity);
/* *INDENT-OFF* */
VkImageView attachments[] = {
view,
};
VkFramebufferCreateInfo framebuffer_info = {
.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
.pNext = NULL,
.renderPass = render->render_pass,
.attachmentCount = 1,
.pAttachments = attachments,
.width = GST_VIDEO_INFO_WIDTH (&render->in_info),
.height = GST_VIDEO_INFO_HEIGHT (&render->in_info),
.layers = 1
};
/* *INDENT-ON* */
VkFramebuffer framebuffer;
GError *error = NULL;
VkResult err;
err =
vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL,
&framebuffer);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) {
GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return framebuffer;
}
static GstFlowReturn
gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf,
GstBuffer * outbuf)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt);
GstVulkanImageMemory *in_img_mem, *out_img_mem;
GstVulkanImageView *in_img_view, *out_img_view;
GstVulkanFence *fence = NULL;
GstVulkanDescriptorSet *set;
GstMemory *in_mem, *out_mem;
VkFramebuffer framebuffer;
GstVulkanCommandBuffer *cmd_buf;
GError *error = NULL;
VkResult err;
fence = gst_vulkan_fence_new (render->device, 0, &error);
if (!fence)
if (!gst_vulkan_full_screen_quad_set_input_buffer (vk_identity->quad, inbuf,
&error))
goto error;
if (!gst_vulkan_full_screen_quad_set_output_buffer (vk_identity->quad, outbuf,
&error))
goto error;
in_mem = gst_buffer_peek_memory (inbuf, 0);
if (!gst_is_vulkan_image_memory (in_mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Input memory must be a GstVulkanImageMemory");
if (!gst_vulkan_full_screen_quad_draw (vk_identity->quad, &error))
goto error;
}
in_img_mem = (GstVulkanImageMemory *) in_mem;
in_img_view = get_or_create_image_view (in_img_mem);
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
GST_MINI_OBJECT_CAST (in_img_view)));
out_mem = gst_buffer_peek_memory (outbuf, 0);
if (!gst_is_vulkan_image_memory (out_mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Input memory must be a GstVulkanImageMemory");
goto error;
}
out_img_mem = (GstVulkanImageMemory *) out_mem;
out_img_view = get_or_create_image_view (out_img_mem);
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
GST_MINI_OBJECT_CAST (out_img_view)));
if (!vk_identity->cmd_pool) {
if (!(vk_identity->cmd_pool =
gst_vulkan_queue_create_command_pool (render->queue, &error)))
goto error;
}
if (!(set =
gst_vulkan_descriptor_cache_acquire (vk_identity->descriptor_pool,
&error)))
goto error;
update_descriptor_set (vk_identity, set->set, in_img_view->view);
if (!(cmd_buf =
gst_vulkan_command_pool_create (vk_identity->cmd_pool, &error)))
goto error;
if (!(framebuffer = _create_framebuffer (vk_identity, out_img_view->view))) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Failed to create framebuffer");
goto error;
}
{
VkCommandBufferBeginInfo cmd_buf_info = { 0, };
/* *INDENT-OFF* */
cmd_buf_info = (VkCommandBufferBeginInfo) {
.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
.pNext = NULL,
.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT,
.pInheritanceInfo = NULL
};
/* *INDENT-ON* */
gst_vulkan_command_buffer_lock (cmd_buf);
err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0)
goto unlock_error;
}
{
/* *INDENT-OFF* */
VkImageMemoryBarrier in_image_memory_barrier = {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = in_img_mem->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT,
.oldLayout = in_img_mem->barrier.image_layout,
.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.image = in_img_mem->image,
.subresourceRange = in_img_mem->barrier.subresource_range
};
VkImageMemoryBarrier out_image_memory_barrier = {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = out_img_mem->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.oldLayout = out_img_mem->barrier.image_layout,
.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.image = out_img_mem->image,
.subresourceRange = out_img_mem->barrier.subresource_range
};
/* *INDENT-ON* */
vkCmdPipelineBarrier (cmd_buf->cmd,
in_img_mem->barrier.parent.pipeline_stages,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1,
&in_image_memory_barrier);
in_img_mem->barrier.parent.pipeline_stages =
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
in_img_mem->barrier.parent.access_flags =
in_image_memory_barrier.dstAccessMask;
in_img_mem->barrier.image_layout = in_image_memory_barrier.newLayout;
vkCmdPipelineBarrier (cmd_buf->cmd,
out_img_mem->barrier.parent.pipeline_stages,
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1,
&out_image_memory_barrier);
out_img_mem->barrier.parent.pipeline_stages =
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
out_img_mem->barrier.parent.access_flags =
out_image_memory_barrier.dstAccessMask;
out_img_mem->barrier.image_layout = out_image_memory_barrier.newLayout;
}
vkCmdBindDescriptorSets (cmd_buf->cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
render->pipeline_layout, 0, 1, &set->set, 0, NULL);
if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd,
framebuffer))
goto unlock_error;
err = vkEndCommandBuffer (cmd_buf->cmd);
gst_vulkan_command_buffer_unlock (cmd_buf);
if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0)
goto error;
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
GST_MINI_OBJECT_CAST (set)));
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_framebuffer (fence, framebuffer));
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
GST_MINI_OBJECT_CAST (cmd_buf)));
if (!gst_vulkan_full_screen_render_submit (render, cmd_buf->cmd, fence))
return GST_FLOW_ERROR;
return GST_FLOW_OK;
unlock_error:
if (cmd_buf) {
gst_vulkan_command_buffer_unlock (cmd_buf);
gst_vulkan_command_buffer_unref (cmd_buf);
}
error:
GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
g_clear_error (&error);

15
ext/vulkan/vkimageidentity.h
View file

@ -24,7 +24,7 @@
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
#include "vkfullscreenrender.h"
#include "vkfullscreenquad.h"
G_BEGIN_DECLS
@ -39,20 +39,15 @@ typedef struct _GstVulkanImageIdentityClass GstVulkanImageIdentityClass;
struct _GstVulkanImageIdentity
{
GstVulkanFullScreenRender parent;
GstVulkanVideoFilter parent;
GstVulkanCommandPool *cmd_pool;
VkSampler sampler;
GstVulkanDescriptorCache *descriptor_pool;
VkDescriptorSetLayoutBinding sampler_layout_binding;
VkDescriptorSetLayoutCreateInfo layout_info;
GstVulkanFullScreenQuad *quad;
GstMemory *uniforms;
};
struct _GstVulkanImageIdentityClass
{
GstVulkanFullScreenRenderClass parent_class;
GstVulkanVideoFilterClass parent_class;
};
GType gst_vulkan_image_identity_get_type(void);

15
ext/vulkan/vkshader.c
View file

@ -27,11 +27,11 @@
#define SPIRV_MAGIC_NUMBER_NE 0x07230203
#define SPIRV_MAGIC_NUMBER_OE 0x03022307
VkShaderModule
GstVulkanHandle *
_vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size,
GError ** error)
{
VkShaderModule ret;
VkShaderModule shader;
VkResult res;
/* *INDENT-OFF* */
@ -49,7 +49,7 @@ _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size,
g_return_val_if_fail (size >= 4, VK_NULL_HANDLE);
g_return_val_if_fail (size % 4 == 0, VK_NULL_HANDLE);
first_word = ((guint32 *) code)[0];
first_word = code[0] | code[1] << 8 | code[2] << 16 | code[3] << 24;
g_return_val_if_fail (first_word == SPIRV_MAGIC_NUMBER_NE
|| first_word == SPIRV_MAGIC_NUMBER_OE, VK_NULL_HANDLE);
if (first_word == SPIRV_MAGIC_NUMBER_OE) {
@ -78,12 +78,11 @@ _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size,
info.pCode = new_code;
}
res = vkCreateShaderModule (device->device, &info, NULL, &ret);
res = vkCreateShaderModule (device->device, &info, NULL, &shader);
g_free (new_code);
if (gst_vulkan_error_to_g_error (res, error, "VkCreateShaderModule") < 0)
return VK_NULL_HANDLE;
return NULL;
g_free (new_code);
return ret;
return gst_vulkan_handle_new_wrapped (device, GST_VULKAN_HANDLE_TYPE_SHADER,
(GstVulkanHandleTypedef) shader, gst_vulkan_handle_free_shader, NULL);
}

2
ext/vulkan/vkshader.h
View file

@ -26,6 +26,6 @@
G_BEGIN_DECLS
VkShaderModule _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error);
GstVulkanHandle * _vk_create_shader (GstVulkanDevice * device, gchar * code, gsize size, GError ** error);
#endif

738
ext/vulkan/vkviewconvert.c
View file

@ -293,30 +293,48 @@ calculate_reorder_indexes (GstVideoFormat in_format,
static void
update_descriptor_set (GstVulkanViewConvert * conv,
VkDescriptorSet descriptor_set, VkImageView * views, guint n_views)
GstVulkanImageView ** in_views)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
VkDescriptorBufferInfo buffer_info;
VkDescriptorImageInfo image_info[2 * GST_VIDEO_MAX_PLANES];
VkWriteDescriptorSet writes[10];
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
VkDescriptorImageInfo image_info[GST_VIDEO_MAX_PLANES];
VkWriteDescriptorSet writes[GST_VIDEO_MAX_PLANES];
GstVideoMultiviewMode in_mode;
GstVideoMultiviewFlags in_flags, out_flags;
VkImageView views[GST_VIDEO_MAX_PLANES];
guint i = 0;
for (; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) {
in_mode = conv->input_mode_override;
in_flags = conv->input_flags_override;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE)
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info); i++) {
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
views[2 * i] = in_views[i]->view;
views[2 * i + 1] = in_views[i]->view;
} else {
views[2 * i] = in_views[i]->view;
views[2 * i + 1] = in_views[i]->view;
}
}
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
/* *INDENT-OFF* */
image_info[i] = (VkDescriptorImageInfo) {
.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
.imageView = views[i],
.sampler = conv->sampler
.sampler = (VkSampler) conv->quad->sampler->handle
};
g_assert (i < n_views);
g_assert (i < GST_VIDEO_MAX_PLANES);
writes[i] = (VkWriteDescriptorSet) {
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
.pNext = NULL,
.dstSet = descriptor_set,
.dstBinding = i,
.dstSet = conv->quad->descriptor_set->set,
.dstBinding = i+1,
.dstArrayElement = 0,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.descriptorCount = 1,
@ -324,34 +342,31 @@ update_descriptor_set (GstVulkanViewConvert * conv,
};
/* *INDENT-ON* */
}
/* *INDENT-OFF* */
buffer_info = (VkDescriptorBufferInfo) {
.buffer = ((GstVulkanBufferMemory *) conv->uniform)->buffer,
.offset = 0,
.range = sizeof (struct ViewUpdate),
};
writes[i] = (VkWriteDescriptorSet) {
.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET,
.pNext = NULL,
.dstSet = descriptor_set,
.dstBinding = i,
.dstArrayElement = 0,
.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
.descriptorCount = 1,
.pBufferInfo = &buffer_info
};
/* *INDENT-ON* */
i++;
g_assert (i <= G_N_ELEMENTS (writes));
vkUpdateDescriptorSets (render->device->device, i, writes, 0, NULL);
vkUpdateDescriptorSets (vfilter->device->device, i, writes, 0, NULL);
}
static gboolean
_update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
GstVulkanImageView ** out_views, VkImageView views[GST_VIDEO_MAX_PLANES])
create_uniform_buffer (GstVulkanViewConvert * conv)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
conv->uniform =
gst_vulkan_buffer_memory_alloc (vfilter->device,
sizeof (struct ViewUpdate),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
return TRUE;
}
static gboolean
update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
GstVulkanImageView ** out_views)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
GstVideoMultiviewMode in_mode, out_mode;
GstVideoMultiviewFlags in_flags, out_flags;
struct ViewUpdate data;
@ -359,8 +374,8 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
guint l_index, r_index;
gboolean mono_input = FALSE;
calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info),
in_views, GST_VIDEO_INFO_FORMAT (&render->out_info),
calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&vfilter->in_info),
in_views, GST_VIDEO_INFO_FORMAT (&vfilter->out_info),
out_views, data.in_reorder_idx, data.out_reorder_idx);
data.tex_scale[0][0] = data.tex_scale[0][1] = 1.;
@ -371,29 +386,24 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
in_mode = conv->input_mode_override;
in_flags = conv->input_flags_override;
if (in_mode == GST_VIDEO_MULTIVIEW_MODE_NONE) {
in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->in_info);
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->in_info);
in_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->in_info);
in_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->in_info);
}
/* Configured output mode already takes any override
* into account */
out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&render->out_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&render->out_info);
out_mode = GST_VIDEO_INFO_MULTIVIEW_MODE (&vfilter->out_info);
out_flags = GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vfilter->out_info);
if ((in_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST) ==
(out_flags & GST_VIDEO_MULTIVIEW_FLAGS_RIGHT_VIEW_FIRST)) {
l_index = 0;
r_index = 1;
} else {
VkImageView scratch_view;
GST_LOG_OBJECT (conv, "Switching left/right views");
l_index = 1;
r_index = 0;
scratch_view = views[0];
views[0] = views[1];
views[1] = scratch_view;
}
if (in_mode < GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE) { /* unknown/mono/left/right single image */
@ -470,8 +480,8 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
data.output_type = GST_VIDEO_MULTIVIEW_MODE_SIDE_BY_SIDE;
}
data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&render->out_info);
data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&render->out_info);
data.tex_size[0] = GST_VIDEO_INFO_WIDTH (&vfilter->out_info);
data.tex_size[1] = GST_VIDEO_INFO_HEIGHT (&vfilter->out_info);
memcpy (&data.downmix[0], &downmix_matrices[conv->downmix_mode][0],
sizeof (data.downmix[0]));
memcpy (&data.downmix[1], &downmix_matrices[conv->downmix_mode][1],
@ -482,59 +492,25 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views,
}
memcpy (map_info.data, &data, sizeof (data));
gst_memory_unmap (conv->uniform, &map_info);
conv->descriptor_up_to_date = TRUE;
return TRUE;
}
static GstVulkanDescriptorSet *
_create_descriptor_set (GstVulkanViewConvert * conv)
static GstMemory *
get_uniforms (GstVulkanViewConvert * conv,
GstVulkanImageView ** in_views, GstVulkanImageView ** out_views)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
GstVulkanDescriptorSet *ret;
GError *error = NULL;
ret = gst_vulkan_descriptor_cache_acquire (conv->descriptor_pool, &error);
if (!ret) {
GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s",
error->message);
g_clear_error (&error);
return NULL;
}
return ret;
}
static gboolean
view_convert_update_command_state (GstVulkanViewConvert * conv,
VkCommandBuffer cmd, GstVulkanImageView ** in_views,
GstVulkanImageView ** out_views, GstVulkanFence * fence)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
GstVulkanDescriptorSet *set;
VkImageView views[GST_VIDEO_MAX_PLANES];
int i;
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) {
views[2 * i] = in_views[i]->view;
views[2 * i + 1] = in_views[i]->view;
}
if (!conv->descriptor_up_to_date) {
if (!_update_uniform (conv, in_views, out_views, views))
if (!conv->uniform) {
if (!create_uniform_buffer (conv))
return NULL;
if (!update_uniform (conv, in_views, out_views)) {
gst_memory_unref (conv->uniform);
conv->uniform = NULL;
return FALSE;
}
}
set = _create_descriptor_set (conv);
update_descriptor_set (conv, set->set, views,
GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2);
vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS,
render->pipeline_layout, 0, 1, &set->set, 0, NULL);
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence, (GstMiniObject *) set));
return TRUE;
return gst_memory_ref (conv->uniform);
}
static void gst_vulkan_view_convert_set_property (GObject * object,
@ -554,21 +530,6 @@ static GstFlowReturn gst_vulkan_view_convert_transform (GstBaseTransform * bt,
static gboolean gst_vulkan_view_convert_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps);
static VkAttachmentReference
* gst_vulkan_view_convert_render_pass_attachment_references
(GstVulkanFullScreenRender * render, guint * n_attachments);
static VkAttachmentDescription
* gst_vulkan_view_convert_render_pass_attachment_descriptions
(GstVulkanFullScreenRender * render, guint * n_descriptions);
static VkDescriptorSetLayoutBinding
* gst_vulkan_view_convert_descriptor_set_layout_bindings
(GstVulkanFullScreenRender * render, guint * n_bindings);
static void
gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render);
static VkPushConstantRange
* gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender *
render, guint * n_constants);
static GstStaticPadTemplate gst_vulkan_sink_template =
GST_STATIC_PAD_TEMPLATE ("sink",
GST_PAD_SINK,
@ -599,7 +560,7 @@ enum
#define gst_vulkan_view_convert_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanViewConvert, gst_vulkan_view_convert,
GST_TYPE_VULKAN_FULL_SCREEN_RENDER,
GST_TYPE_VULKAN_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_view_convert,
"vulkanviewconvert", 0, "Vulkan View Convert"));
@ -609,12 +570,10 @@ gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass)
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
GstVulkanFullScreenRenderClass *fullscreenrender_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass;
gobject_class->set_property = gst_vulkan_view_convert_set_property;
gobject_class->get_property = gst_vulkan_view_convert_get_property;
@ -668,17 +627,6 @@ gst_vulkan_view_convert_class_init (GstVulkanViewConvertClass * klass)
gstbasetransform_class->fixate_caps = gst_vulkan_view_convert_fixate_caps;
gstbasetransform_class->set_caps = gst_vulkan_view_convert_set_caps;
gstbasetransform_class->transform = gst_vulkan_view_convert_transform;
fullscreenrender_class->render_pass_attachment_references =
gst_vulkan_view_convert_render_pass_attachment_references;
fullscreenrender_class->render_pass_attachment_descriptions =
gst_vulkan_view_convert_render_pass_attachment_descriptions;
fullscreenrender_class->descriptor_set_layout_bindings =
gst_vulkan_view_convert_descriptor_set_layout_bindings;
fullscreenrender_class->shader_create_info =
gst_vulkan_view_convert_shader_create_info;
fullscreenrender_class->push_constant_ranges =
gst_vulkan_view_convert_push_constant_ranges;
}
static void
@ -722,9 +670,6 @@ gst_vulkan_view_convert_set_property (GObject * object, guint prop_id,
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
GST_OBJECT_LOCK (conv);
conv->descriptor_up_to_date = FALSE;
GST_OBJECT_UNLOCK (conv);
}
static void
@ -1829,272 +1774,92 @@ done:
return othercaps;
}
static void
destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data)
static gboolean
create_descriptor_set_layout (GstVulkanViewConvert * conv, GError ** error)
{
VkPipelineShaderStageCreateInfo *info = data;
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (conv);
VkDescriptorSetLayoutBinding bindings[GST_VIDEO_MAX_PLANES * 2 + 1] =
{ {0,}, };
VkDescriptorSetLayoutCreateInfo layout_info;
VkDescriptorSetLayout descriptor_set_layout;
int descriptor_n = 0;
VkResult err;
int i;
for (i = 0; i < render->n_shader_stages; i++) {
vkDestroyShaderModule (render->device->device, info[i].module, NULL);
}
g_free (info);
}
static void
gst_vulkan_view_convert_shader_create_info (GstVulkanFullScreenRender * render)
{
VkShaderModule vert_module, frag_module;
vert_module =
_vk_create_shader (render->device, identity_vert, identity_vert_size,
NULL);
frag_module =
_vk_create_shader (render->device, view_convert_frag,
view_convert_frag_size, NULL);
render->n_shader_stages = 2;
render->shader_create_info =
g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages);
render->destroy_shader_create_info = destroy_shader_create_info;
/* *INDENT-OFF* */
render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.pNext = NULL,
.stage = VK_SHADER_STAGE_VERTEX_BIT,
.module = vert_module,
.pName = "main"
};
render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) {
.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
.pNext = NULL,
.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
.module = frag_module,
.pName = "main"
};
/* *INDENT-ON* */
}
static VkPushConstantRange *
gst_vulkan_view_convert_push_constant_ranges (GstVulkanFullScreenRender *
render, guint * n_constants)
{
*n_constants = 0;
return NULL;
}
static VkDescriptorSetLayoutBinding
* gst_vulkan_view_convert_descriptor_set_layout_bindings
(GstVulkanFullScreenRender * render, guint * n_bindings)
{
VkDescriptorSetLayoutBinding *bindings;
guint i;
*n_bindings = 0;
*n_bindings += GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2;
*n_bindings += 1; /* uniform binding */
bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings);
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2; i++) {
/* *INDENT-OFF* */
bindings[i] = (VkDescriptorSetLayoutBinding) {
.binding = i,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
};
i++;
bindings[i] = (VkDescriptorSetLayoutBinding) {
.binding = i,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
};
/* *INDENT-ON* */
}
/* *INDENT-OFF* */
bindings[i] = (VkDescriptorSetLayoutBinding) {
.binding = i,
bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
.binding = 0,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT
};
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vfilter->in_info) * 2; i++) {
bindings[descriptor_n++] = (VkDescriptorSetLayoutBinding) {
.binding = i+1,
.descriptorCount = 1,
.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.pImmutableSamplers = NULL,
.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT
};
};
layout_info = (VkDescriptorSetLayoutCreateInfo) {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO,
.pNext = NULL,
.bindingCount = descriptor_n,
.pBindings = bindings
};
/* *INDENT-ON* */
i++;
g_assert (i == *n_bindings);
return bindings;
}
static VkAttachmentReference
* gst_vulkan_view_convert_render_pass_attachment_references
(GstVulkanFullScreenRender * render, guint * n_attachments)
{
VkAttachmentReference *attachments;
int i;
*n_attachments = GST_VIDEO_INFO_N_PLANES (&render->out_info);
attachments = g_new0 (VkAttachmentReference, *n_attachments);
for (i = 0; i < *n_attachments; i++) {
/* *INDENT-OFF* */
attachments[i] = (VkAttachmentReference) {
.attachment = i,
.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
};
/* *INDENT-ON* */
err =
vkCreateDescriptorSetLayout (vfilter->device->device, &layout_info,
NULL, &descriptor_set_layout);
if (gst_vulkan_error_to_g_error (err, error,
"vkCreateDescriptorSetLayout") < 0) {
return FALSE;
}
return attachments;
}
conv->quad->descriptor_set_layout =
gst_vulkan_handle_new_wrapped (vfilter->device,
GST_VULKAN_HANDLE_TYPE_DESCRIPTOR_SET_LAYOUT,
(GstVulkanHandleTypedef) descriptor_set_layout,
gst_vulkan_handle_free_descriptor_set_layout, NULL);
static VkAttachmentDescription
* gst_vulkan_view_convert_render_pass_attachment_descriptions
(GstVulkanFullScreenRender * render, guint * n_descriptions)
{
VkAttachmentDescription *color_attachments;
int i;
*n_descriptions = GST_VIDEO_INFO_N_PLANES (&render->out_info);
color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions);
for (i = 0; i < *n_descriptions; i++) {
/* *INDENT-OFF* */
color_attachments[i] = (VkAttachmentDescription) {
.format = gst_vulkan_format_from_video_info (&render->out_info, i),
.samples = VK_SAMPLE_COUNT_1_BIT,
.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR,
.storeOp = VK_ATTACHMENT_STORE_OP_STORE,
.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE,
.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE,
/* FIXME: share this between elements to avoid pipeline barriers */
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL
};
/* *INDENT-ON* */
}
return color_attachments;
}
static VkSampler
_create_sampler (GstVulkanViewConvert * conv)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
/* *INDENT-OFF* */
VkSamplerCreateInfo samplerInfo = {
.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
.magFilter = VK_FILTER_NEAREST,
.minFilter = VK_FILTER_NEAREST,
.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
.anisotropyEnable = VK_FALSE,
.maxAnisotropy = 1,
.borderColor = VK_BORDER_COLOR_INT_OPAQUE_BLACK,
.unnormalizedCoordinates = VK_FALSE,
.compareEnable = VK_FALSE,
.compareOp = VK_COMPARE_OP_ALWAYS,
.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR,
.mipLodBias = 0.0f,
.minLod = 0.0f,
.maxLod = 0.0f
};
/* *INDENT-ON* */
GError *error = NULL;
VkSampler sampler;
VkResult err;
err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) {
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return sampler;
return TRUE;
}
static gboolean
gst_vulkan_view_convert_start (GstBaseTransform * bt)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanHandle *vert, *frag;
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt))
return FALSE;
if (!(conv->sampler = _create_sampler (conv)))
conv->quad = gst_vulkan_full_screen_quad_new (vfilter->queue);
if (!(vert =
_vk_create_shader (vfilter->device, identity_vert, identity_vert_size,
NULL))) {
return FALSE;
}
if (!(frag =
_vk_create_shader (vfilter->device, view_convert_frag,
view_convert_frag_size, NULL))) {
gst_vulkan_handle_unref (vert);
return FALSE;
return TRUE;
}
static GstVulkanDescriptorCache *
_create_descriptor_pool (GstVulkanViewConvert * conv)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
guint max_sets = 32; /* FIXME: don't hardcode this! */
/* *INDENT-OFF* */
VkDescriptorPoolSize pool_sizes[] = {
{
.type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
.descriptorCount = max_sets * GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2,
},
{
.type = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
.descriptorCount = max_sets
},
};
VkDescriptorPoolCreateInfo pool_info = {
.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO,
.pNext = NULL,
.poolSizeCount = G_N_ELEMENTS (pool_sizes),
.pPoolSizes = pool_sizes,
.maxSets = max_sets
};
/* *INDENT-ON* */
VkDescriptorPool pool;
GstVulkanDescriptorPool *ret;
GstVulkanDescriptorCache *cache;
GError *error = NULL;
VkResult err;
err =
vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) {
GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
ret = gst_vulkan_descriptor_pool_new_wrapped (render->device, pool, 32);
cache =
gst_vulkan_descriptor_cache_new (ret, 1, &render->descriptor_set_layout);
gst_object_unref (ret);
return cache;
}
static gboolean
_create_uniform_buffer (GstVulkanViewConvert * conv)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
conv->uniform =
gst_vulkan_buffer_memory_alloc (render->device,
sizeof (struct ViewUpdate),
VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT,
VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
if (!gst_vulkan_full_screen_quad_set_shaders (conv->quad, vert, frag)) {
gst_vulkan_handle_unref (vert);
gst_vulkan_handle_unref (frag);
return FALSE;
}
gst_vulkan_handle_unref (vert);
gst_vulkan_handle_unref (frag);
return TRUE;
}
@ -2103,36 +1868,15 @@ static gboolean
gst_vulkan_view_convert_set_caps (GstBaseTransform * bt, GstCaps * in_caps,
GstCaps * out_caps)
{
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstVulkanFence *last_fence;
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (conv->descriptor_pool)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_object_unref (last_fence,
(GstObject *) conv->descriptor_pool));
conv->descriptor_pool = NULL;
if (conv->uniform)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (last_fence,
(GstMiniObject *) conv->uniform));
conv->uniform = NULL;
gst_vulkan_fence_unref (last_fence);
if (!GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps,
out_caps))
return FALSE;
if (!(conv->descriptor_pool = _create_descriptor_pool (conv)))
return FALSE;
if (!_create_uniform_buffer (conv))
if (!gst_vulkan_full_screen_quad_set_info (conv->quad, &vfilter->in_info,
&vfilter->out_info))
return FALSE;
return TRUE;
@ -2142,134 +1886,82 @@ static gboolean
gst_vulkan_view_convert_stop (GstBaseTransform * bt)
{
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
conv->descriptor_up_to_date = FALSE;
if (render->device) {
GstVulkanFence *last_fence;
if (render->last_fence)
last_fence = gst_vulkan_fence_ref (render->last_fence);
else
last_fence = gst_vulkan_fence_new_always_signalled (render->device);
if (conv->descriptor_pool)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_object_unref (last_fence,
(GstObject *) conv->descriptor_pool));
conv->descriptor_pool = NULL;
if (conv->sampler)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_sampler (last_fence, conv->sampler));
conv->sampler = VK_NULL_HANDLE;
if (conv->uniform)
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (last_fence,
(GstMiniObject *) conv->uniform));
conv->uniform = NULL;
gst_vulkan_fence_unref (last_fence);
}
if (conv->cmd_pool)
gst_object_unref (conv->cmd_pool);
conv->cmd_pool = NULL;
gst_clear_object (&conv->quad);
if (conv->uniform)
gst_memory_unref (conv->uniform);
conv->uniform = NULL;
return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt);
}
static VkFramebuffer
_create_framebuffer (GstVulkanViewConvert * conv, guint n_views,
VkImageView * views)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv);
/* *INDENT-OFF* */
VkFramebufferCreateInfo framebuffer_info = {
.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO,
.pNext = NULL,
.renderPass = render->render_pass,
.attachmentCount = n_views,
.pAttachments = views,
.width = GST_VIDEO_INFO_WIDTH (&render->out_info),
.height = GST_VIDEO_INFO_HEIGHT (&render->out_info),
.layers = 1
};
/* *INDENT-ON* */
VkFramebuffer framebuffer;
GError *error = NULL;
VkResult err;
err =
vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL,
&framebuffer);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) {
GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s",
error->message);
g_clear_error (&error);
return VK_NULL_HANDLE;
}
return framebuffer;
}
static GstFlowReturn
gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf,
GstBuffer * outbuf)
{
GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt);
GstVulkanVideoFilter *vfilter = GST_VULKAN_VIDEO_FILTER (bt);
GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt);
GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanImageMemory *out_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, };
GstVulkanCommandBuffer *cmd_buf = NULL;
GstVulkanFence *fence = NULL;
GstVulkanCommandBuffer *cmd_buf;
VkFramebuffer framebuffer;
GError *error = NULL;
VkResult err;
int i;
fence = gst_vulkan_fence_new (render->device, 0, &error);
if (!gst_vulkan_full_screen_quad_set_input_buffer (conv->quad, inbuf, &error))
goto error;
if (!gst_vulkan_full_screen_quad_set_output_buffer (conv->quad, outbuf,
&error))
goto error;
fence = gst_vulkan_fence_new (vfilter->device, 0, &error);
if (!fence)
goto error;
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) {
GstMemory *mem = gst_buffer_peek_memory (inbuf, i);
if (!gst_is_vulkan_image_memory (mem)) {
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->in_info); i++) {
GstMemory *img_mem = gst_buffer_peek_memory (inbuf, i);
if (!gst_is_vulkan_image_memory (img_mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Input memory must be a GstVulkanImageMemory");
goto error;
}
in_img_mems[i] = (GstVulkanImageMemory *) mem;
in_img_views[i] = get_or_create_image_view (in_img_mems[i]);
gst_vulkan_trash_list_add (render->trash_list,
in_img_views[i] =
get_or_create_image_view ((GstVulkanImageMemory *) img_mem);
gst_vulkan_trash_list_add (conv->quad->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
(GstMiniObject *) in_img_views[i]));
}
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) {
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&conv->quad->out_info); i++) {
GstMemory *mem = gst_buffer_peek_memory (outbuf, i);
if (!gst_is_vulkan_image_memory (mem)) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Output memory must be a GstVulkanImageMemory");
goto error;
}
out_img_mems[i] = (GstVulkanImageMemory *) mem;
out_img_views[i] = get_or_create_image_view (out_img_mems[i]);
gst_vulkan_trash_list_add (render->trash_list,
out_img_views[i] = get_or_create_image_view ((GstVulkanImageMemory *) mem);
gst_vulkan_trash_list_add (conv->quad->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
(GstMiniObject *) out_img_views[i]));
}
if (!conv->cmd_pool) {
if (!(conv->cmd_pool =
gst_vulkan_queue_create_command_pool (render->queue, &error)))
{
GstMemory *uniforms = get_uniforms (conv, in_img_views, out_img_views);
if (!gst_vulkan_full_screen_quad_set_uniform_buffer (conv->quad, uniforms,
&error))
goto error;
gst_memory_unref (uniforms);
}
if (!(cmd_buf = gst_vulkan_command_pool_create (conv->cmd_pool, &error)))
if (!conv->quad->descriptor_set_layout)
if (!create_descriptor_set_layout (conv, &error))
goto error;
if (!gst_vulkan_full_screen_quad_prepare_draw (conv->quad, fence, &error))
goto error;
if (!(cmd_buf =
gst_vulkan_command_pool_create (conv->quad->cmd_pool, &error)))
goto error;
{
@ -2287,106 +1979,23 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf,
gst_vulkan_command_buffer_lock (cmd_buf);
err = vkBeginCommandBuffer (cmd_buf->cmd, &cmd_buf_info);
if (gst_vulkan_error_to_g_error (err, &error, "vkBeginCommandBuffer") < 0)
goto unlock_error;
goto error;
}
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) {
/* *INDENT-OFF* */
VkImageMemoryBarrier in_image_memory_barrier = {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = in_img_mems[i]->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_INPUT_ATTACHMENT_READ_BIT,
.oldLayout = in_img_mems[i]->barrier.image_layout,
.newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.image = in_img_mems[i]->image,
.subresourceRange = in_img_mems[i]->barrier.subresource_range
};
/* *INDENT-ON* */
vkCmdPipelineBarrier (cmd_buf->cmd,
in_img_mems[i]->barrier.parent.pipeline_stages,
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1,
&in_image_memory_barrier);
in_img_mems[i]->barrier.parent.pipeline_stages =
VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
in_img_mems[i]->barrier.parent.access_flags =
in_image_memory_barrier.dstAccessMask;
in_img_mems[i]->barrier.image_layout = in_image_memory_barrier.newLayout;
}
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) {
VkImageMemoryBarrier out_image_memory_barrier;
/* *INDENT-OFF* */
out_image_memory_barrier = (VkImageMemoryBarrier) {
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
.pNext = NULL,
.srcAccessMask = out_img_mems[i]->barrier.parent.access_flags,
.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
.oldLayout = out_img_mems[i]->barrier.image_layout,
.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
/* FIXME: implement exclusive transfers */
.srcQueueFamilyIndex = 0,
.dstQueueFamilyIndex = 0,
.image = out_img_mems[i]->image,
.subresourceRange = out_img_mems[i]->barrier.subresource_range
};
/* *INDENT-ON* */
vkCmdPipelineBarrier (cmd_buf->cmd,
out_img_mems[i]->barrier.parent.pipeline_stages,
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, NULL, 0, NULL, 1,
&out_image_memory_barrier);
out_img_mems[i]->barrier.parent.pipeline_stages =
VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
out_img_mems[i]->barrier.parent.access_flags =
out_image_memory_barrier.dstAccessMask;
out_img_mems[i]->barrier.image_layout = out_image_memory_barrier.newLayout;
}
{
VkImageView attachments[4] = { 0, };
for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) {
attachments[i] = out_img_views[i]->view;
}
if (!(framebuffer = _create_framebuffer (conv,
GST_VIDEO_INFO_N_PLANES (&render->out_info), attachments))) {
g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Failed to create framebuffer");
goto unlock_error;
}
}
view_convert_update_command_state (conv, cmd_buf->cmd, in_img_views,
out_img_views, fence);
if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd,
framebuffer)) {
g_set_error (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED,
"Failed to fill framebuffer");
update_descriptor_set (conv, in_img_views);
if (!gst_vulkan_full_screen_quad_fill_command_buffer (conv->quad, cmd_buf,
fence, &error))
goto unlock_error;
}
err = vkEndCommandBuffer (cmd_buf->cmd);
gst_vulkan_command_buffer_unlock (cmd_buf);
if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0)
goto error;
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_free_framebuffer (fence, framebuffer));
gst_vulkan_trash_list_add (render->trash_list,
gst_vulkan_trash_new_mini_object_unref (fence,
GST_MINI_OBJECT_CAST (cmd_buf)));
if (!gst_vulkan_full_screen_quad_submit (conv->quad, cmd_buf, fence, &error))
goto error;
if (!gst_vulkan_full_screen_render_submit (render, cmd_buf->cmd, fence))
return GST_FLOW_ERROR;
gst_vulkan_fence_unref (fence);
return GST_FLOW_OK;
@ -2395,7 +2004,10 @@ unlock_error:
gst_vulkan_command_buffer_unlock (cmd_buf);
gst_vulkan_command_buffer_unref (cmd_buf);
}
error:
gst_clear_mini_object ((GstMiniObject **) & fence);
GST_ELEMENT_ERROR (bt, LIBRARY, FAILED, ("%s", error->message), (NULL));
g_clear_error (&error);
return GST_FLOW_ERROR;

32
ext/vulkan/vkviewconvert.h
View file

@ -24,7 +24,7 @@
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
#include "vkfullscreenrender.h"
#include "vkfullscreenquad.h"
G_BEGIN_DECLS
@ -46,34 +46,24 @@ typedef enum
struct _GstVulkanViewConvert
{
GstVulkanFullScreenRender parent;
GstVulkanVideoFilter parent;
GstVulkanCommandPool *cmd_pool;
VkSampler sampler;
GstVulkanDescriptorCache *descriptor_pool;
VkShaderModule vert_module;
VkShaderModule frag_module;
VkDescriptorSetLayoutBinding sampler_layout_binding;
VkDescriptorSetLayoutCreateInfo layout_info;
GstMemory *uniform;
gboolean descriptor_up_to_date;
GstVulkanFullScreenQuad *quad;
/* properties */
GstVideoMultiviewMode input_mode_override;
GstVideoMultiviewFlags input_flags_override;
GstVideoMultiviewMode output_mode_override;
GstVideoMultiviewFlags output_flags_override;
GstVideoMultiviewMode input_mode_override;
GstVideoMultiviewFlags input_flags_override;
GstVideoMultiviewMode output_mode_override;
GstVideoMultiviewFlags output_flags_override;
GstVulkanStereoDownmix downmix_mode;
GstVulkanStereoDownmix downmix_mode;
GstMemory *uniform;
};
struct _GstVulkanViewConvertClass
{
GstVulkanFullScreenRenderClass parent_class;
GstVulkanVideoFilterClass parent_class;
};
GType gst_vulkan_view_convert_get_type(void);

338
gst-libs/gst/vulkan/gstvkvideofilter.c Normal file
View file

@ -0,0 +1,338 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:vulkanvideofilter
* @title: vulkanvideofilter
*
* vulkanvideofilter is a helper base class for retrieving and holding the
* #GstVulkanInstance, #GstVulkanDevice and #GstVulkanQueue used by an element.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include <string.h>
#include <gst/vulkan/gstvkvideofilter.h>
GST_DEBUG_CATEGORY (gst_debug_vulkan_video_filter);
#define GST_CAT_DEFAULT gst_debug_vulkan_video_filter
static void gst_vulkan_video_filter_finalize (GObject * object);
static gboolean gst_vulkan_video_filter_query (GstBaseTransform * bt,
GstPadDirection direction, GstQuery * query);
static void gst_vulkan_video_filter_set_context (GstElement * element,
GstContext * context);
static gboolean gst_vulkan_video_filter_start (GstBaseTransform * bt);
static gboolean gst_vulkan_video_filter_stop (GstBaseTransform * bt);
static gboolean gst_vulkan_video_filter_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps);
static GstCaps *gst_vulkan_video_filter_transform_caps (GstBaseTransform *
bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
static gboolean
gst_vulkan_video_filter_propose_allocation (GstBaseTransform * bt,
GstQuery * decide_query, GstQuery * query);
static gboolean
gst_vulkan_video_filter_decide_allocation (GstBaseTransform * bt,
GstQuery * query);
enum
{
PROP_0,
};
enum
{
SIGNAL_0,
LAST_SIGNAL
};
/* static guint gst_vulkan_video_filter_signals[LAST_SIGNAL] = { 0 }; */
#define gst_vulkan_video_filter_parent_class parent_class
G_DEFINE_TYPE_WITH_CODE (GstVulkanVideoFilter,
gst_vulkan_video_filter, GST_TYPE_BASE_TRANSFORM,
GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_video_filter,
"vulkanvideofilter", 0, "Vulkan Video Filter"));
static void
gst_vulkan_video_filter_class_init (GstVulkanVideoFilterClass * klass)
{
GObjectClass *gobject_class;
GstElementClass *gstelement_class;
GstBaseTransformClass *gstbasetransform_class;
gobject_class = (GObjectClass *) klass;
gstelement_class = (GstElementClass *) klass;
gstbasetransform_class = (GstBaseTransformClass *) klass;
gobject_class->finalize = gst_vulkan_video_filter_finalize;
gstelement_class->set_context = gst_vulkan_video_filter_set_context;
gstbasetransform_class->start =
GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_start);
gstbasetransform_class->stop =
GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_stop);
gstbasetransform_class->query =
GST_DEBUG_FUNCPTR (gst_vulkan_video_filter_query);
gstbasetransform_class->set_caps = gst_vulkan_video_filter_set_caps;
gstbasetransform_class->transform_caps =
gst_vulkan_video_filter_transform_caps;
gstbasetransform_class->propose_allocation =
gst_vulkan_video_filter_propose_allocation;
gstbasetransform_class->decide_allocation =
gst_vulkan_video_filter_decide_allocation;
}
static void
gst_vulkan_video_filter_init (GstVulkanVideoFilter * render)
{
}
static void
gst_vulkan_video_filter_finalize (GObject * object)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (object);
gst_caps_replace (&render->in_caps, NULL);
gst_caps_replace (&render->out_caps, NULL);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static gboolean
gst_vulkan_video_filter_query (GstBaseTransform * bt,
GstPadDirection direction, GstQuery * query)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_CONTEXT:{
if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query,
NULL, render->instance, render->device))
return TRUE;
if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render),
query, render->queue))
return TRUE;
break;
}
default:
break;
}
return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query);
}
static void
gst_vulkan_video_filter_set_context (GstElement * element, GstContext * context)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (element);
gst_vulkan_handle_set_context (element, context, NULL, &render->instance);
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
}
struct choose_data
{
GstVulkanVideoFilter *upload;
GstVulkanQueue *queue;
};
static gboolean
_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue,
struct choose_data *data)
{
guint flags =
device->physical_device->queue_family_props[queue->family].queueFlags;
if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) {
if (data->queue)
gst_object_unref (data->queue);
data->queue = gst_object_ref (queue);
return FALSE;
}
return TRUE;
}
static GstVulkanQueue *
_find_graphics_queue (GstVulkanVideoFilter * upload)
{
struct choose_data data;
data.upload = upload;
data.queue = NULL;
gst_vulkan_device_foreach_queue (upload->device,
(GstVulkanDeviceForEachQueueFunc) _choose_queue, &data);
return data.queue;
}
static GstCaps *
gst_vulkan_video_filter_transform_caps (GstBaseTransform * bt,
GstPadDirection direction, GstCaps * caps, GstCaps * filter)
{
GstCaps *result, *tmp;
tmp = gst_caps_copy (caps);
if (filter) {
result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (tmp);
} else {
result = tmp;
}
return result;
}
static gboolean
gst_vulkan_video_filter_set_caps (GstBaseTransform * bt,
GstCaps * in_caps, GstCaps * out_caps)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt);
if (!gst_video_info_from_caps (&render->in_info, in_caps))
return FALSE;
if (!gst_video_info_from_caps (&render->out_info, out_caps))
return FALSE;
gst_caps_replace (&render->in_caps, in_caps);
gst_caps_replace (&render->out_caps, out_caps);
GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps);
return TRUE;
}
static gboolean
gst_vulkan_video_filter_propose_allocation (GstBaseTransform * bt,
GstQuery * decide_query, GstQuery * query)
{
/* FIXME: */
return FALSE;
}
static gboolean
gst_vulkan_video_filter_decide_allocation (GstBaseTransform * bt,
GstQuery * query)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt);
GstBufferPool *pool = NULL;
GstStructure *config;
GstCaps *caps;
guint min, max, size;
gboolean update_pool;
gst_query_parse_allocation (query, &caps, NULL);
if (!caps)
return FALSE;
if (gst_query_get_n_allocation_pools (query) > 0) {
gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max);
update_pool = TRUE;
} else {
GstVideoInfo vinfo;
gst_video_info_init (&vinfo);
gst_video_info_from_caps (&vinfo, caps);
size = vinfo.size;
min = max = 0;
update_pool = FALSE;
}
if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) {
if (pool)
gst_object_unref (pool);
pool = gst_vulkan_image_buffer_pool_new (render->device);
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
gst_buffer_pool_set_config (pool, config);
if (update_pool)
gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max);
else
gst_query_add_allocation_pool (query, pool, size, min, max);
gst_object_unref (pool);
return TRUE;
}
static gboolean
gst_vulkan_video_filter_start (GstBaseTransform * bt)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt);
if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL,
&render->instance)) {
GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
("Failed to retrieve vulkan instance"), (NULL));
return FALSE;
}
if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render),
&render->device)) {
GError *error = NULL;
GST_DEBUG_OBJECT (render, "No device retrieved from peer elements");
if (!(render->device =
gst_vulkan_instance_create_device (render->instance, &error))) {
GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND,
("Failed to create vulkan device"), ("%s", error->message));
g_clear_error (&error);
return FALSE;
}
}
if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render),
&render->queue)) {
GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements");
render->queue = _find_graphics_queue (render);
}
if (!render->queue)
return FALSE;
return TRUE;
}
static gboolean
gst_vulkan_video_filter_stop (GstBaseTransform * bt)
{
GstVulkanVideoFilter *render = GST_VULKAN_VIDEO_FILTER (bt);
gst_clear_object (&render->device);
gst_clear_object (&render->queue);
gst_clear_object (&render->instance);
return TRUE;
}

63
gst-libs/gst/vulkan/gstvkvideofilter.h Normal file
View file

@ -0,0 +1,63 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _VK_FULL_SCREEN_RENDER_H_
#define _VK_FULL_SCREEN_RENDER_H_
#include <gst/gst.h>
#include <gst/video/video.h>
#include <gst/vulkan/gstvkqueue.h>
G_BEGIN_DECLS
GST_VULKAN_API
GType gst_vulkan_video_filter_get_type(void);
#define GST_TYPE_VULKAN_VIDEO_FILTER (gst_vulkan_video_filter_get_type())
#define GST_VULKAN_VIDEO_FILTER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilter))
#define GST_VULKAN_VIDEO_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilterClass))
#define GST_VULKAN_VIDEO_FILTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VULKAN_VIDEO_FILTER,GstVulkanVideoFilterClass))
#define GST_IS_VULKAN_VIDEO_FILTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_VIDEO_FILTER))
#define GST_IS_VULKAN_VIDEO_FILTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_VIDEO_FILTER))
typedef struct _GstVulkanVideoFilter GstVulkanVideoFilter;
typedef struct _GstVulkanVideoFilterClass GstVulkanVideoFilterClass;
struct _GstVulkanVideoFilter
{
GstBaseTransform parent;
GstVulkanInstance *instance;
GstVulkanDevice *device;
GstVulkanQueue *queue;
GstCaps *in_caps;
GstVideoInfo in_info;
GstCaps *out_caps;
GstVideoInfo out_info;
};
struct _GstVulkanVideoFilterClass
{
GstBaseTransformClass video_sink_class;
};
G_END_DECLS
#endif

2
gst-libs/gst/vulkan/meson.build
View file

@ -27,6 +27,7 @@ vulkan_sources = [
'gstvkqueue.c',
'gstvkswapper.c',
'gstvktrash.c',
'gstvkvideofilter.c',
'gstvkutils.c',
'gstvkwindow.c',
]
@ -58,6 +59,7 @@ vulkan_headers = [
'gstvkswapper.h',
'gstvktrash.h',
'gstvkutils.h',
'gstvkvideofilter.h',
'gstvkwindow.h',
'vulkan-prelude.h',
'vulkan_fwd.h',

4
gst-libs/gst/vulkan/vulkan.h
View file

@ -25,6 +25,7 @@
#include <gst/vulkan/gstvkapi.h>
/* helper vulkan objects */
#include <gst/vulkan/gstvkdebug.h>
#include <gst/vulkan/gstvkerror.h>
#include <gst/vulkan/gstvkinstance.h>
@ -52,4 +53,7 @@
#include <gst/vulkan/gstvktrash.h>
#include <gst/vulkan/gstvkswapper.h>
/* helper elements */
#include <gst/vulkan/gstvkvideofilter.h>
#endif /* __GST_VULKAN_H__ */