diff --git a/ext/vulkan/meson.build b/ext/vulkan/meson.build index 4883de9ae8..99b2b37325 100644 --- a/ext/vulkan/meson.build +++ b/ext/vulkan/meson.build @@ -10,6 +10,7 @@ subdir('shaders') vulkan_sources = [ 'gstvulkan.c', + 'vkfullscreenrender.c', 'vkimageidentity.c', 'vksink.c', 'vkshader.c', diff --git a/ext/vulkan/shaders/identity.frag b/ext/vulkan/shaders/identity.frag index a7a476446a..a998ee44d7 100644 --- a/ext/vulkan/shaders/identity.frag +++ b/ext/vulkan/shaders/identity.frag @@ -1,4 +1,4 @@ -#version 420 core +#version 450 core layout(location = 0) in vec2 inTexCoord; diff --git a/ext/vulkan/shaders/identity.vert b/ext/vulkan/shaders/identity.vert index 67e61df342..84e9b2464d 100644 --- a/ext/vulkan/shaders/identity.vert +++ b/ext/vulkan/shaders/identity.vert @@ -1,4 +1,4 @@ -#version 420 core +#version 450 core layout(location = 0) in vec4 inPos; layout(location = 1) in vec2 inTexCoord; diff --git a/ext/vulkan/vkfullscreenrender.c b/ext/vulkan/vkfullscreenrender.c new file mode 100644 index 0000000000..b48c154845 --- /dev/null +++ b/ext/vulkan/vkfullscreenrender.c @@ -0,0 +1,880 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-vulkanimageidentity + * @title: vulkanimgeidentity + * + * vulkanimageidentity produces a vulkan image that is a copy of the input image. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "vkimageidentity.h" +#include "vktrash.h" +#include "vkshader.h" + +GST_DEBUG_CATEGORY (gst_debug_vulkan_full_screen_render); +#define GST_CAT_DEFAULT gst_debug_vulkan_full_screen_render + +struct Vertex vertices[] = { + {-1.0f, -1.0f, 0.0f, 0.0f, 0.0f}, + {1.0f, -1.0f, 0.0f, 1.0f, 0.0f}, + {1.0f, 1.0f, 0.0f, 1.0f, 1.0f}, + {-1.0f, 1.0f, 0.0f, 0.0f, 1.0f}, +}; + +gushort indices[] = { + 0, 1, 2, 0, 2, 3, +}; + +static void gst_vulkan_full_screen_render_finalize (GObject * object); + +static gboolean gst_vulkan_full_screen_render_query (GstBaseTransform * bt, + GstPadDirection direction, GstQuery * query); +static void gst_vulkan_full_screen_render_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_vulkan_full_screen_render_start (GstBaseTransform * bt); +static gboolean gst_vulkan_full_screen_render_stop (GstBaseTransform * bt); + +static gboolean gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt, + GstCaps * in_caps, GstCaps * out_caps); +static GstCaps *gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * + bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter); +static gboolean +gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt, + GstQuery * decide_query, GstQuery * query); +static gboolean +gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt, + GstQuery * query); + +#define IMAGE_FORMATS " { BGRA }" + +static GstStaticPadTemplate gst_vulkan_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, + IMAGE_FORMATS))); + +static GstStaticPadTemplate gst_vulkan_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, + IMAGE_FORMATS))); + +enum +{ + PROP_0, +}; + +enum +{ + SIGNAL_0, + LAST_SIGNAL +}; + +/* static guint gst_vulkan_full_screen_render_signals[LAST_SIGNAL] = { 0 }; */ + +#define gst_vulkan_full_screen_render_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstVulkanFullScreenRender, + gst_vulkan_full_screen_render, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_full_screen_render, + "vulkanimageidentity", 0, "Vulkan Image identity")); + +static void +gst_vulkan_full_screen_render_class_init (GstVulkanFullScreenRenderClass * + klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstBaseTransformClass *gstbasetransform_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstbasetransform_class = (GstBaseTransformClass *) klass; + + gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", + "Filter/Video", "A Vulkan image copier", + "Matthew Waters "); + + gst_element_class_add_static_pad_template (gstelement_class, + &gst_vulkan_sink_template); + gst_element_class_add_static_pad_template (gstelement_class, + &gst_vulkan_src_template); + + gobject_class->finalize = gst_vulkan_full_screen_render_finalize; + + gstelement_class->set_context = gst_vulkan_full_screen_render_set_context; + gstbasetransform_class->start = + GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_start); + gstbasetransform_class->stop = + GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_stop); + gstbasetransform_class->query = + GST_DEBUG_FUNCPTR (gst_vulkan_full_screen_render_query); + gstbasetransform_class->set_caps = gst_vulkan_full_screen_render_set_caps; + gstbasetransform_class->transform_caps = + gst_vulkan_full_screen_render_transform_caps; + gstbasetransform_class->propose_allocation = + gst_vulkan_full_screen_render_propose_allocation; + gstbasetransform_class->decide_allocation = + gst_vulkan_full_screen_render_decide_allocation; +} + +static void +gst_vulkan_full_screen_render_init (GstVulkanFullScreenRender * render) +{ +} + +static void +gst_vulkan_full_screen_render_finalize (GObject * object) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (object); + + gst_caps_replace (&render->in_caps, NULL); + gst_caps_replace (&render->out_caps, NULL); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +gst_vulkan_full_screen_render_query (GstBaseTransform * bt, + GstPadDirection direction, GstQuery * query) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT:{ + if (gst_vulkan_handle_context_query (GST_ELEMENT (render), query, + NULL, &render->instance, &render->device)) + return TRUE; + + if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (render), + query, &render->queue)) + return TRUE; + + break; + } + default: + break; + } + + return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query); +} + +static void +gst_vulkan_full_screen_render_set_context (GstElement * element, + GstContext * context) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (element); + + gst_vulkan_handle_set_context (element, context, NULL, &render->instance); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +struct choose_data +{ + GstVulkanFullScreenRender *upload; + GstVulkanQueue *queue; +}; + +static gboolean +_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue, + struct choose_data *data) +{ + guint flags = device->queue_family_props[queue->family].queueFlags; + + GST_ERROR ("flags 0x%x", flags); + + if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) { + if (data->queue) + gst_object_unref (data->queue); + data->queue = gst_object_ref (queue); + return FALSE; + } + + return TRUE; +} + +static GstVulkanQueue * +_find_graphics_queue (GstVulkanFullScreenRender * upload) +{ + struct choose_data data; + + data.upload = upload; + data.queue = NULL; + + gst_vulkan_device_foreach_queue (upload->device, + (GstVulkanDeviceForEachQueueFunc) _choose_queue, &data); + + return data.queue; +} + +static GstCaps * +gst_vulkan_full_screen_render_transform_caps (GstBaseTransform * bt, + GstPadDirection direction, GstCaps * caps, GstCaps * filter) +{ + GstCaps *result, *tmp; + + tmp = gst_caps_copy (caps); + + if (filter) { + result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (tmp); + } else { + result = tmp; + } + + return result; +} + +static void +clear_shader_create_info (GstVulkanFullScreenRender * render) +{ + if (render->shader_create_info) { + if (render->destroy_shader_create_info) + render->destroy_shader_create_info (render, render->shader_create_info); + } + render->n_shader_stages = 0; + render->shader_create_info = NULL; + render->destroy_shader_create_info = NULL; +} + +static VkPipeline +_create_pipeline (GstVulkanFullScreenRender * render) +{ + GstVulkanFullScreenRenderClass *render_class = + GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); + VkVertexInputBindingDescription vertex_binding_description; + VkVertexInputAttributeDescription vertex_attribute_description[2]; + VkPipelineVertexInputStateCreateInfo vertex_input_info; + VkPipelineInputAssemblyStateCreateInfo input_assembly; + VkPipelineViewportStateCreateInfo viewport_state; + VkPipelineRasterizationStateCreateInfo rasterizer; + VkPipelineMultisampleStateCreateInfo multisampling; + VkPipelineColorBlendAttachmentState color_blend_attachment; + VkPipelineColorBlendStateCreateInfo color_blending; + VkGraphicsPipelineCreateInfo pipeline_info; + VkPipeline pipeline; + GError *error = NULL; + VkResult err; + + render_class->shader_create_info (render); + + /* *INDENT-OFF* */ + vertex_binding_description = (VkVertexInputBindingDescription) { + .binding = 0, + .stride = sizeof (struct Vertex), + .inputRate = VK_VERTEX_INPUT_RATE_VERTEX + }; + + vertex_attribute_description[0] = (VkVertexInputAttributeDescription) { + .binding = 0, + .location = 0, + .format = VK_FORMAT_R32G32B32_SFLOAT, + .offset = G_STRUCT_OFFSET (struct Vertex, x) + }; + vertex_attribute_description[1] = (VkVertexInputAttributeDescription) { + .binding = 0, + .location = 1, + .format = VK_FORMAT_R32G32_SFLOAT, + .offset = G_STRUCT_OFFSET (struct Vertex, s) + }; + + vertex_input_info = (VkPipelineVertexInputStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, + .pNext = NULL, + .vertexBindingDescriptionCount = 1, + .pVertexBindingDescriptions = &vertex_binding_description, + .vertexAttributeDescriptionCount = G_N_ELEMENTS (vertex_attribute_description), + .pVertexAttributeDescriptions = vertex_attribute_description + }; + + input_assembly = (VkPipelineInputAssemblyStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, + .pNext = NULL, + .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, + .primitiveRestartEnable = VK_FALSE + }; + + viewport_state = (VkPipelineViewportStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, + .pNext = NULL, + .viewportCount = 1, + .pViewports = &(VkViewport) { + .x = 0.0f, + .y = 0.0f, + .width = (float) GST_VIDEO_INFO_WIDTH (&render->out_info), + .height = (float) GST_VIDEO_INFO_HEIGHT (&render->out_info), + .minDepth = 0.0f, + .maxDepth = 1.0f + }, + .scissorCount = 1, + .pScissors = &(VkRect2D) { + .offset = { 0, 0 }, + .extent = { + GST_VIDEO_INFO_WIDTH (&render->out_info), + GST_VIDEO_INFO_HEIGHT (&render->out_info) + } + } + }; + + rasterizer = (VkPipelineRasterizationStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, + .pNext = NULL, + .depthClampEnable = VK_FALSE, + .rasterizerDiscardEnable = VK_FALSE, + .polygonMode = VK_POLYGON_MODE_FILL, + .lineWidth = 1.0f, + .cullMode = VK_CULL_MODE_BACK_BIT, + .frontFace = VK_FRONT_FACE_CLOCKWISE, + .depthBiasEnable = VK_FALSE + }; + + multisampling = (VkPipelineMultisampleStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pNext = NULL, + .sampleShadingEnable = VK_FALSE, + .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT + }; + + color_blend_attachment = (VkPipelineColorBlendAttachmentState) { + .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, + .blendEnable = VK_FALSE + }; + + color_blending = (VkPipelineColorBlendStateCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = NULL, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_COPY, + .attachmentCount = 1, + .pAttachments = &color_blend_attachment, + .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f } + }; + + pipeline_info = (VkGraphicsPipelineCreateInfo) { + .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, + .pNext = NULL, + .stageCount = render->n_shader_stages, + .pStages = render->shader_create_info, + .pVertexInputState = &vertex_input_info, + .pInputAssemblyState = &input_assembly, + .pViewportState = &viewport_state, + .pRasterizationState = &rasterizer, + .pMultisampleState = &multisampling, + .pColorBlendState = &color_blending, + .layout = render->pipeline_layout, + .renderPass = render->render_pass, + .subpass = 0, + .basePipelineHandle = VK_NULL_HANDLE + }; + /* *INDENT-ON* */ + + err = + vkCreateGraphicsPipelines (render->device->device, VK_NULL_HANDLE, 1, + &pipeline_info, NULL, &pipeline); + clear_shader_create_info (render); + if (gst_vulkan_error_to_g_error (err, &error, + "vkCreateGraphicsPipelines") < 0) { + GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return pipeline; +} + +static VkPipelineLayout +_create_pipeline_layout (GstVulkanFullScreenRender * render) +{ + GstVulkanFullScreenRenderClass *render_class = + GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); + VkPipelineLayoutCreateInfo pipeline_layout_info; + VkPipelineLayout pipeline_layout; + VkPushConstantRange *constants = NULL; + guint n_constants = 0; + GError *error = NULL; + VkResult err; + + if (render_class->push_constant_ranges) + constants = render_class->push_constant_ranges (render, &n_constants); + + /* *INDENT-OFF* */ + pipeline_layout_info = (VkPipelineLayoutCreateInfo) { + .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, + .pNext = NULL, + .setLayoutCount = 1, + .pSetLayouts = &render->descriptor_set_layout, + .pushConstantRangeCount = n_constants, + .pPushConstantRanges = constants, + }; + /* *INDENT-ON* */ + + err = + vkCreatePipelineLayout (render->device->device, + &pipeline_layout_info, NULL, &pipeline_layout); + g_free (constants); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreatePipelineLayout") < 0) { + GST_ERROR_OBJECT (render, "Failed to create pipeline layout: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return pipeline_layout; +} + +static VkRenderPass +_create_render_pass (GstVulkanFullScreenRender * render) +{ + GstVulkanFullScreenRenderClass *render_class = + GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); + + guint n_descriptions; + VkAttachmentDescription *descriptions = + render_class->render_pass_attachment_descriptions (render, + &n_descriptions); + + guint n_refs; + VkAttachmentReference *color_attachment_refs = + render_class->render_pass_attachment_references (render, &n_refs); + + /* *INDENT-OFF* */ + VkSubpassDescription subpass = { + .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, + .colorAttachmentCount = n_refs, + .pColorAttachments = color_attachment_refs + }; + + VkRenderPassCreateInfo render_pass_info = { + .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, + .pNext = NULL, + .attachmentCount = n_descriptions, + .pAttachments = descriptions, + .subpassCount = 1, + .pSubpasses = &subpass + }; + /* *INDENT-ON* */ + VkRenderPass render_pass = NULL; + VkResult err; + GError *error = NULL; + + err = + vkCreateRenderPass (render->device->device, &render_pass_info, NULL, + &render_pass); + g_free (color_attachment_refs); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateRenderPass") < 0) { + GST_ERROR_OBJECT (render, "Failed to create renderpass: %s", + error->message); + return NULL; + } + + return render_pass; +} + +static gboolean +gst_vulkan_full_screen_render_set_caps (GstBaseTransform * bt, + GstCaps * in_caps, GstCaps * out_caps) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + if (!gst_video_info_from_caps (&render->in_info, in_caps)) + return FALSE; + if (!gst_video_info_from_caps (&render->out_info, out_caps)) + return FALSE; + + gst_caps_replace (&render->in_caps, in_caps); + gst_caps_replace (&render->out_caps, out_caps); + + if (render->render_pass) { + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_render_pass (gst_vulkan_fence_ref + (render->last_fence), render->render_pass)); + render->render_pass = NULL; + } + + if (!(render->render_pass = _create_render_pass (render))) + return FALSE; + + if (render->graphics_pipeline) { + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_pipeline (gst_vulkan_fence_ref + (render->last_fence), render->graphics_pipeline)); + render->graphics_pipeline = NULL; + } + if (!(render->graphics_pipeline = _create_pipeline (render))) + return FALSE; + + GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps); + + return TRUE; +} + +static gboolean +gst_vulkan_full_screen_render_propose_allocation (GstBaseTransform * bt, + GstQuery * decide_query, GstQuery * query) +{ + /* FIXME: */ + return FALSE; +} + +static gboolean +gst_vulkan_full_screen_render_decide_allocation (GstBaseTransform * bt, + GstQuery * query) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + GstBufferPool *pool = NULL; + GstStructure *config; + GstCaps *caps; + guint min, max, size; + gboolean update_pool; + + gst_query_parse_allocation (query, &caps, NULL); + if (!caps) + return FALSE; + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + update_pool = TRUE; + } else { + GstVideoInfo vinfo; + + gst_video_info_init (&vinfo); + gst_video_info_from_caps (&vinfo, caps); + size = vinfo.size; + min = max = 0; + update_pool = FALSE; + } + + if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) { + if (pool) + gst_object_unref (pool); + pool = gst_vulkan_image_buffer_pool_new (render->device); + } + + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + gst_buffer_pool_set_config (pool, config); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + gst_object_unref (pool); + + return TRUE; +} + +static VkDescriptorSetLayout +_create_descriptor_set_layout (GstVulkanFullScreenRender * render) +{ + GstVulkanFullScreenRenderClass *render_class = + GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS (render); + guint n_bindings; + VkDescriptorSetLayoutBinding *bindings = + render_class->descriptor_set_layout_bindings (render, &n_bindings); + + /* *INDENT-OFF* */ + VkDescriptorSetLayoutCreateInfo layout_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + .pNext = NULL, + .bindingCount = n_bindings, + .pBindings = bindings + }; + /* *INDENT-ON* */ + VkDescriptorSetLayout descriptor_set_layout; + VkResult err; + GError *error = NULL; + + err = + vkCreateDescriptorSetLayout (render->device->device, &layout_info, + NULL, &descriptor_set_layout); + g_free (bindings); + if (gst_vulkan_error_to_g_error (err, &error, + "vkCreateDescriptorSetLayout") < 0) { + GST_ERROR_OBJECT (render, "Failed to create renderpass: %s", + error->message); + return NULL; + } + + return descriptor_set_layout; +} + +static gboolean +_create_vertex_buffers (GstVulkanFullScreenRender * render) +{ + GstMapInfo map_info; + + render->vertices = + gst_vulkan_buffer_memory_alloc (render->device, VK_FORMAT_R8_UNORM, + sizeof (vertices), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + + if (!gst_memory_map (render->vertices, &map_info, GST_MAP_WRITE)) { + gst_memory_unref (render->vertices); + render->vertices = NULL; + return FALSE; + } + memcpy (map_info.data, vertices, sizeof (vertices)); + gst_memory_unmap (render->vertices, &map_info); + + render->indices = + gst_vulkan_buffer_memory_alloc (render->device, VK_FORMAT_R8_UNORM, + sizeof (indices), + VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + + if (!gst_memory_map (render->indices, &map_info, GST_MAP_WRITE)) { + gst_memory_unref (render->vertices); + render->vertices = NULL; + gst_memory_unref (render->indices); + render->indices = NULL; + return FALSE; + } + memcpy (map_info.data, indices, sizeof (indices)); + gst_memory_unmap (render->indices, &map_info); + + return TRUE; +} + +static gboolean +gst_vulkan_full_screen_render_start (GstBaseTransform * bt) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL, + &render->instance)) { + GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, + ("Failed to retreive vulkan instance"), (NULL)); + return FALSE; + } + if (!gst_vulkan_device_run_context_query (GST_ELEMENT (render), + &render->device)) { + GError *error = NULL; + GST_DEBUG_OBJECT (render, "No device retrieved from peer elements"); + if (!(render->device = + gst_vulkan_instance_create_device (render->instance, &error))) { + GST_ELEMENT_ERROR (render, RESOURCE, NOT_FOUND, + ("Failed to create vulkan device"), ("%s", error->message)); + g_clear_error (&error); + return FALSE; + } + } + + if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (render), + &render->queue)) { + GST_DEBUG_OBJECT (render, "No queue retrieved from peer elements"); + render->queue = _find_graphics_queue (render); + } + if (!render->queue) + return FALSE; + + if (!(render->descriptor_set_layout = _create_descriptor_set_layout (render))) + return FALSE; + if (!(render->pipeline_layout = _create_pipeline_layout (render))) + return FALSE; + + if (!_create_vertex_buffers (render)) + return FALSE; + + return TRUE; +} + +static gboolean +gst_vulkan_full_screen_render_stop (GstBaseTransform * bt) +{ + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); + + if (render->device) { + if (render->last_fence) { + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_pipeline (gst_vulkan_fence_ref + (render->last_fence), render->graphics_pipeline)); + render->graphics_pipeline = NULL; + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_pipeline_layout (gst_vulkan_fence_ref + (render->last_fence), render->pipeline_layout)); + render->pipeline_layout = NULL; + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_render_pass (gst_vulkan_fence_ref + (render->last_fence), render->render_pass)); + render->render_pass = NULL; + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_descriptor_set_layout (gst_vulkan_fence_ref + (render->last_fence), render->descriptor_set_layout)); + render->descriptor_set_layout = NULL; + + gst_vulkan_fence_unref (render->last_fence); + render->last_fence = NULL; + } else { + vkDestroyPipeline (render->device->device, + render->graphics_pipeline, NULL); + render->graphics_pipeline = NULL; + + vkDestroyPipelineLayout (render->device->device, + render->pipeline_layout, NULL); + render->pipeline_layout = NULL; + + vkDestroyRenderPass (render->device->device, render->render_pass, NULL); + render->render_pass = NULL; + + vkDestroyDescriptorSetLayout (render->device->device, + render->descriptor_set_layout, NULL); + render->descriptor_set_layout = NULL; + } + + if (!gst_vulkan_trash_list_wait (render->trash_list, -1)) + GST_WARNING_OBJECT (render, + "Failed to wait for all resources to be freed"); + render->trash_list = NULL; + + if (render->vertices) + gst_memory_unref (render->vertices); + render->vertices = NULL; + + if (render->indices) + gst_memory_unref (render->indices); + render->indices = NULL; + + gst_object_unref (render->device); + } + render->device = NULL; + + if (render->queue) + gst_object_unref (render->queue); + render->queue = NULL; + + if (render->instance) + gst_object_unref (render->instance); + render->instance = NULL; + + return TRUE; +} + +/** + * gst_vulkan_full_screen_render_fill_command_buffer: + * @render: a #GstVulkanFullScreenRender + * @cmd: a `VkCommandBuffer` + * @framebuffer: a `VkFramebuffer` + * + * Returns: whether @cmd could be filled with the commands necessary to render + */ +gboolean +gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender * + render, VkCommandBuffer cmd, VkFramebuffer framebuffer) +{ + /* *INDENT-OFF* */ + VkClearValue clearColor = {{{ 0.0f, 0.0f, 0.0f, 1.0f }}}; + VkRenderPassBeginInfo render_pass_info = { + .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, + .renderPass = render->render_pass, + .framebuffer = framebuffer, + .renderArea.offset = { 0, 0 }, + .renderArea.extent = { + GST_VIDEO_INFO_WIDTH (&render->out_info), + GST_VIDEO_INFO_HEIGHT (&render->out_info) + }, + .clearValueCount = 1, + .pClearValues = &clearColor + }; + /* *INDENI-ON* */ + VkDeviceSize offsets[] = { 0 }; + + vkCmdBeginRenderPass (cmd, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); + vkCmdBindPipeline (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + render->graphics_pipeline); + vkCmdBindVertexBuffers (cmd, 0, 1, + &((GstVulkanBufferMemory *) render->vertices)->buffer, offsets); + vkCmdBindIndexBuffer (cmd, + ((GstVulkanBufferMemory *) render->indices)->buffer, 0, + VK_INDEX_TYPE_UINT16); + vkCmdDrawIndexed (cmd, G_N_ELEMENTS (indices), 1, 0, 0, 0); + vkCmdEndRenderPass (cmd); + + return TRUE; +} + +gboolean +gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render, + VkCommandBuffer cmd, GstVulkanFence * fence) +{ + VkSubmitInfo submit_info; + GError *error = NULL; + VkResult err; + + if (!fence) + fence = gst_vulkan_fence_new (render->device, 0, &error); + if (!fence) + goto error; + + /* *INDENT-OFF* */ + submit_info = (VkSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + .pNext = NULL, + .waitSemaphoreCount = 0, + .pWaitSemaphores = NULL, + .pWaitDstStageMask = NULL, + .commandBufferCount = 1, + .pCommandBuffers = &cmd, + .signalSemaphoreCount = 0, + .pSignalSemaphores = NULL, + }; + /* *INDENT-ON* */ + + if (render->last_fence) + gst_vulkan_fence_unref (render->last_fence); + render->last_fence = gst_vulkan_fence_ref (fence); + + err = + vkQueueSubmit (render->queue->queue, 1, &submit_info, + GST_VULKAN_FENCE_FENCE (fence)); + if (gst_vulkan_error_to_g_error (err, &error, "vkQueueSubmit") < 0) + goto error; + + render->trash_list = gst_vulkan_trash_list_gc (render->trash_list); + + gst_vulkan_fence_unref (fence); + + return TRUE; + +error: + GST_ELEMENT_ERROR (render, LIBRARY, FAILED, ("%s", error->message), (NULL)); + g_clear_error (&error); + return FALSE; +} diff --git a/ext/vulkan/vkfullscreenrender.h b/ext/vulkan/vkfullscreenrender.h new file mode 100644 index 0000000000..2d90fefabf --- /dev/null +++ b/ext/vulkan/vkfullscreenrender.h @@ -0,0 +1,96 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _VK_FULL_SCREEN_RENDER_H_ +#define _VK_FULL_SCREEN_RENDER_H_ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_VULKAN_FULL_SCREEN_RENDER (gst_vulkan_full_screen_render_get_type()) +#define GST_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRender)) +#define GST_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass)) +#define GST_VULKAN_FULL_SCREEN_RENDER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER,GstVulkanFullScreenRenderClass)) +#define GST_IS_VULKAN_FULL_SCREEN_RENDER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_FULL_SCREEN_RENDER)) +#define GST_IS_VULKAN_FULL_SCREEN_RENDER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VULKAN_FULL_SCREEN_RENDER)) + +typedef struct _GstVulkanFullScreenRender GstVulkanFullScreenRender; +typedef struct _GstVulkanFullScreenRenderClass GstVulkanFullScreenRenderClass; + +typedef void (*GstVulkanFullScreenRenderDestroyInfoNotify) + (GstVulkanFullScreenRender * render, gpointer info); + +struct Vertex +{ + gfloat x, y, z; + gfloat s, t; +}; + +struct _GstVulkanFullScreenRender +{ + GstBaseTransform parent; + + GstVulkanInstance *instance; + GstVulkanDevice *device; + GstVulkanQueue *queue; + + GstCaps *in_caps; + GstVideoInfo in_info; + GstCaps *out_caps; + GstVideoInfo out_info; + + guint n_shader_stages; + VkPipelineShaderStageCreateInfo *shader_create_info; + GstVulkanFullScreenRenderDestroyInfoNotify destroy_shader_create_info; + + VkRenderPass render_pass; + VkPipelineLayout pipeline_layout; + VkPipeline graphics_pipeline; + VkDescriptorSetLayout descriptor_set_layout; + + GstMemory *vertices; + GstMemory *indices; + + GList *trash_list; + GstVulkanFence *last_fence; +}; + +struct _GstVulkanFullScreenRenderClass +{ + GstBaseTransformClass video_sink_class; + + void (*shader_create_info) (GstVulkanFullScreenRender * render); + VkDescriptorSetLayoutBinding * (*descriptor_set_layout_bindings) (GstVulkanFullScreenRender * render, guint * n_bindings); + VkAttachmentReference * (*render_pass_attachment_references) (GstVulkanFullScreenRender * render, guint * n_refs); + VkAttachmentDescription * (*render_pass_attachment_descriptions) (GstVulkanFullScreenRender * render, guint * n_descriptions); + VkPushConstantRange * (*push_constant_ranges) (GstVulkanFullScreenRender * render, guint * n_constants); +}; + +gboolean gst_vulkan_full_screen_render_fill_command_buffer (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, VkFramebuffer framebuffer); +gboolean gst_vulkan_full_screen_render_submit (GstVulkanFullScreenRender * render, VkCommandBuffer cmd, GstVulkanFence *fence); + +GType gst_vulkan_full_screen_render_get_type(void); + +G_END_DECLS + +#endif diff --git a/ext/vulkan/vkimageidentity.c b/ext/vulkan/vkimageidentity.c index c228e2044f..469556d7ef 100644 --- a/ext/vulkan/vkimageidentity.c +++ b/ext/vulkan/vkimageidentity.c @@ -40,48 +40,27 @@ GST_DEBUG_CATEGORY (gst_debug_vulkan_image_identity); #define GST_CAT_DEFAULT gst_debug_vulkan_image_identity -struct Vertex -{ - gfloat x, y, z; - gfloat s, t; -}; - -struct Vertex vertices[] = { - {-1.0f, -1.0f, 0.0f, 0.0f, 0.0f}, - {1.0f, -1.0f, 0.0f, 1.0f, 0.0f}, - {1.0f, 1.0f, 0.0f, 1.0f, 1.0f}, - {-1.0f, 1.0f, 0.0f, 0.0f, 1.0f}, -}; - -gushort indices[] = { - 0, 1, 2, 0, 2, 3, -}; - -static void gst_vulkan_image_identity_finalize (GObject * object); -static void gst_vulkan_image_identity_set_property (GObject * object, - guint prop_id, const GValue * value, GParamSpec * param_spec); -static void gst_vulkan_image_identity_get_property (GObject * object, - guint prop_id, GValue * value, GParamSpec * param_spec); - -static gboolean gst_vulkan_image_identity_query (GstBaseTransform * bt, - GstPadDirection direction, GstQuery * query); -static void gst_vulkan_image_identity_set_context (GstElement * element, - GstContext * context); - static gboolean gst_vulkan_image_identity_start (GstBaseTransform * bt); static gboolean gst_vulkan_image_identity_stop (GstBaseTransform * bt); -static gboolean gst_vulkan_image_identity_set_caps (GstBaseTransform * bt, - GstCaps * in_caps, GstCaps * out_caps); static GstCaps *gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter); -static gboolean gst_vulkan_image_identity_propose_allocation (GstBaseTransform * - bt, GstQuery * decide_query, GstQuery * query); -static gboolean gst_vulkan_image_identity_decide_allocation (GstBaseTransform * - bt, GstQuery * query); static GstFlowReturn gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf); +static VkAttachmentReference + * gst_vulkan_image_identity_render_pass_attachment_references + (GstVulkanFullScreenRender * render, guint * n_attachments); +static VkAttachmentDescription + * gst_vulkan_image_identity_render_pass_attachment_descriptions + (GstVulkanFullScreenRender * render, guint * n_descriptions); +static VkDescriptorSetLayoutBinding + * gst_vulkan_image_identity_descriptor_set_layout_bindings + (GstVulkanFullScreenRender * render, guint * n_bindings); +static void +gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender * + render); + #define IMAGE_FORMATS " { BGRA }" static GstStaticPadTemplate gst_vulkan_sink_template = @@ -115,23 +94,20 @@ enum #define gst_vulkan_image_identity_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstVulkanImageIdentity, gst_vulkan_image_identity, - GST_TYPE_BASE_TRANSFORM, + GST_TYPE_VULKAN_FULL_SCREEN_RENDER, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_image_identity, "vulkanimageidentity", 0, "Vulkan Image identity")); static void gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass) { - GObjectClass *gobject_class; GstElementClass *gstelement_class; GstBaseTransformClass *gstbasetransform_class; + GstVulkanFullScreenRenderClass *fullscreenrender_class; - gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; gstbasetransform_class = (GstBaseTransformClass *) klass; - - gobject_class->set_property = gst_vulkan_image_identity_set_property; - gobject_class->get_property = gst_vulkan_image_identity_get_property; + fullscreenrender_class = (GstVulkanFullScreenRenderClass *) klass; gst_element_class_set_metadata (gstelement_class, "Vulkan Uploader", "Filter/Video", "A Vulkan image copier", @@ -142,23 +118,22 @@ gst_vulkan_image_identity_class_init (GstVulkanImageIdentityClass * klass) gst_element_class_add_static_pad_template (gstelement_class, &gst_vulkan_src_template); - gobject_class->finalize = gst_vulkan_image_identity_finalize; - - gstelement_class->set_context = gst_vulkan_image_identity_set_context; gstbasetransform_class->start = GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_start); gstbasetransform_class->stop = GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_stop); - gstbasetransform_class->query = - GST_DEBUG_FUNCPTR (gst_vulkan_image_identity_query); - gstbasetransform_class->set_caps = gst_vulkan_image_identity_set_caps; gstbasetransform_class->transform_caps = gst_vulkan_image_identity_transform_caps; - gstbasetransform_class->propose_allocation = - gst_vulkan_image_identity_propose_allocation; - gstbasetransform_class->decide_allocation = - gst_vulkan_image_identity_decide_allocation; gstbasetransform_class->transform = gst_vulkan_image_identity_transform; + + fullscreenrender_class->render_pass_attachment_references = + gst_vulkan_image_identity_render_pass_attachment_references; + fullscreenrender_class->render_pass_attachment_descriptions = + gst_vulkan_image_identity_render_pass_attachment_descriptions; + fullscreenrender_class->descriptor_set_layout_bindings = + gst_vulkan_image_identity_descriptor_set_layout_bindings; + fullscreenrender_class->shader_create_info = + gst_vulkan_image_identity_shader_create_info; } static void @@ -166,117 +141,6 @@ gst_vulkan_image_identity_init (GstVulkanImageIdentity * vk_identity) { } -static void -gst_vulkan_image_identity_finalize (GObject * object) -{ - GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (object); - - gst_caps_replace (&vk_identity->caps, NULL); - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static void -gst_vulkan_image_identity_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ -// GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (object); - - switch (prop_id) { - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_vulkan_image_identity_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ -// GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (object); - - switch (prop_id) { - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static gboolean -gst_vulkan_image_identity_query (GstBaseTransform * bt, - GstPadDirection direction, GstQuery * query) -{ - GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - - switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONTEXT:{ - if (gst_vulkan_handle_context_query (GST_ELEMENT (vk_identity), query, - NULL, &vk_identity->instance, &vk_identity->device)) - return TRUE; - - if (gst_vulkan_queue_handle_context_query (GST_ELEMENT (vk_identity), - query, &vk_identity->queue)) - return TRUE; - - break; - } - default: - break; - } - - return GST_BASE_TRANSFORM_CLASS (parent_class)->query (bt, direction, query); -} - -static void -gst_vulkan_image_identity_set_context (GstElement * element, - GstContext * context) -{ - GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (element); - - gst_vulkan_handle_set_context (element, context, NULL, - &vk_identity->instance); - - GST_ELEMENT_CLASS (parent_class)->set_context (element, context); -} - -struct choose_data -{ - GstVulkanImageIdentity *upload; - GstVulkanQueue *queue; -}; - -static gboolean -_choose_queue (GstVulkanDevice * device, GstVulkanQueue * queue, - struct choose_data *data) -{ - guint flags = device->queue_family_props[queue->family].queueFlags; - - GST_ERROR ("flags 0x%x", flags); - - if ((flags & VK_QUEUE_GRAPHICS_BIT) != 0) { - if (data->queue) - gst_object_unref (data->queue); - data->queue = gst_object_ref (queue); - return FALSE; - } - - return TRUE; -} - -static GstVulkanQueue * -_find_graphics_queue (GstVulkanImageIdentity * upload) -{ - struct choose_data data; - - data.upload = upload; - data.queue = NULL; - - gst_vulkan_device_foreach_queue (upload->device, - (GstVulkanDeviceForEachQueueFunc) _choose_queue, &data); - - return data.queue; -} - static GstCaps * gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter) @@ -295,18 +159,39 @@ gst_vulkan_image_identity_transform_caps (GstBaseTransform * bt, return result; } -static VkPipeline -_create_pipeline (GstVulkanImageIdentity * vk_identity) +static void +destroy_shader_create_info (GstVulkanFullScreenRender * render, gpointer data) { - VkShaderModule vert_module = - _vk_create_shader (vk_identity->device, identity_vert, identity_vert_size, + VkPipelineShaderStageCreateInfo *info = data; + int i; + + for (i = 0; i < render->n_shader_stages; i++) { + vkDestroyShaderModule (render->device->device, info[i].module, NULL); + } + + g_free (info); +} + +static void +gst_vulkan_image_identity_shader_create_info (GstVulkanFullScreenRender * + render) +{ + VkShaderModule vert_module, frag_module; + + vert_module = + _vk_create_shader (render->device, identity_vert, identity_vert_size, NULL); - VkShaderModule frag_module = - _vk_create_shader (vk_identity->device, identity_frag, identity_frag_size, + frag_module = + _vk_create_shader (render->device, identity_frag, identity_frag_size, NULL); + render->n_shader_stages = 2; + render->shader_create_info = + g_new0 (VkPipelineShaderStageCreateInfo, render->n_shader_stages); + render->destroy_shader_create_info = destroy_shader_create_info; + /* *INDENT-OFF* */ - VkPipelineShaderStageCreateInfo vert_stage = { + render->shader_create_info[0] = (VkPipelineShaderStageCreateInfo) { .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = NULL, .stage = VK_SHADER_STAGE_VERTEX_BIT, @@ -314,271 +199,65 @@ _create_pipeline (GstVulkanImageIdentity * vk_identity) .pName = "main" }; - VkPipelineShaderStageCreateInfo frag_stage = { + render->shader_create_info[1] = (VkPipelineShaderStageCreateInfo) { .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, .pNext = NULL, .stage = VK_SHADER_STAGE_FRAGMENT_BIT, .module = frag_module, .pName = "main" }; +} - VkPipelineShaderStageCreateInfo shader_stages[] = {vert_stage, frag_stage}; +static VkDescriptorSetLayoutBinding * +gst_vulkan_image_identity_descriptor_set_layout_bindings (GstVulkanFullScreenRender * render, guint * n_bindings) +{ + VkDescriptorSetLayoutBinding *bindings; - VkVertexInputBindingDescription vertex_binding_description = { + *n_bindings = 1; + bindings = g_new0 (VkDescriptorSetLayoutBinding, *n_bindings); + + /* *INDENT-OFF* */ + bindings[0] = (VkDescriptorSetLayoutBinding) { .binding = 0, - .stride = sizeof (struct Vertex), - .inputRate = VK_VERTEX_INPUT_RATE_VERTEX - }; - - VkVertexInputAttributeDescription vertex_attribute_description[] = { - { - .binding = 0, - .location = 0, - .format = VK_FORMAT_R32G32B32_SFLOAT, - .offset = G_STRUCT_OFFSET (struct Vertex, x) - }, { - .binding = 0, - .location = 1, - .format = VK_FORMAT_R32G32_SFLOAT, - .offset = G_STRUCT_OFFSET (struct Vertex, s) - } - }; - - VkPipelineVertexInputStateCreateInfo vertex_input_info = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, - .pNext = NULL, - .vertexBindingDescriptionCount = 1, - .pVertexBindingDescriptions = &vertex_binding_description, - .vertexAttributeDescriptionCount = G_N_ELEMENTS (vertex_attribute_description), - .pVertexAttributeDescriptions = vertex_attribute_description - }; - - VkPipelineInputAssemblyStateCreateInfo input_assembly = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, - .pNext = NULL, - .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, - .primitiveRestartEnable = VK_FALSE - }; - - VkViewport viewport = { - .x = 0.0f, - .y = 0.0f, - .width = (float) GST_VIDEO_INFO_WIDTH (&vk_identity->v_info), - .height = (float) GST_VIDEO_INFO_HEIGHT (&vk_identity->v_info), - .minDepth = 0.0f, - .maxDepth = 1.0f - }; - - VkRect2D scissor = { - .offset = { 0, 0 }, - .extent = { - GST_VIDEO_INFO_WIDTH (&vk_identity->v_info), - GST_VIDEO_INFO_HEIGHT (&vk_identity->v_info) - } - }; - - VkPipelineViewportStateCreateInfo viewport_state = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, - .pNext = NULL, - .viewportCount = 1, - .pViewports = &viewport, - .scissorCount = 1, - .pScissors = &scissor - }; - - VkPipelineRasterizationStateCreateInfo rasterizer = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pNext = NULL, - .depthClampEnable = VK_FALSE, - .rasterizerDiscardEnable = VK_FALSE, - .polygonMode = VK_POLYGON_MODE_FILL, - .lineWidth = 1.0f, - .cullMode = VK_CULL_MODE_BACK_BIT, - .frontFace = VK_FRONT_FACE_CLOCKWISE, - .depthBiasEnable = VK_FALSE - }; - - VkPipelineMultisampleStateCreateInfo multisampling = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pNext = NULL, - .sampleShadingEnable = VK_FALSE, - .rasterizationSamples = VK_SAMPLE_COUNT_1_BIT - }; - - VkPipelineColorBlendAttachmentState color_blend_attachment = { - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - .blendEnable = VK_FALSE - }; - - VkPipelineColorBlendStateCreateInfo color_blending = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, - .pNext = NULL, - .logicOpEnable = VK_FALSE, - .logicOp = VK_LOGIC_OP_COPY, - .attachmentCount = 1, - .pAttachments = &color_blend_attachment, - .blendConstants = { 0.0f, 0.0f, 0.0f, 0.0f } - }; - - VkGraphicsPipelineCreateInfo pipeline_info = { - .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = NULL, - .stageCount = 2, - .pStages = shader_stages, - .pVertexInputState = &vertex_input_info, - .pInputAssemblyState = &input_assembly, - .pViewportState = &viewport_state, - .pRasterizationState = &rasterizer, - .pMultisampleState = &multisampling, - .pColorBlendState = &color_blending, - .layout = vk_identity->pipeline_layout, - .renderPass = vk_identity->render_pass, - .subpass = 0, - .basePipelineHandle = VK_NULL_HANDLE + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImmutableSamplers = NULL, + .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT }; /* *INDENT-ON* */ - VkResult err; - GError *error = NULL; - VkPipeline pipeline; - err = - vkCreateGraphicsPipelines (vk_identity->device->device, VK_NULL_HANDLE, 1, - &pipeline_info, NULL, &pipeline); - if (gst_vulkan_error_to_g_error (err, &error, - "vkCreateGraphicsPipelines") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create pipeline layout: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - vkDestroyShaderModule (vk_identity->device->device, frag_module, NULL); - vkDestroyShaderModule (vk_identity->device->device, vert_module, NULL); - - return pipeline; + return bindings; } -static VkPipelineLayout -_create_pipeline_layout (GstVulkanImageIdentity * vk_identity) +static VkAttachmentReference + * gst_vulkan_image_identity_render_pass_attachment_references + (GstVulkanFullScreenRender * render, guint * n_attachments) { + VkAttachmentReference *attachments; + + *n_attachments = 1; + attachments = g_new0 (VkAttachmentReference, *n_attachments); /* *INDENT-OFF* */ - VkPipelineLayoutCreateInfo pipeline_layout_info = { - .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, - .pNext = NULL, - .setLayoutCount = 1, - .pSetLayouts = &vk_identity->descriptor_set_layout, - .pushConstantRangeCount = 0 + attachments[0] = (VkAttachmentReference) { + .attachment = 0, + .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL }; /* *INDENT-ON* */ - VkResult err; - GError *error = NULL; - VkPipelineLayout pipeline_layout; - err = - vkCreatePipelineLayout (vk_identity->device->device, - &pipeline_layout_info, NULL, &pipeline_layout); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreatePipelineLayout") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create pipeline layout: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return pipeline_layout; + return attachments; } -static gboolean -gst_vulkan_image_identity_set_caps (GstBaseTransform * bt, GstCaps * in_caps, - GstCaps * out_caps) +static VkAttachmentDescription + * gst_vulkan_image_identity_render_pass_attachment_descriptions + (GstVulkanFullScreenRender * render, guint * n_descriptions) { - GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); + VkAttachmentDescription *color_attachments; - if (!gst_video_info_from_caps (&vk_identity->v_info, in_caps)) - return FALSE; - - if (vk_identity->graphics_pipeline) { - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_pipeline (vk_identity->last_fence, - vk_identity->graphics_pipeline)); - vk_identity->graphics_pipeline = NULL; - } - if (!(vk_identity->graphics_pipeline = _create_pipeline (vk_identity))) - return FALSE; - - gst_caps_replace (&vk_identity->caps, in_caps); - - GST_DEBUG_OBJECT (bt, "set caps: %" GST_PTR_FORMAT, in_caps); - - return TRUE; -} - -static gboolean -gst_vulkan_image_identity_propose_allocation (GstBaseTransform * bt, - GstQuery * decide_query, GstQuery * query) -{ - /* FIXME: */ - return FALSE; -} - -static gboolean -gst_vulkan_image_identity_decide_allocation (GstBaseTransform * bt, - GstQuery * query) -{ - GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - GstBufferPool *pool = NULL; - GstStructure *config; - GstCaps *caps; - guint min, max, size; - gboolean update_pool; - - gst_query_parse_allocation (query, &caps, NULL); - if (!caps) - return FALSE; - - if (gst_query_get_n_allocation_pools (query) > 0) { - gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); - - update_pool = TRUE; - } else { - GstVideoInfo vinfo; - - gst_video_info_init (&vinfo); - gst_video_info_from_caps (&vinfo, caps); - size = vinfo.size; - min = max = 0; - update_pool = FALSE; - } - - if (!pool || !GST_IS_VULKAN_IMAGE_BUFFER_POOL (pool)) { - if (pool) - gst_object_unref (pool); - pool = gst_vulkan_image_buffer_pool_new (vk_identity->device); - } - - config = gst_buffer_pool_get_config (pool); - - gst_buffer_pool_config_set_params (config, caps, size, min, max); - gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - - gst_buffer_pool_set_config (pool, config); - - if (update_pool) - gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); - else - gst_query_add_allocation_pool (query, pool, size, min, max); - - gst_object_unref (pool); - - return TRUE; -} - -static VkRenderPass -_create_render_pass (GstVulkanImageIdentity * vk_identity) -{ + *n_descriptions = 1; + color_attachments = g_new0 (VkAttachmentDescription, *n_descriptions); /* *INDENT-OFF* */ - VkAttachmentDescription color_attachment = { - /* FIXME: */ - .format = VK_FORMAT_B8G8R8A8_UNORM, + color_attachments[0] = (VkAttachmentDescription) { + .format = gst_vulkan_format_from_video_format (GST_VIDEO_INFO_FORMAT (&render->in_info), 0), .samples = VK_SAMPLE_COUNT_1_BIT, .loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR, .storeOp = VK_ATTACHMENT_STORE_OP_STORE, @@ -588,81 +267,85 @@ _create_render_pass (GstVulkanImageIdentity * vk_identity) .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, .finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL }; - - VkAttachmentReference color_attachment_ref = { - .attachment = 0, - .layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL - }; - - VkSubpassDescription subpass = { - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .colorAttachmentCount = 1, - .pColorAttachments = &color_attachment_ref - }; - - VkRenderPassCreateInfo render_pass_info = { - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, - .pNext = NULL, - .attachmentCount = 1, - .pAttachments = &color_attachment, - .subpassCount = 1, - .pSubpasses = &subpass - }; /* *INDENT-ON* */ - VkRenderPass render_pass = NULL; - VkResult err; - GError *error = NULL; - err = - vkCreateRenderPass (vk_identity->device->device, &render_pass_info, NULL, - &render_pass); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateRenderPass") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create renderpass: %s", - error->message); - return NULL; - } - - return render_pass; + return color_attachments; } -static VkDescriptorSetLayout -_create_descriptor_set_layout (GstVulkanImageIdentity * vk_identity) +static VkDescriptorPool +_create_descriptor_pool (GstVulkanImageIdentity * vk_identity) { + GstVulkanFullScreenRender *render = + GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); + /* *INDENT-OFF* */ - VkDescriptorSetLayoutBinding sampler_layout_binding = { - .binding = 0, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .pImmutableSamplers = NULL, - .stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT + VkDescriptorPoolSize pool_sizes = { + .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .descriptorCount = 1 }; - VkDescriptorSetLayoutCreateInfo layout_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, + + VkDescriptorPoolCreateInfo pool_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, .pNext = NULL, - .bindingCount = 1, - .pBindings = &sampler_layout_binding + .poolSizeCount = 1, + .pPoolSizes = &pool_sizes, + .maxSets = 1 }; /* *INDENT-ON* */ - VkDescriptorSetLayout descriptor_set_layout; - VkResult err; + VkDescriptorPool pool; GError *error = NULL; + VkResult err; err = - vkCreateDescriptorSetLayout (vk_identity->device->device, &layout_info, - NULL, &descriptor_set_layout); - if (gst_vulkan_error_to_g_error (err, &error, - "vkCreateDescriptorSetLayout") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create renderpass: %s", + vkCreateDescriptorPool (render->device->device, &pool_info, NULL, &pool); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { + GST_ERROR_OBJECT (render, "Failed to create descriptor pool: %s", error->message); + g_clear_error (&error); return NULL; } - return descriptor_set_layout; + return pool; +} + +static VkDescriptorSet +_create_descriptor_set (GstVulkanImageIdentity * vk_identity) +{ + GstVulkanFullScreenRender *render = + GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); + + /* *INDENT-OFF* */ + VkDescriptorSetAllocateInfo alloc_info = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, + .pNext = NULL, + .descriptorPool = vk_identity->descriptor_pool, + .descriptorSetCount = 1, + .pSetLayouts = &render->descriptor_set_layout + }; + /* *INDENT-ON* */ + VkDescriptorSet descriptor; + GError *error = NULL; + VkResult err; + + err = + vkAllocateDescriptorSets (render->device->device, &alloc_info, + &descriptor); + if (gst_vulkan_error_to_g_error (err, &error, "vkAllocateDescriptorSets") < 0) { + GST_ERROR_OBJECT (vk_identity, "Failed to allocate descriptor: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return descriptor; } static VkSampler _create_sampler (GstVulkanImageIdentity * vk_identity) { + GstVulkanFullScreenRender *render = + GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); + /* *INDENT-OFF* */ VkSamplerCreateInfo samplerInfo = { .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, @@ -687,9 +370,7 @@ _create_sampler (GstVulkanImageIdentity * vk_identity) VkSampler sampler; VkResult err; - err = - vkCreateSampler (vk_identity->device->device, &samplerInfo, NULL, - &sampler); + err = vkCreateSampler (render->device->device, &samplerInfo, NULL, &sampler); if (gst_vulkan_error_to_g_error (err, &error, "vkCreateSampler") < 0) { GST_ERROR_OBJECT (vk_identity, "Failed to create sampler: %s", error->message); @@ -700,193 +381,20 @@ _create_sampler (GstVulkanImageIdentity * vk_identity) return sampler; } -static gboolean -_create_vertex_buffers (GstVulkanImageIdentity * vk_identity) -{ - GstMapInfo map_info; - - vk_identity->vertices = - gst_vulkan_buffer_memory_alloc (vk_identity->device, VK_FORMAT_R8_UNORM, - sizeof (vertices), - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - if (!gst_memory_map (vk_identity->vertices, &map_info, GST_MAP_WRITE)) { - gst_memory_unref (vk_identity->vertices); - vk_identity->vertices = NULL; - return FALSE; - } - memcpy (map_info.data, vertices, sizeof (vertices)); - gst_memory_unmap (vk_identity->vertices, &map_info); - - vk_identity->indices = - gst_vulkan_buffer_memory_alloc (vk_identity->device, VK_FORMAT_R8_UNORM, - sizeof (indices), - VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - - if (!gst_memory_map (vk_identity->indices, &map_info, GST_MAP_WRITE)) { - gst_memory_unref (vk_identity->vertices); - vk_identity->vertices = NULL; - gst_memory_unref (vk_identity->indices); - vk_identity->indices = NULL; - return FALSE; - } - memcpy (map_info.data, indices, sizeof (indices)); - gst_memory_unmap (vk_identity->indices, &map_info); - - return TRUE; -} - -static VkFramebuffer -_create_framebuffer (GstVulkanImageIdentity * vk_identity, VkImageView view) -{ - /* *INDENT-OFF* */ - VkImageView attachments[] = { - view, - }; - VkFramebufferCreateInfo framebuffer_info = { - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = NULL, - .renderPass = vk_identity->render_pass, - .attachmentCount = 1, - .pAttachments = attachments, - .width = GST_VIDEO_INFO_WIDTH (&vk_identity->v_info), - .height = GST_VIDEO_INFO_HEIGHT (&vk_identity->v_info), - .layers = 1 - }; - /* *INDENT-ON* */ - VkFramebuffer framebuffer; - GError *error = NULL; - VkResult err; - - err = - vkCreateFramebuffer (vk_identity->device->device, &framebuffer_info, NULL, - &framebuffer); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create framebuffer: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return framebuffer; -} - -static VkDescriptorPool -_create_descriptor_pool (GstVulkanImageIdentity * vk_identity) -{ - /* *INDENT-OFF* */ - VkDescriptorPoolSize pool_sizes = { - .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 1 - }; - - VkDescriptorPoolCreateInfo pool_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO, - .pNext = NULL, - .poolSizeCount = 1, - .pPoolSizes = &pool_sizes, - .maxSets = 1 - }; - /* *INDENT-ON* */ - VkDescriptorPool pool; - GError *error = NULL; - VkResult err; - - err = - vkCreateDescriptorPool (vk_identity->device->device, &pool_info, NULL, - &pool); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateDescriptorPool") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to create descriptor pool: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return pool; -} - -static VkDescriptorSet -_create_descriptor_set (GstVulkanImageIdentity * vk_identity) -{ - /* *INDENT-OFF* */ - VkDescriptorSetAllocateInfo alloc_info = { - .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO, - .pNext = NULL, - .descriptorPool = vk_identity->descriptor_pool, - .descriptorSetCount = 1, - .pSetLayouts = &vk_identity->descriptor_set_layout - }; - /* *INDENT-ON* */ - VkDescriptorSet descriptor; - GError *error = NULL; - VkResult err; - - err = - vkAllocateDescriptorSets (vk_identity->device->device, &alloc_info, - &descriptor); - if (gst_vulkan_error_to_g_error (err, &error, "vkAllocateDescriptorSets") < 0) { - GST_ERROR_OBJECT (vk_identity, "Failed to allocate descriptor: %s", - error->message); - g_clear_error (&error); - return NULL; - } - - return descriptor; -} - static gboolean gst_vulkan_image_identity_start (GstBaseTransform * bt) { GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); - if (!gst_vulkan_ensure_element_data (GST_ELEMENT (bt), NULL, - &vk_identity->instance)) { - GST_ELEMENT_ERROR (vk_identity, RESOURCE, NOT_FOUND, - ("Failed to retreive vulkan instance"), (NULL)); - return FALSE; - } - if (!gst_vulkan_device_run_context_query (GST_ELEMENT (vk_identity), - &vk_identity->device)) { - GError *error = NULL; - GST_DEBUG_OBJECT (vk_identity, "No device retrieved from peer elements"); - if (!(vk_identity->device = - gst_vulkan_instance_create_device (vk_identity->instance, - &error))) { - GST_ELEMENT_ERROR (vk_identity, RESOURCE, NOT_FOUND, - ("Failed to create vulkan device"), ("%s", error->message)); - g_clear_error (&error); - return FALSE; - } - } - - if (!gst_vulkan_queue_run_context_query (GST_ELEMENT (vk_identity), - &vk_identity->queue)) { - GST_DEBUG_OBJECT (vk_identity, "No queue retrieved from peer elements"); - vk_identity->queue = _find_graphics_queue (vk_identity); - } - if (!vk_identity->queue) + if (!GST_BASE_TRANSFORM_CLASS (parent_class)->start (bt)) return FALSE; - if (!(vk_identity->render_pass = _create_render_pass (vk_identity))) - return FALSE; if (!(vk_identity->sampler = _create_sampler (vk_identity))) return FALSE; - if (!(vk_identity->descriptor_set_layout = - _create_descriptor_set_layout (vk_identity))) - return FALSE; if (!(vk_identity->descriptor_pool = _create_descriptor_pool (vk_identity))) return FALSE; if (!(vk_identity->descriptor_set = _create_descriptor_set (vk_identity))) return FALSE; - if (!(vk_identity->pipeline_layout = _create_pipeline_layout (vk_identity))) - return FALSE; - - if (!_create_vertex_buffers (vk_identity)) - return FALSE; return TRUE; } @@ -895,101 +403,42 @@ static gboolean gst_vulkan_image_identity_stop (GstBaseTransform * bt) { GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); - if (vk_identity->device) { - if (vk_identity->last_fence) { - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_pipeline (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->graphics_pipeline)); - vk_identity->graphics_pipeline = NULL; - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_pipeline_layout (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->pipeline_layout)); - vk_identity->pipeline_layout = NULL; - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_render_pass (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->render_pass)); - vk_identity->render_pass = NULL; - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, + if (render->device) { + if (render->last_fence) { + render->trash_list = g_list_prepend (render->trash_list, gst_vulkan_trash_new_free_descriptor_pool (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->descriptor_pool)); - vk_identity->descriptor_pool = NULL; - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_descriptor_set_layout (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->descriptor_set_layout)); - vk_identity->descriptor_set_layout = NULL; - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_sampler (gst_vulkan_fence_ref - (vk_identity->last_fence), vk_identity->sampler)); - vk_identity->sampler = NULL; - - gst_vulkan_fence_unref (vk_identity->last_fence); - vk_identity->last_fence = NULL; - } else { - vkDestroyPipeline (vk_identity->device->device, - vk_identity->graphics_pipeline, NULL); - vk_identity->graphics_pipeline = NULL; - - vkDestroyPipelineLayout (vk_identity->device->device, - vk_identity->pipeline_layout, NULL); - vk_identity->pipeline_layout = NULL; - - vkDestroyRenderPass (vk_identity->device->device, - vk_identity->render_pass, NULL); - vk_identity->render_pass = NULL; - - vkFreeDescriptorSets (vk_identity->device->device, - vk_identity->descriptor_pool, 1, &vk_identity->descriptor_set); + (render->last_fence), vk_identity->descriptor_pool)); vk_identity->descriptor_set = NULL; - - vkDestroyDescriptorPool (vk_identity->device->device, - vk_identity->descriptor_pool, NULL); vk_identity->descriptor_pool = NULL; - - vkDestroyDescriptorSetLayout (vk_identity->device->device, - vk_identity->descriptor_set_layout, NULL); - vk_identity->descriptor_set_layout = NULL; - - vkDestroySampler (vk_identity->device->device, vk_identity->sampler, - NULL); + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_sampler (gst_vulkan_fence_ref + (render->last_fence), vk_identity->sampler)); + vk_identity->sampler = NULL; + } else { + vkDestroyDescriptorPool (render->device->device, + vk_identity->descriptor_pool, NULL); + vk_identity->descriptor_set = NULL; + vk_identity->descriptor_pool = NULL; + vkDestroySampler (render->device->device, vk_identity->sampler, NULL); vk_identity->sampler = NULL; } - - if (!gst_vulkan_trash_list_wait (vk_identity->trash_list, -1)) - GST_WARNING_OBJECT (vk_identity, - "Failed to wait for all resources to be freed"); - vk_identity->trash_list = NULL; - - if (vk_identity->vertices) - gst_memory_unref (vk_identity->vertices); - vk_identity->vertices = NULL; - - if (vk_identity->indices) - gst_memory_unref (vk_identity->indices); - vk_identity->indices = NULL; - - gst_object_unref (vk_identity->device); } - vk_identity->device = NULL; if (vk_identity->cmd_pool) gst_object_unref (vk_identity->cmd_pool); vk_identity->cmd_pool = NULL; - if (vk_identity->queue) - gst_object_unref (vk_identity->queue); - vk_identity->queue = NULL; - - if (vk_identity->instance) - gst_object_unref (vk_identity->instance); - vk_identity->instance = NULL; - - return TRUE; + return GST_BASE_TRANSFORM_CLASS (parent_class)->stop (bt); } static void -_update_descriptor (GstVulkanImageIdentity * vk_identity, VkImageView view) +update_descriptor_set (GstVulkanImageIdentity * vk_identity, VkImageView view) { + GstVulkanFullScreenRender *render = + GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); + /* *INDENT-OFF* */ VkDescriptorImageInfo image_info = { .imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, @@ -1008,14 +457,52 @@ _update_descriptor (GstVulkanImageIdentity * vk_identity, VkImageView view) .pImageInfo = &image_info }; /* *INDENT-ON* */ + vkUpdateDescriptorSets (render->device->device, 1, &writes, 0, NULL); +} - vkUpdateDescriptorSets (vk_identity->device->device, 1, &writes, 0, NULL); +static VkFramebuffer +_create_framebuffer (GstVulkanImageIdentity * vk_identity, VkImageView view) +{ + GstVulkanFullScreenRender *render = + GST_VULKAN_FULL_SCREEN_RENDER (vk_identity); + + /* *INDENT-OFF* */ + VkImageView attachments[] = { + view, + }; + VkFramebufferCreateInfo framebuffer_info = { + .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, + .pNext = NULL, + .renderPass = render->render_pass, + .attachmentCount = 1, + .pAttachments = attachments, + .width = GST_VIDEO_INFO_WIDTH (&render->in_info), + .height = GST_VIDEO_INFO_HEIGHT (&render->in_info), + .layers = 1 + }; + /* *INDENT-ON* */ + VkFramebuffer framebuffer; + GError *error = NULL; + VkResult err; + + err = + vkCreateFramebuffer (render->device->device, &framebuffer_info, NULL, + &framebuffer); + if (gst_vulkan_error_to_g_error (err, &error, "vkCreateFramebuffer") < 0) { + GST_ERROR_OBJECT (render, "Failed to create framebuffer: %s", + error->message); + g_clear_error (&error); + return NULL; + } + + return framebuffer; } static GstFlowReturn gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer * outbuf) { + GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); GstVulkanImageMemory *in_img_mem, *out_img_mem; GstVulkanFence *fence = NULL; @@ -1043,9 +530,9 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, if (!vk_identity->cmd_pool) { if (!(vk_identity->cmd_pool = - gst_vulkan_queue_create_command_pool (vk_identity->queue, &error))) + gst_vulkan_queue_create_command_pool (render->queue, &error))) goto error; - _update_descriptor (vk_identity, in_img_mem->view); + update_descriptor_set (vk_identity, in_img_mem->view); } if (!(cmd = gst_vulkan_command_pool_create (vk_identity->cmd_pool, &error))) @@ -1057,6 +544,10 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } + fence = gst_vulkan_fence_new (render->device, 0, &error); + if (!fence) + goto error; + { VkCommandBufferBeginInfo cmd_buf_info = { 0, }; @@ -1076,20 +567,6 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, { /* *INDENT-OFF* */ - VkClearValue clearColor = {{{ 0.0f, 1.0f, 0.0f, 1.0f }}}; - VkRenderPassBeginInfo render_pass_info = { - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, - .renderPass = vk_identity->render_pass, - .framebuffer = framebuffer, - .renderArea.offset = { 0, 0 }, - .renderArea.extent = { - GST_VIDEO_INFO_WIDTH (&vk_identity->v_info), - GST_VIDEO_INFO_HEIGHT (&vk_identity->v_info) - }, - .clearValueCount = 1, - .pClearValues = &clearColor - }; - VkImageMemoryBarrier in_image_memory_barrier = { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = NULL, @@ -1118,7 +595,6 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, .subresourceRange = out_img_mem->barrier.subresource_range }; /* *INDENT-ON* */ - VkDeviceSize offsets[] = { 0 }; vkCmdPipelineBarrier (cmd, in_img_mem->barrier.parent.pipeline_stages, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 0, 0, NULL, 0, NULL, 1, @@ -1139,69 +615,28 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, out_img_mem->barrier.parent.access_flags = out_image_memory_barrier.dstAccessMask; out_img_mem->barrier.image_layout = out_image_memory_barrier.newLayout; - - vkCmdBeginRenderPass (cmd, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); - vkCmdBindPipeline (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - vk_identity->graphics_pipeline); - vkCmdBindVertexBuffers (cmd, 0, 1, - &((GstVulkanBufferMemory *) vk_identity->vertices)->buffer, offsets); - vkCmdBindIndexBuffer (cmd, - ((GstVulkanBufferMemory *) vk_identity->indices)->buffer, 0, - VK_INDEX_TYPE_UINT16); - vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, - vk_identity->pipeline_layout, 0, 1, &vk_identity->descriptor_set, 0, - NULL); - vkCmdDrawIndexed (cmd, G_N_ELEMENTS (indices), 1, 0, 0, 0); - vkCmdEndRenderPass (cmd); } + vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, + render->pipeline_layout, 0, 1, &vk_identity->descriptor_set, 0, NULL); + if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd, + framebuffer)) + return GST_FLOW_ERROR; + err = vkEndCommandBuffer (cmd); if (gst_vulkan_error_to_g_error (err, &error, "vkEndCommandBuffer") < 0) goto error; - fence = gst_vulkan_fence_new (vk_identity->device, 0, &error); - if (!fence) - goto error; - - { - VkSubmitInfo submit_info = { 0, }; - - /* *INDENT-OFF* */ - submit_info = (VkSubmitInfo) { - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .pNext = NULL, - .waitSemaphoreCount = 0, - .pWaitSemaphores = NULL, - .pWaitDstStageMask = NULL, - .commandBufferCount = 1, - .pCommandBuffers = &cmd, - .signalSemaphoreCount = 0, - .pSignalSemaphores = NULL, - }; - /* *INDENT-ON* */ - - if (vk_identity->last_fence) - gst_vulkan_fence_unref (vk_identity->last_fence); - vk_identity->last_fence = gst_vulkan_fence_ref (fence); - - err = - vkQueueSubmit (vk_identity->queue->queue, 1, &submit_info, - GST_VULKAN_FENCE_FENCE (fence)); - if (gst_vulkan_error_to_g_error (err, &error, "vkQueueSubmit") < 0) - goto error; - } - - vk_identity->trash_list = g_list_prepend (vk_identity->trash_list, - gst_vulkan_trash_new_free_command_buffer (gst_vulkan_fence_ref (fence), - vk_identity->cmd_pool, cmd)); - vk_identity->trash_list = - g_list_prepend (vk_identity->trash_list, + render->trash_list = + g_list_prepend (render->trash_list, gst_vulkan_trash_new_free_framebuffer (gst_vulkan_fence_ref (fence), framebuffer)); + render->trash_list = g_list_prepend (render->trash_list, + gst_vulkan_trash_new_free_command_buffer (gst_vulkan_fence_ref (fence), + vk_identity->cmd_pool, cmd)); - vk_identity->trash_list = gst_vulkan_trash_list_gc (vk_identity->trash_list); - - gst_vulkan_fence_unref (fence); + if (!gst_vulkan_full_screen_render_submit (render, cmd, fence)) + return GST_FLOW_ERROR; return GST_FLOW_OK; diff --git a/ext/vulkan/vkimageidentity.h b/ext/vulkan/vkimageidentity.h index 740622b80e..2856ae4808 100644 --- a/ext/vulkan/vkimageidentity.h +++ b/ext/vulkan/vkimageidentity.h @@ -24,6 +24,7 @@ #include #include #include +#include "vkfullscreenrender.h" G_BEGIN_DECLS @@ -38,36 +39,21 @@ typedef struct _GstVulkanImageIdentityClass GstVulkanImageIdentityClass; struct _GstVulkanImageIdentity { - GstBaseTransform parent; + GstVulkanFullScreenRender parent; - GstVulkanInstance *instance; - GstVulkanDevice *device; - GstVulkanQueue *queue; - GstVulkanCommandPool *cmd_pool; + GstVulkanCommandPool *cmd_pool; - GstCaps *caps; - GstVideoInfo v_info; + VkSampler sampler; + VkDescriptorPool descriptor_pool; + VkDescriptorSet descriptor_set; - VkRenderPass render_pass; - VkPipelineLayout pipeline_layout; - VkPipeline graphics_pipeline; - VkSampler sampler; - VkDescriptorSetLayout descriptor_set_layout; - VkDescriptorPool descriptor_pool; - VkDescriptorSet descriptor_set; - - GstMemory *vertices; - GstMemory *indices; - - GList *trash_list; - GstVulkanFence *last_fence; - - GArray *frames; + VkDescriptorSetLayoutBinding sampler_layout_binding; + VkDescriptorSetLayoutCreateInfo layout_info; }; struct _GstVulkanImageIdentityClass { - GstBaseTransformClass video_sink_class; + GstVulkanFullScreenRenderClass parent_class; }; GType gst_vulkan_image_identity_get_type(void);