From 34ff895040df199af9ae66f66d256d4a1e761270 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 24 Sep 2019 17:24:38 +1000 Subject: [PATCH] vulkan: remove VkImageView from the memory There can be multiple views per image for different subresource ranges or planes in multi-planer images. --- ext/vulkan/meson.build | 1 + ext/vulkan/vkcolorconvert.c | 52 +++++---- ext/vulkan/vkcolorconvert.h | 2 +- ext/vulkan/vkelementutils.c | 98 ++++++++++++++++ ext/vulkan/vkelementutils.h | 33 ++++++ ext/vulkan/vkimageidentity.c | 15 ++- ext/vulkan/vkviewconvert.c | 46 +++++--- gst-libs/gst/vulkan/gstvkimagememory.c | 156 +++++++++++++++++-------- gst-libs/gst/vulkan/gstvkimagememory.h | 19 ++- gst-libs/gst/vulkan/gstvkimageview.c | 125 ++++++++++++++++++++ gst-libs/gst/vulkan/gstvkimageview.h | 83 +++++++++++++ gst-libs/gst/vulkan/meson.build | 2 + gst-libs/gst/vulkan/vulkan.h | 1 + gst-libs/gst/vulkan/vulkan_fwd.h | 2 + 14 files changed, 543 insertions(+), 92 deletions(-) create mode 100644 ext/vulkan/vkelementutils.c create mode 100644 ext/vulkan/vkelementutils.h create mode 100644 gst-libs/gst/vulkan/gstvkimageview.c create mode 100644 gst-libs/gst/vulkan/gstvkimageview.h diff --git a/ext/vulkan/meson.build b/ext/vulkan/meson.build index 8659cc2dcc..63e130396b 100644 --- a/ext/vulkan/meson.build +++ b/ext/vulkan/meson.build @@ -13,6 +13,7 @@ vulkan_sources = [ 'vkcolorconvert.c', 'vkdownload.c', 'vkdeviceprovider.c', + 'vkelementutils.c', 'vkfullscreenrender.c', 'vkimageidentity.c', 'vksink.c', diff --git a/ext/vulkan/vkcolorconvert.c b/ext/vulkan/vkcolorconvert.c index b03fc698c7..a6777e632a 100644 --- a/ext/vulkan/vkcolorconvert.c +++ b/ext/vulkan/vkcolorconvert.c @@ -33,6 +33,8 @@ #include "vkcolorconvert.h" #include "vkshader.h" +#include "vkelementutils.h" + #include "shaders/identity.vert.h" #include "shaders/swizzle.frag.h" #include "shaders/swizzle_and_clobber_alpha.frag.h" @@ -649,9 +651,9 @@ get_vulkan_format_swizzle_order (GstVideoFormat v_format, static void calculate_reorder_indexes (GstVideoFormat in_format, - GstVulkanImageMemory * in_mems[GST_VIDEO_MAX_COMPONENTS], + GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS], GstVideoFormat out_format, - GstVulkanImageMemory * out_mems[GST_VIDEO_MAX_COMPONENTS], + GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS], int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS]) { const GstVideoFormatInfo *in_finfo, *out_finfo; @@ -668,9 +670,9 @@ calculate_reorder_indexes (GstVideoFormat in_format, out_finfo = gst_video_format_get_info (out_format); for (i = 0; i < in_finfo->n_planes; i++) - in_vk_formats[i] = in_mems[i]->create_info.format; + in_vk_formats[i] = in_views[i]->image->create_info.format; for (i = 0; i < out_finfo->n_planes; i++) - out_vk_formats[i] = out_mems[i]->create_info.format; + out_vk_formats[i] = out_views[i]->image->create_info.format; get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order); video_format_to_reorder (in_format, in_reorder, TRUE); @@ -701,19 +703,19 @@ calculate_reorder_indexes (GstVideoFormat in_format, static gboolean swizzle_rgb_update_command_state (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, - GstVulkanImageMemory ** in_mems, GstVulkanImageMemory ** out_mems) + GstVulkanImageView ** in_views, GstVulkanImageView ** out_views) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); gint reorder[8]; calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), - out_mems, reorder, &reorder[4]); + in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_views, reorder, &reorder[4]); vkCmdPushConstants (cmd, render->pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof (reorder), (const void *) reorder); - update_descriptor_set (conv, &in_mems[0]->view, 1); + update_descriptor_set (conv, &in_views[0]->view, 1); vkCmdBindDescriptorSets (cmd, VK_PIPELINE_BIND_POINT_GRAPHICS, render->pipeline_layout, 0, 1, &conv->descriptor_set, 0, NULL); @@ -739,8 +741,8 @@ struct YUVUpdateData static gboolean yuv_to_rgb_update_command_state (GstVulkanColorConvert * conv, - VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageMemory ** in_mems, - GstVulkanImageMemory ** out_mems) + VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); @@ -752,8 +754,8 @@ yuv_to_rgb_update_command_state (GstVulkanColorConvert * conv, int i; calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), - out_mems, data.in_reorder, data.out_reorder); + in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_views, data.in_reorder, data.out_reorder); conv_info = convert_info_new (&render->in_info, &render->out_info); matrix_to_float (&conv_info->to_RGB_matrix, data.matrices.to_RGB); @@ -772,7 +774,7 @@ yuv_to_rgb_update_command_state (GstVulkanColorConvert * conv, gst_memory_unmap (conv->uniform, &map_info); for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) - views[i] = in_mems[i]->view; + views[i] = in_views[i]->view; update_descriptor_set (conv, views, GST_VIDEO_INFO_N_PLANES (&render->in_info)); @@ -1657,8 +1659,10 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanColorConvert *conv = GST_VULKAN_COLOR_CONVERT (bt); GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanImageMemory *render_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; - GstVulkanImageMemory *out_img_mems[4] = { NULL, }; + GstVulkanImageView *render_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageMemory *out_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanFence *fence = NULL; VkFramebuffer framebuffer; GstVulkanCommandBuffer *cmd_buf; @@ -1666,6 +1670,10 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, VkResult err; int i; + fence = gst_vulkan_fence_new (render->device, 0, &error); + if (!fence) + goto error; + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { GstMemory *mem = gst_buffer_peek_memory (inbuf, i); if (!gst_is_vulkan_image_memory (mem)) { @@ -1674,6 +1682,10 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } in_img_mems[i] = (GstVulkanImageMemory *) mem; + in_img_views[i] = get_or_create_image_view (in_img_mems[i]); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + (GstMiniObject *) in_img_views[i])); } for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { @@ -1695,10 +1707,6 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, if (!(cmd_buf = gst_vulkan_command_pool_create (conv->cmd_pool, &error))) goto error; - fence = gst_vulkan_fence_new (render->device, 0, &error); - if (!fence) - goto error; - { VkCommandBufferBeginInfo cmd_buf_info = { 0, }; @@ -1803,7 +1811,11 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, { VkImageView attachments[4] = { 0, }; for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - attachments[i] = render_img_mems[i]->view; + render_img_views[i] = get_or_create_image_view (render_img_mems[i]); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + (GstMiniObject *) render_img_views[i])); + attachments[i] = render_img_views[i]->view; } if (!(framebuffer = _create_framebuffer (conv, @@ -1815,7 +1827,7 @@ gst_vulkan_color_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, } conv->current_shader->cmd_state_update (conv, cmd_buf->cmd, - conv->current_shader, in_img_mems, render_img_mems); + conv->current_shader, in_img_views, render_img_views); if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd, framebuffer)) { g_set_error (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, diff --git a/ext/vulkan/vkcolorconvert.h b/ext/vulkan/vkcolorconvert.h index 3e9e1787bd..b39e01bbfd 100644 --- a/ext/vulkan/vkcolorconvert.h +++ b/ext/vulkan/vkcolorconvert.h @@ -41,7 +41,7 @@ typedef struct _GstVulkanColorConvertClass GstVulkanColorConvertClass; typedef struct _shader_info shader_info; -typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageMemory ** src_mems, GstVulkanImageMemory ** dest_mems); +typedef gboolean (*CommandStateUpdate) (GstVulkanColorConvert * conv, VkCommandBuffer cmd, shader_info * sinfo, GstVulkanImageView ** src_views, GstVulkanImageView ** dest_views); struct _shader_info { diff --git a/ext/vulkan/vkelementutils.c b/ext/vulkan/vkelementutils.c new file mode 100644 index 0000000000..82f50b5ea8 --- /dev/null +++ b/ext/vulkan/vkelementutils.c @@ -0,0 +1,98 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "vkelementutils.h" + +static void +fill_vulkan_image_view_info (VkImage image, VkFormat format, + VkImageViewCreateInfo * info) +{ + /* *INDENT-OFF* */ + *info = (VkImageViewCreateInfo) { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, + .pNext = NULL, + .image = image, + .format = format, + .viewType = VK_IMAGE_VIEW_TYPE_2D, + .flags = 0, + .components = (VkComponentMapping) { + VK_COMPONENT_SWIZZLE_R, + VK_COMPONENT_SWIZZLE_G, + VK_COMPONENT_SWIZZLE_B, + VK_COMPONENT_SWIZZLE_A + }, + .subresourceRange = (VkImageSubresourceRange) { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + } + }; + /* *INDENT-ON* */ +} + +static gboolean +find_compatible_view (GstVulkanImageView * view, VkImageViewCreateInfo * info) +{ + return view->create_info.image == info->image + && view->create_info.format == info->format + && view->create_info.viewType == info->viewType + && view->create_info.flags == info->flags + && view->create_info.components.r == info->components.r + && view->create_info.components.g == info->components.g + && view->create_info.components.b == info->components.b + && view->create_info.components.a == info->components.a + && view->create_info.subresourceRange.aspectMask == + info->subresourceRange.aspectMask + && view->create_info.subresourceRange.baseMipLevel == + info->subresourceRange.baseMipLevel + && view->create_info.subresourceRange.levelCount == + info->subresourceRange.levelCount + && view->create_info.subresourceRange.levelCount == + info->subresourceRange.levelCount + && view->create_info.subresourceRange.baseArrayLayer == + info->subresourceRange.baseArrayLayer + && view->create_info.subresourceRange.layerCount == + info->subresourceRange.layerCount; +} + +GstVulkanImageView * +get_or_create_image_view (GstVulkanImageMemory * image) +{ + VkImageViewCreateInfo create_info; + GstVulkanImageView *ret = NULL; + + fill_vulkan_image_view_info (image->image, image->create_info.format, + &create_info); + + ret = gst_vulkan_image_memory_find_view (image, + (GstVulkanImageMemoryFindViewFunc) find_compatible_view, &create_info); + if (!ret) { + ret = gst_vulkan_image_view_new (image, &create_info); + gst_vulkan_image_memory_add_view (image, ret); + } + + return ret; +} diff --git a/ext/vulkan/vkelementutils.h b/ext/vulkan/vkelementutils.h new file mode 100644 index 0000000000..1fdc4eaebc --- /dev/null +++ b/ext/vulkan/vkelementutils.h @@ -0,0 +1,33 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _VK_ELEMENT_UTILS_H_ +#define _VK_ELEMENT_UTILS_H_ + +#include +#include + +G_BEGIN_DECLS + +GstVulkanImageView * get_or_create_image_view (GstVulkanImageMemory * image); + +G_END_DECLS + +#endif diff --git a/ext/vulkan/vkimageidentity.c b/ext/vulkan/vkimageidentity.c index ccfb42862f..1f81031183 100644 --- a/ext/vulkan/vkimageidentity.c +++ b/ext/vulkan/vkimageidentity.c @@ -33,6 +33,8 @@ #include "vkimageidentity.h" #include "vkshader.h" +#include "vkelementutils.h" + #include "shaders/identity.vert.h" #include "shaders/identity.frag.h" @@ -535,6 +537,7 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanImageIdentity *vk_identity = GST_VULKAN_IMAGE_IDENTITY (bt); GstVulkanImageMemory *in_img_mem, *out_img_mem; + GstVulkanImageView *in_img_view, *out_img_view; GstVulkanFence *fence = NULL; GstMemory *in_mem, *out_mem; VkFramebuffer framebuffer; @@ -549,6 +552,10 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } in_img_mem = (GstVulkanImageMemory *) in_mem; + in_img_view = get_or_create_image_view (in_img_mem); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + GST_MINI_OBJECT_CAST (in_img_view))); out_mem = gst_buffer_peek_memory (outbuf, 0); if (!gst_is_vulkan_image_memory (out_mem)) { @@ -557,19 +564,23 @@ gst_vulkan_image_identity_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } out_img_mem = (GstVulkanImageMemory *) out_mem; + out_img_view = get_or_create_image_view (out_img_mem); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + GST_MINI_OBJECT_CAST (out_img_view))); if (!vk_identity->cmd_pool) { if (!(vk_identity->cmd_pool = gst_vulkan_queue_create_command_pool (render->queue, &error))) goto error; - update_descriptor_set (vk_identity, in_img_mem->view); + update_descriptor_set (vk_identity, in_img_view->view); } if (!(cmd_buf = gst_vulkan_command_pool_create (vk_identity->cmd_pool, &error))) goto error; - if (!(framebuffer = _create_framebuffer (vk_identity, out_img_mem->view))) { + if (!(framebuffer = _create_framebuffer (vk_identity, out_img_view->view))) { g_set_error_literal (&error, GST_VULKAN_ERROR, GST_VULKAN_FAILED, "Failed to create framebuffer"); goto error; diff --git a/ext/vulkan/vkviewconvert.c b/ext/vulkan/vkviewconvert.c index 8711e3f297..51fc0d6bd7 100644 --- a/ext/vulkan/vkviewconvert.c +++ b/ext/vulkan/vkviewconvert.c @@ -33,6 +33,8 @@ #include "vkviewconvert.h" #include "vkshader.h" +#include "vkelementutils.h" + #include "shaders/identity.vert.h" #include "shaders/view_convert.frag.h" #include "gstvulkan-plugins-enumtypes.h" @@ -240,9 +242,9 @@ get_vulkan_format_swizzle_order (GstVideoFormat v_format, static void calculate_reorder_indexes (GstVideoFormat in_format, - GstVulkanImageMemory * in_mems[GST_VIDEO_MAX_COMPONENTS], + GstVulkanImageView * in_views[GST_VIDEO_MAX_COMPONENTS], GstVideoFormat out_format, - GstVulkanImageMemory * out_mems[GST_VIDEO_MAX_COMPONENTS], + GstVulkanImageView * out_views[GST_VIDEO_MAX_COMPONENTS], int ret_in[GST_VIDEO_MAX_COMPONENTS], int ret_out[GST_VIDEO_MAX_COMPONENTS]) { const GstVideoFormatInfo *in_finfo, *out_finfo; @@ -259,9 +261,9 @@ calculate_reorder_indexes (GstVideoFormat in_format, out_finfo = gst_video_format_get_info (out_format); for (i = 0; i < in_finfo->n_planes; i++) - in_vk_formats[i] = in_mems[i]->create_info.format; + in_vk_formats[i] = in_views[i]->image->create_info.format; for (i = 0; i < out_finfo->n_planes; i++) - out_vk_formats[i] = out_mems[i]->create_info.format; + out_vk_formats[i] = out_views[i]->image->create_info.format; get_vulkan_format_swizzle_order (in_format, in_vk_formats, in_vk_order); video_format_to_reorder (in_format, in_reorder, TRUE); @@ -346,8 +348,8 @@ update_descriptor_set (GstVulkanViewConvert * conv, VkImageView * views, } static gboolean -_update_uniform (GstVulkanViewConvert * conv, GstVulkanImageMemory ** in_mems, - GstVulkanImageMemory ** out_mems, VkImageView views[GST_VIDEO_MAX_PLANES]) +_update_uniform (GstVulkanViewConvert * conv, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views, VkImageView views[GST_VIDEO_MAX_PLANES]) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); GstVideoMultiviewMode in_mode, out_mode; @@ -358,8 +360,8 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageMemory ** in_mems, gboolean mono_input = FALSE; calculate_reorder_indexes (GST_VIDEO_INFO_FORMAT (&render->in_info), - in_mems, GST_VIDEO_INFO_FORMAT (&render->out_info), - out_mems, data.in_reorder_idx, data.out_reorder_idx); + in_views, GST_VIDEO_INFO_FORMAT (&render->out_info), + out_views, data.in_reorder_idx, data.out_reorder_idx); data.tex_scale[0][0] = data.tex_scale[0][1] = 1.; data.tex_scale[1][0] = data.tex_scale[1][1] = 1.; @@ -487,20 +489,20 @@ _update_uniform (GstVulkanViewConvert * conv, GstVulkanImageMemory ** in_mems, static gboolean view_convert_update_command_state (GstVulkanViewConvert * conv, - VkCommandBuffer cmd, GstVulkanImageMemory ** in_mems, - GstVulkanImageMemory ** out_mems) + VkCommandBuffer cmd, GstVulkanImageView ** in_views, + GstVulkanImageView ** out_views) { GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (conv); VkImageView views[GST_VIDEO_MAX_PLANES]; int i; for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->in_info); i++) { - views[2 * i] = in_mems[i]->view; - views[2 * i + 1] = in_mems[i]->view; + views[2 * i] = in_views[i]->view; + views[2 * i + 1] = in_views[i]->view; } if (!conv->descriptor_up_to_date) { - if (!_update_uniform (conv, in_mems, out_mems, views)) + if (!_update_uniform (conv, in_views, out_views, views)) return FALSE; update_descriptor_set (conv, views, GST_VIDEO_INFO_N_PLANES (&render->in_info) * 2); @@ -2232,7 +2234,9 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, GstVulkanFullScreenRender *render = GST_VULKAN_FULL_SCREEN_RENDER (bt); GstVulkanViewConvert *conv = GST_VULKAN_VIEW_CONVERT (bt); GstVulkanImageMemory *in_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; - GstVulkanImageMemory *out_img_mems[4] = { NULL, }; + GstVulkanImageView *in_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageMemory *out_img_mems[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstVulkanImageView *out_img_views[GST_VIDEO_MAX_PLANES] = { NULL, }; GstVulkanFence *fence = NULL; GstVulkanCommandBuffer *cmd_buf; VkFramebuffer framebuffer; @@ -2248,6 +2252,10 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } in_img_mems[i] = (GstVulkanImageMemory *) mem; + in_img_views[i] = get_or_create_image_view (in_img_mems[i]); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + (GstMiniObject *) in_img_views[i])); } for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { @@ -2258,6 +2266,10 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, goto error; } out_img_mems[i] = (GstVulkanImageMemory *) mem; + in_img_views[i] = get_or_create_image_view (out_img_mems[i]); + gst_vulkan_trash_list_add (render->trash_list, + gst_vulkan_trash_new_mini_object_unref (gst_vulkan_fence_ref (fence), + (GstMiniObject *) out_img_views[i])); } if (!conv->cmd_pool) { @@ -2354,7 +2366,7 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, { VkImageView attachments[4] = { 0, }; for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&render->out_info); i++) { - attachments[i] = out_img_mems[i]->view; + attachments[i] = out_img_views[i]->view; } if (!(framebuffer = _create_framebuffer (conv, @@ -2365,8 +2377,8 @@ gst_vulkan_view_convert_transform (GstBaseTransform * bt, GstBuffer * inbuf, } } - view_convert_update_command_state (conv, cmd_buf->cmd, in_img_mems, - out_img_mems); + view_convert_update_command_state (conv, cmd_buf->cmd, in_img_views, + out_img_views); if (!gst_vulkan_full_screen_render_fill_command_buffer (render, cmd_buf->cmd, framebuffer)) { diff --git a/gst-libs/gst/vulkan/gstvkimagememory.c b/gst-libs/gst/vulkan/gstvkimagememory.c index 5a92fb32f9..f691773cfc 100644 --- a/gst-libs/gst/vulkan/gstvkimagememory.c +++ b/gst-libs/gst/vulkan/gstvkimagememory.c @@ -113,34 +113,6 @@ gst_vulkan_format_from_video_format (GstVideoFormat v_format, guint plane) } } -static void -_view_create_info (VkImage image, VkFormat format, VkImageViewCreateInfo * info) -{ - /* *INDENT-OFF* */ - *info = (VkImageViewCreateInfo) { - .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .pNext = NULL, - .image = image, - .format = format, - .viewType = VK_IMAGE_VIEW_TYPE_2D, - .flags = 0, - .components = (VkComponentMapping) { - VK_COMPONENT_SWIZZLE_R, - VK_COMPONENT_SWIZZLE_G, - VK_COMPONENT_SWIZZLE_B, - VK_COMPONENT_SWIZZLE_A - }, - .subresourceRange = (VkImageSubresourceRange) { - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - } - }; - /* *INDENT-ON* */ -} - static gboolean _create_info_from_args (VkImageCreateInfo * info, VkFormat format, gsize width, gsize height, VkImageTiling tiling, VkImageUsageFlags usage) @@ -210,6 +182,8 @@ _vk_image_mem_init (GstVulkanImageMemory * mem, GstAllocator * allocator, g_mutex_init (&mem->lock); + mem->views = g_ptr_array_new (); + GST_CAT_DEBUG (GST_CAT_VULKAN_IMAGE_MEMORY, "new Vulkan Image memory:%p size:%" G_GSIZE_FORMAT, mem, maxsize); } @@ -223,7 +197,6 @@ _vk_image_mem_new_alloc (GstAllocator * allocator, GstMemory * parent, { GstVulkanImageMemory *mem = NULL; GstAllocationParams params = { 0, }; - VkImageViewCreateInfo view_info; VkImageCreateInfo image_info; VkPhysicalDevice gpu; GError *error = NULL; @@ -276,14 +249,6 @@ _vk_image_mem_new_alloc (GstAllocator * allocator, GstMemory * parent, if (gst_vulkan_error_to_g_error (err, &error, "vkBindImageMemory") < 0) goto vk_error; - if (usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) { - _view_create_info (mem->image, format, &view_info); - err = vkCreateImageView (device->device, &view_info, NULL, &mem->view); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateImageView") < 0) - goto vk_error; - } - return mem; vk_error: @@ -310,7 +275,6 @@ _vk_image_mem_new_wrapped (GstAllocator * allocator, GstMemory * parent, { GstVulkanImageMemory *mem = g_new0 (GstVulkanImageMemory, 1); GstAllocationParams params = { 0, }; - VkImageViewCreateInfo view_info; VkPhysicalDevice gpu; GError *error = NULL; VkResult err; @@ -339,16 +303,6 @@ _vk_image_mem_new_wrapped (GstAllocator * allocator, GstMemory * parent, "vkGetPhysicalDeviceImageFormatProperties") < 0) goto vk_error; - /* XXX: we don't actually if the image has a vkDeviceMemory bound so - * this may fail */ - if (usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_STORAGE_BIT | - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT)) { - _view_create_info (mem->image, format, &view_info); - err = vkCreateImageView (device->device, &view_info, NULL, &mem->view); - if (gst_vulkan_error_to_g_error (err, &error, "vkCreateImageView") < 0) - goto vk_error; - } - return mem; vk_error: @@ -438,12 +392,15 @@ _vk_image_mem_free (GstAllocator * allocator, GstMemory * memory) GST_CAT_TRACE (GST_CAT_VULKAN_IMAGE_MEMORY, "freeing image memory:%p " "id:%" G_GUINT64_FORMAT, mem, (guint64) mem->image); + if (mem->views->len > 0) { + g_warning ("GstVulkanImageMemory <%p> is being freed with outstanding " + "GstVulkanImageView's. This usually means there is a reference " + "counting issue.", mem); + } + if (mem->image && !mem->wrapped) vkDestroyImage (mem->device->device, mem->image, NULL); - if (mem->view) - vkDestroyImageView (mem->device->device, mem->view, NULL); - if (mem->vk_mem) gst_memory_unref ((GstMemory *) mem->vk_mem); @@ -547,6 +504,103 @@ gst_vulkan_image_memory_get_height (GstVulkanImageMemory * image) return image->create_info.extent.height; } +static gint +find_view_index_unlocked (GstVulkanImageMemory * image, + GstVulkanImageView * view) +{ + guint index; + + if (!g_ptr_array_find (image->views, view, &index)) + return -1; + + return (gint) index; +} + +static void +gst_vulkan_image_memory_remove_view (GstVulkanImageMemory * image, + GstVulkanImageView * view) +{ + guint index; + + g_return_if_fail (gst_is_vulkan_image_memory (GST_MEMORY_CAST (image))); + + g_mutex_lock (&image->lock); + if (!g_ptr_array_find (image->views, view, &index)) { + g_warning ("GstVulkanImageMemory:%p Attempting to remove a view %p" + "that we do not own", image, view); + g_assert_not_reached (); + } + g_ptr_array_remove_index_fast (image->views, index); + g_mutex_unlock (&image->lock); +} + +static void +view_free_notify (GstVulkanImageMemory * image, GstVulkanImageView * view) +{ + gst_vulkan_image_memory_remove_view (image, view); +} + +/** + * gst_vulkan_image_memory_add_view: + * @image: a #GstVulkanImageMemory + * @view: a #GstVulkanImageView + * + * Return: the height of @image + * + * Since: 1.18 + */ +void +gst_vulkan_image_memory_add_view (GstVulkanImageMemory * image, + GstVulkanImageView * view) +{ + g_return_if_fail (gst_is_vulkan_image_memory (GST_MEMORY_CAST (image))); + g_return_if_fail (view != NULL); + g_return_if_fail (image == view->image); + + g_mutex_lock (&image->lock); + if (find_view_index_unlocked (image, view) != -1) { + g_warn_if_reached (); + g_mutex_unlock (&image->lock); + return; + } + g_ptr_array_add (image->views, view); + gst_mini_object_weak_ref (GST_MINI_OBJECT_CAST (view), + (GstMiniObjectNotify) view_free_notify, image); + + g_mutex_unlock (&image->lock); +} + +/** + * gst_vulkan_image_memory_find_view: + * @image: a #GstVulkanImageMemory + * @find_func: #GstVulkanImageMemoryFindViewFunc to search with + * @data: user data to call @finc_func with + * + * Return: (transfer full): the first #GstVulkanImageView that @find_func + * returns %TRUE for, or %NULL + * + * Since: 1.18 + */ +GstVulkanImageView * +gst_vulkan_image_memory_find_view (GstVulkanImageMemory * image, + GstVulkanImageMemoryFindViewFunc find_func, gpointer data) +{ + GstVulkanImageView *ret = NULL; + guint index; + + g_return_val_if_fail (gst_is_vulkan_image_memory (GST_MEMORY_CAST (image)), + NULL); + g_return_val_if_fail (find_func != NULL, NULL); + + g_mutex_lock (&image->lock); + if (g_ptr_array_find_with_equal_func (image->views, data, + (GEqualFunc) find_func, &index)) + ret = gst_vulkan_image_view_ref (g_ptr_array_index (image->views, index)); + g_mutex_unlock (&image->lock); + + return ret; +} + G_DEFINE_TYPE (GstVulkanImageMemoryAllocator, gst_vulkan_image_memory_allocator, GST_TYPE_ALLOCATOR); diff --git a/gst-libs/gst/vulkan/gstvkimagememory.h b/gst-libs/gst/vulkan/gstvkimagememory.h index 9e0329ce8b..e0cd975ceb 100644 --- a/gst-libs/gst/vulkan/gstvkimagememory.h +++ b/gst-libs/gst/vulkan/gstvkimagememory.h @@ -58,7 +58,6 @@ struct _GstVulkanImageMemory GstVulkanDevice * device; VkImage image; - VkImageView view; GstVulkanMemory *vk_mem; VkImageCreateInfo create_info; @@ -72,8 +71,18 @@ struct _GstVulkanImageMemory gboolean wrapped; GDestroyNotify notify; gpointer user_data; + + GPtrArray *views; }; +/** + * GstVulkanImageMemoryFindViewFunc: + * + * Function definition used to find views. Return %TRUE if @view matches the + * criteria. + */ +typedef gboolean (*GstVulkanImageMemoryFindViewFunc) (GstVulkanImageView * view, gpointer user_data); + /** * GstVulkanImageMemoryAllocator * @@ -124,6 +133,14 @@ guint32 gst_vulkan_image_memory_get_width (GstVulkanImageMemory * GST_VULKAN_API guint32 gst_vulkan_image_memory_get_height (GstVulkanImageMemory * image); +GST_VULKAN_API +GstVulkanImageView *gst_vulkan_image_memory_find_view (GstVulkanImageMemory * image, + GstVulkanImageMemoryFindViewFunc find_func, + gpointer user_data); +GST_VULKAN_API +void gst_vulkan_image_memory_add_view (GstVulkanImageMemory * image, + GstVulkanImageView * view); + GST_VULKAN_API VkFormat gst_vulkan_format_from_video_format (GstVideoFormat v_format, guint plane); diff --git a/gst-libs/gst/vulkan/gstvkimageview.c b/gst-libs/gst/vulkan/gstvkimageview.c new file mode 100644 index 0000000000..1c23ce1cee --- /dev/null +++ b/gst-libs/gst/vulkan/gstvkimageview.c @@ -0,0 +1,125 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +/** + * SECTION:vkimageview + * @title: GstVulkanImageView + * @short_description: wrapper for `VkImageView`'s + * @see_also: #GstVulkanImageMemory + * + * #GstVulkanImageView is a wrapper around a `VkImageView` mostly for + * usage across element boundaries with #GstVulkanImageMemory + */ + +#define GST_CAT_DEFUALT GST_CAT_VULKAN_IMAGE_VIEW +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFUALT); + +static void +init_debug (void) +{ + static volatile gsize _init = 0; + + if (g_once_init_enter (&_init)) { + GST_DEBUG_CATEGORY_INIT (GST_CAT_VULKAN_IMAGE_VIEW, "vulkanimageview", + 0, "Vulkan Image View"); + g_once_init_leave (&_init, 1); + } +} + +static void +gst_vulkan_image_view_free (GstVulkanImageView * view) +{ + GST_CAT_TRACE (GST_CAT_VULKAN_IMAGE_VIEW, "freeing image view:%p ", view); + + if (view->view) + vkDestroyImageView (view->device->device, view->view, NULL); + + if (view->image) + gst_memory_unref (GST_MEMORY_CAST (view->image)); + view->image = NULL; + gst_clear_object (&view->device); + + g_free (view); +} + +/** + * gst_vulkan_image_view_new: + * @image: a #GstVulkanImageMemory to create the new view from + * @create_info: the creation information to create the view from + * + * Returns: (transfer full): A new #GstVulkanImageView from @image and + * @create_info + */ +GstVulkanImageView * +gst_vulkan_image_view_new (GstVulkanImageMemory * image, + VkImageViewCreateInfo * create_info) +{ + GstVulkanImageView *view; + GError *error = NULL; + VkResult err; + + g_return_val_if_fail (create_info != NULL, NULL); + g_return_val_if_fail (gst_is_vulkan_image_memory (GST_MEMORY_CAST (image)), + NULL); + + init_debug (); + + view = g_new0 (GstVulkanImageView, 1); + + gst_mini_object_init ((GstMiniObject *) view, 0, + gst_vulkan_image_view_get_type (), NULL, NULL, + (GstMiniObjectFreeFunction) gst_vulkan_image_view_free); + + err = + vkCreateImageView (image->device->device, create_info, NULL, &view->view); + if (gst_vulkan_error_to_g_error (err, &error, "vkImageCreateView") < 0) + goto vk_error; + + view->image = + (GstVulkanImageMemory *) gst_memory_ref (GST_MEMORY_CAST (image)); + view->device = gst_object_ref (image->device); + view->create_info = *create_info; + /* we cannot keep this as it may point to stack allocated memory */ + view->create_info.pNext = NULL; + + return view; + +vk_error: + { + GST_CAT_ERROR (GST_CAT_VULKAN_IMAGE_VIEW, + "Failed to allocate image memory %s", error->message); + g_clear_error (&error); + goto error; + } + +error: + { + g_free (view); + return NULL; + } +} + +GST_DEFINE_MINI_OBJECT_TYPE (GstVulkanImageView, gst_vulkan_image_view); diff --git a/gst-libs/gst/vulkan/gstvkimageview.h b/gst-libs/gst/vulkan/gstvkimageview.h new file mode 100644 index 0000000000..6ba0eae337 --- /dev/null +++ b/gst-libs/gst/vulkan/gstvkimageview.h @@ -0,0 +1,83 @@ +/* + * GStreamer + * Copyright (C) 2019 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_VULKAN_IMAGE_VIEW_H__ +#define __GST_VULKAN_IMAGE_VIEW_H__ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_VULKAN_IMAGE_VIEW (gst_vulkan_image_view_get_type()) +GST_VULKAN_API +GType gst_vulkan_image_view_get_type(void); + +struct _GstVulkanImageView +{ + GstMiniObject parent; + + GstVulkanDevice * device; + + GstVulkanImageMemory *image; + VkImageView view; + + VkImageViewCreateInfo create_info; +}; + +/** + * gst_vulkan_image_view_ref: (skip) + * @trash: a #GstVulkanImageView. + * + * Increases the refcount of the given trash object by one. + * + * Returns: (transfer full): @trash + */ +static inline GstVulkanImageView* gst_vulkan_image_view_ref(GstVulkanImageView* trash); +static inline GstVulkanImageView * +gst_vulkan_image_view_ref (GstVulkanImageView * trash) +{ + return (GstVulkanImageView *) gst_mini_object_ref (GST_MINI_OBJECT_CAST (trash)); +} + +/** + * gst_vulkan_image_view_unref: (skip) + * @trash: (transfer full): a #GstVulkanImageView. + * + * Decreases the refcount of the trash object. If the refcount reaches 0, the + * trash will be freed. + */ +static inline void gst_vulkan_image_view_unref(GstVulkanImageView* trash); +static inline void +gst_vulkan_image_view_unref (GstVulkanImageView * trash) +{ + gst_mini_object_unref (GST_MINI_OBJECT_CAST (trash)); +} + +#ifdef G_DEFINE_AUTOPTR_CLEANUP_FUNC +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstVulkanImageView, gst_vulkan_image_view_unref) +#endif + +GST_VULKAN_API +GstVulkanImageView * gst_vulkan_image_view_new (GstVulkanImageMemory * image, + VkImageViewCreateInfo * create_info); + +G_END_DECLS + +#endif /* __GST_VULKAN_IMAGE_MEMORY_H__ */ diff --git a/gst-libs/gst/vulkan/meson.build b/gst-libs/gst/vulkan/meson.build index 5d5e286353..306be4a69a 100644 --- a/gst-libs/gst/vulkan/meson.build +++ b/gst-libs/gst/vulkan/meson.build @@ -16,6 +16,7 @@ vulkan_sources = [ 'gstvkformat.c', 'gstvkimagememory.c', 'gstvkimagebufferpool.c', + 'gstvkimageview.c', 'gstvkinstance.c', 'gstvkmemory.c', 'gstvkphysicaldevice.c', @@ -41,6 +42,7 @@ vulkan_headers = [ 'gstvkformat.h', 'gstvkimagememory.h', 'gstvkimagebufferpool.h', + 'gstvkimageview.h', 'gstvkinstance.h', 'gstvkmemory.h', 'gstvkphysicaldevice.h', diff --git a/gst-libs/gst/vulkan/vulkan.h b/gst-libs/gst/vulkan/vulkan.h index ec57cb75ef..db3f633382 100644 --- a/gst-libs/gst/vulkan/vulkan.h +++ b/gst-libs/gst/vulkan/vulkan.h @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include diff --git a/gst-libs/gst/vulkan/vulkan_fwd.h b/gst-libs/gst/vulkan/vulkan_fwd.h index e8c3b3f860..5f9b6e85aa 100644 --- a/gst-libs/gst/vulkan/vulkan_fwd.h +++ b/gst-libs/gst/vulkan/vulkan_fwd.h @@ -76,6 +76,8 @@ typedef struct _GstVulkanImageBufferPool GstVulkanImageBufferPool; typedef struct _GstVulkanImageBufferPoolClass GstVulkanImageBufferPoolClass; typedef struct _GstVulkanImageBufferPoolPrivate GstVulkanImageBufferPoolPrivate; +typedef struct _GstVulkanImageView GstVulkanImageView; + typedef struct _GstVulkanBarrierMemoryInfo GstVulkanBarrierMemoryInfo; typedef struct _GstVulkanBarrierBufferInfo GstVulkanBarrierBufferInfo; typedef struct _GstVulkanBarrierImageInfo GstVulkanBarrierImageInfo;