From b02493091b23a48d387d6e7667074cc9901993b2 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 8 May 2019 21:26:17 +1000 Subject: [PATCH] vkswapper: support displaying memory:VulkanImage --- ext/vulkan/vksink.c | 5 ++- ext/vulkan/vkswapper.c | 71 ++++++++++++++++++++++++++++-------------- ext/vulkan/vkupload.c | 8 +++++ 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/ext/vulkan/vksink.c b/ext/vulkan/vksink.c index f8d9c7a43e..60354e146f 100644 --- a/ext/vulkan/vksink.c +++ b/ext/vulkan/vksink.c @@ -70,10 +70,13 @@ static void gst_vulkan_sink_video_overlay_init (GstVideoOverlayInterface * static GstStaticPadTemplate gst_vulkan_sink_template = -GST_STATIC_PAD_TEMPLATE ("sink", + GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE, + GST_VULKAN_SWAPPER_VIDEO_FORMATS) ";" + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_VULKAN_BUFFER, GST_VULKAN_SWAPPER_VIDEO_FORMATS))); diff --git a/ext/vulkan/vkswapper.c b/ext/vulkan/vkswapper.c index 001f2edd2d..97d8af25b5 100644 --- a/ext/vulkan/vkswapper.c +++ b/ext/vulkan/vkswapper.c @@ -450,7 +450,7 @@ gst_vulkan_swapper_get_supported_caps (GstVulkanSwapper * swapper, caps = gst_caps_new_empty_simple ("video/x-raw"); gst_caps_set_features (caps, 0, - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VULKAN_BUFFER)); + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE)); s = gst_caps_get_structure (caps, 0); { @@ -481,6 +481,9 @@ gst_vulkan_swapper_get_supported_caps (GstVulkanSwapper * swapper, G_MAXINT, 1, NULL); } + gst_caps_append_structure_full (caps, gst_structure_copy (s), + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_VULKAN_BUFFER)); + GST_INFO_OBJECT (swapper, "Probed the following caps %" GST_PTR_FORMAT, caps); return caps; @@ -649,8 +652,8 @@ _allocate_swapchain (GstVulkanSwapper * swapper, GstCaps * caps, n_images_wanted = swapper->surf_props.maxImageCount; } - if (swapper-> - surf_props.supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) { + if (swapper->surf_props. + supportedTransforms & VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR) { preTransform = VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; } else { preTransform = swapper->surf_props.currentTransform; @@ -681,8 +684,8 @@ _allocate_swapchain (GstVulkanSwapper * swapper, GstCaps * caps, "Incorrect usage flags available for the swap images"); return FALSE; } - if ((swapper->surf_props. - supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) + if ((swapper-> + surf_props.supportedUsageFlags & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) != 0) { usage |= VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; } else { @@ -804,9 +807,10 @@ static gboolean _build_render_buffer_cmd (GstVulkanSwapper * swapper, guint32 swap_idx, GstBuffer * buffer, VkCommandBuffer * cmd_ret, GError ** error) { - GstVulkanBufferMemory *buf_mem; + GstMemory *in_mem; GstVulkanImageMemory *swap_mem; VkCommandBuffer cmd; + GstVideoRectangle src, dst, rslt; VkResult err; g_return_val_if_fail (swap_idx < swapper->n_swap_chain_images, FALSE); @@ -815,8 +819,6 @@ _build_render_buffer_cmd (GstVulkanSwapper * swapper, guint32 swap_idx, if (!(cmd = gst_vulkan_command_pool_create (swapper->cmd_pool, error))) return FALSE; - buf_mem = (GstVulkanBufferMemory *) gst_buffer_peek_memory (buffer, 0); - { VkCommandBufferInheritanceInfo buf_inh = { 0, }; VkCommandBufferBeginInfo cmd_buf_info = { 0, }; @@ -845,23 +847,25 @@ _build_render_buffer_cmd (GstVulkanSwapper * swapper, guint32 swap_idx, return FALSE; } - { + src.x = src.y = 0; + src.w = GST_VIDEO_INFO_WIDTH (&swapper->v_info); + src.h = GST_VIDEO_INFO_HEIGHT (&swapper->v_info); + + dst.x = dst.y = 0; + dst.w = gst_vulkan_image_memory_get_width (swap_mem); + dst.h = gst_vulkan_image_memory_get_height (swap_mem); + + gst_video_sink_center_rect (src, dst, &rslt, FALSE); + + GST_TRACE_OBJECT (swapper, "rendering into result rectangle %ux%u+%u,%u " + "src %ux%u dst %ux%u", rslt.w, rslt.h, rslt.x, rslt.y, src.w, src.h, + dst.w, dst.h); + + in_mem = gst_buffer_peek_memory (buffer, 0); + if (gst_is_vulkan_buffer_memory (in_mem)) { + GstVulkanBufferMemory *buf_mem = (GstVulkanBufferMemory *) in_mem; VkBufferImageCopy region = { 0, }; - GstVideoRectangle src, dst, rslt; - src.x = src.y = 0; - src.w = GST_VIDEO_INFO_WIDTH (&swapper->v_info); - src.h = GST_VIDEO_INFO_HEIGHT (&swapper->v_info); - - dst.x = dst.y = 0; - dst.w = gst_vulkan_image_memory_get_width (swap_mem); - dst.h = gst_vulkan_image_memory_get_height (swap_mem); - - gst_video_sink_center_rect (src, dst, &rslt, FALSE); - - GST_TRACE_OBJECT (swapper, "rendering into result rectangle %ux%u+%u,%u " - "src %ux%u dst %ux%u", rslt.w, rslt.h, rslt.x, rslt.y, src.w, src.h, - dst.w, dst.h); GST_VK_BUFFER_IMAGE_COPY (region, 0, src.w, src.h, GST_VK_IMAGE_SUBRESOURCE_LAYERS_INIT (VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1), GST_VK_OFFSET3D_INIT (rslt.x, rslt.y, 0), @@ -869,6 +873,25 @@ _build_render_buffer_cmd (GstVulkanSwapper * swapper, guint32 swap_idx, vkCmdCopyBufferToImage (cmd, buf_mem->buffer, swap_mem->image, swap_mem->image_layout, 1, ®ion); + } else if (gst_is_vulkan_image_memory (in_mem)) { + GstVulkanImageMemory *img_mem = (GstVulkanImageMemory *) in_mem; + VkImageCopy region = { 0, }; + + /* FIXME: should really be a blit to resize to the output dimensions */ + GST_VK_IMAGE_COPY (region, + GST_VK_IMAGE_SUBRESOURCE_LAYERS_INIT (VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, + 1), GST_VK_OFFSET3D_INIT (src.x, src.y, 0), + GST_VK_IMAGE_SUBRESOURCE_LAYERS_INIT (VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, + 1), GST_VK_OFFSET3D_INIT (rslt.x, rslt.y, 0), + GST_VK_EXTENT3D_INIT (rslt.w, rslt.h, 1)); + + if (!_swapper_set_image_layout_with_cmd (swapper, cmd, img_mem, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, error)) { + return FALSE; + } + + vkCmdCopyImage (cmd, img_mem->image, img_mem->image_layout, swap_mem->image, + swap_mem->image_layout, 1, ®ion); } if (!_swapper_set_image_layout_with_cmd (swapper, cmd, swap_mem, @@ -1054,7 +1077,7 @@ gst_vulkan_swapper_render_buffer (GstVulkanSwapper * swapper, "Buffer has no memory"); return FALSE; } - if (!gst_is_vulkan_buffer_memory (mem)) { + if (!gst_is_vulkan_buffer_memory (mem) && !gst_is_vulkan_image_memory (mem)) { g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FORMAT_NOT_SUPPORTED, "Incorrect memory type"); return FALSE; diff --git a/ext/vulkan/vkupload.c b/ext/vulkan/vkupload.c index 725b8e9913..791e69ae04 100644 --- a/ext/vulkan/vkupload.c +++ b/ext/vulkan/vkupload.c @@ -503,6 +503,7 @@ _buffer_to_image_perform (gpointer impl, GstBuffer * inbuf, GstBuffer ** outbuf) GstMemory *in_mem, *out_mem; GstVulkanBufferMemory *buf_mem; GstVulkanImageMemory *img_mem; + VkImageMemoryBarrier image_memory_barrier; in_mem = gst_buffer_peek_memory (inbuf, i); if (!gst_is_vulkan_buffer_memory (in_mem)) { @@ -526,6 +527,13 @@ _buffer_to_image_perform (gpointer impl, GstBuffer * inbuf, GstBuffer ** outbuf) GST_VK_EXTENT3D_INIT (GST_VIDEO_INFO_COMP_WIDTH (&raw->out_info, i), GST_VIDEO_INFO_COMP_HEIGHT (&raw->out_info, i), 1)); + gst_vulkan_image_memory_set_layout (img_mem, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &image_memory_barrier); + + vkCmdPipelineBarrier (cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 0, 0, NULL, 0, NULL, 1, + &image_memory_barrier); + vkCmdCopyBufferToImage (cmd, buf_mem->buffer, img_mem->image, img_mem->image_layout, 1, ®ion); }