vkimagebufferpool: allow specifying initial image layout

The pool currently defaults to performing a layout transition to
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, with some special exceptions for
video usages. This may not be a legal transition depending on the usage.
Provide an API to explicitly control the initial image layout.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5881>
This commit is contained in:
Chris Spencer 2023-11-28 14:07:32 +00:00 committed by GStreamer Marge Bot
parent f0d15f335d
commit 865227b750
10 changed files with 67 additions and 42 deletions

View file

@ -3501,6 +3501,15 @@ inside @handle.</doc>
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Vulkan memory property flags.</doc> <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Vulkan memory property flags.</doc>
<type name="Vulkan.MemoryPropertyFlags" c:type="VkMemoryPropertyFlags"/> <type name="Vulkan.MemoryPropertyFlags" c:type="VkMemoryPropertyFlags"/>
</parameter> </parameter>
<parameter name="initial_layout" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Initial Vulkan image layout.</doc>
<type name="Vulkan.ImageLayout" c:type="VkImageLayout"/>
</parameter>
<parameter name="initial_access" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Access flags for the layout transition if @initial_layout is
not VK_IMAGE_LAYOUT_UNDEFINED or VK_IMAGE_LAYOUT_PREINITIALIZED.</doc>
<type name="guint64" c:type="guint64"/>
</parameter>
</parameters> </parameters>
</function> </function>
<function name="config_set_decode_caps" c:identifier="gst_vulkan_image_buffer_pool_config_set_decode_caps" version="1.24"> <function name="config_set_decode_caps" c:identifier="gst_vulkan_image_buffer_pool_config_set_decode_caps" version="1.24">
@ -3704,6 +3713,9 @@ dest or DPB images.</doc>
<parameter name="usage" transfer-ownership="none"> <parameter name="usage" transfer-ownership="none">
<type name="Vulkan.ImageUsageFlags" c:type="VkImageUsageFlags"/> <type name="Vulkan.ImageUsageFlags" c:type="VkImageUsageFlags"/>
</parameter> </parameter>
<parameter name="initial_layout" transfer-ownership="none">
<type name="Vulkan.ImageLayout" c:type="VkImageLayout"/>
</parameter>
<parameter name="params" transfer-ownership="none"> <parameter name="params" transfer-ownership="none">
<type name="Gst.AllocationParams" c:type="GstAllocationParams*"/> <type name="Gst.AllocationParams" c:type="GstAllocationParams*"/>
</parameter> </parameter>

View file

@ -370,7 +370,8 @@ gst_vulkan_h264_decoder_decide_allocation (GstVideoDecoder * decoder,
gst_buffer_pool_config_set_params (config, new_caps, size, min, max); gst_buffer_pool_config_set_params (config, new_caps, size, min, max);
gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage, gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR,
VK_ACCESS_TRANSFER_WRITE_BIT);
gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps); gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps);
gst_caps_unref (profile_caps); gst_caps_unref (profile_caps);

View file

@ -588,7 +588,8 @@ gst_vulkan_decoder_create_dpb_pool (GstVulkanDecoder * self, GstCaps * caps)
gst_buffer_pool_config_set_params (config, caps, 1024, min_buffers, gst_buffer_pool_config_set_params (config, caps, 1024, min_buffers,
max_buffers); max_buffers);
gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage, gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
if (self->layered_dpb) { if (self->layered_dpb) {
gst_structure_set (config, "num-layers", G_TYPE_UINT, gst_structure_set (config, "num-layers", G_TYPE_UINT,

View file

@ -43,6 +43,8 @@ struct _GstVulkanImageBufferPoolPrivate
GstVideoInfo v_info; GstVideoInfo v_info;
VkImageUsageFlags usage; VkImageUsageFlags usage;
VkMemoryPropertyFlags mem_props; VkMemoryPropertyFlags mem_props;
VkImageLayout initial_layout;
guint64 initial_access;
VkFormat vk_fmts[GST_VIDEO_MAX_PLANES]; VkFormat vk_fmts[GST_VIDEO_MAX_PLANES];
int n_imgs; int n_imgs;
guint32 n_layers; guint32 n_layers;
@ -69,6 +71,9 @@ G_DEFINE_TYPE_WITH_CODE (GstVulkanImageBufferPool, gst_vulkan_image_buffer_pool,
* @config: the #GstStructure with the pool's configuration. * @config: the #GstStructure with the pool's configuration.
* @usage: The Vulkan image usage flags. * @usage: The Vulkan image usage flags.
* @mem_properties: Vulkan memory property flags. * @mem_properties: Vulkan memory property flags.
* @initial_layout: Initial Vulkan image layout.
* @initial_access: Access flags for the layout transition if @initial_layout is
* not VK_IMAGE_LAYOUT_UNDEFINED or VK_IMAGE_LAYOUT_PREINITIALIZED.
* *
* Sets the @usage and @mem_properties of the images to setup. * Sets the @usage and @mem_properties of the images to setup.
* *
@ -76,11 +81,13 @@ G_DEFINE_TYPE_WITH_CODE (GstVulkanImageBufferPool, gst_vulkan_image_buffer_pool,
*/ */
void void
gst_vulkan_image_buffer_pool_config_set_allocation_params (GstStructure * gst_vulkan_image_buffer_pool_config_set_allocation_params (GstStructure *
config, VkImageUsageFlags usage, VkMemoryPropertyFlags mem_properties) config, VkImageUsageFlags usage, VkMemoryPropertyFlags mem_properties,
VkImageLayout initial_layout, guint64 initial_access)
{ {
/* assumption: G_TYPE_UINT is compatible with uint32_t (VkFlags) */ /* assumption: G_TYPE_UINT is compatible with uint32_t (VkFlags) */
gst_structure_set (config, "usage", G_TYPE_UINT, usage, "memory-properties", gst_structure_set (config, "usage", G_TYPE_UINT, usage, "memory-properties",
G_TYPE_UINT, mem_properties, NULL); G_TYPE_UINT, mem_properties, "initial-layout", G_TYPE_UINT,
initial_layout, "initial-access", G_TYPE_UINT64, initial_access, NULL);
} }
/** /**
@ -105,6 +112,7 @@ gst_vulkan_image_buffer_pool_config_set_decode_caps (GstStructure * config,
static inline gboolean static inline gboolean
gst_vulkan_image_buffer_pool_config_get_allocation_params (GstStructure * gst_vulkan_image_buffer_pool_config_get_allocation_params (GstStructure *
config, VkImageUsageFlags * usage, VkMemoryPropertyFlags * mem_props, config, VkImageUsageFlags * usage, VkMemoryPropertyFlags * mem_props,
VkImageLayout * initial_layout, guint64 * initial_access,
guint32 * n_layers, GstCaps ** decode_caps) guint32 * n_layers, GstCaps ** decode_caps)
{ {
if (!gst_structure_get_uint (config, "usage", usage)) { if (!gst_structure_get_uint (config, "usage", usage)) {
@ -116,6 +124,12 @@ gst_vulkan_image_buffer_pool_config_get_allocation_params (GstStructure *
if (!gst_structure_get_uint (config, "memory-properties", mem_props)) if (!gst_structure_get_uint (config, "memory-properties", mem_props))
*mem_props = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; *mem_props = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
if (!gst_structure_get_uint (config, "initial-layout", initial_layout))
*initial_layout = VK_IMAGE_LAYOUT_UNDEFINED;
if (!gst_structure_get_uint64 (config, "initial-access", initial_access))
*initial_access = 0;
if (!gst_structure_get_uint (config, "num-layers", n_layers)) if (!gst_structure_get_uint (config, "num-layers", n_layers))
*n_layers = 1; *n_layers = 1;
@ -161,7 +175,8 @@ gst_vulkan_image_buffer_pool_set_config (GstBufferPool * pool,
GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY); GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY);
gst_vulkan_image_buffer_pool_config_get_allocation_params (config, gst_vulkan_image_buffer_pool_config_get_allocation_params (config,
&priv->usage, &priv->mem_props, &priv->n_layers, &decode_caps); &priv->usage, &priv->mem_props, &priv->initial_layout,
&priv->initial_access, &priv->n_layers, &decode_caps);
priv->has_profile = FALSE; priv->has_profile = FALSE;
#if GST_VULKAN_HAVE_VIDEO_EXTENSIONS #if GST_VULKAN_HAVE_VIDEO_EXTENSIONS
@ -224,7 +239,9 @@ gst_vulkan_image_buffer_pool_set_config (GstBufferPool * pool,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE, .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0, .queueFamilyIndexCount = 0,
.pQueueFamilyIndices = NULL, .pQueueFamilyIndices = NULL,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, .initialLayout = priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED
? VK_IMAGE_LAYOUT_PREINITIALIZED
: VK_IMAGE_LAYOUT_UNDEFINED,
}; };
/* *INDENT-ON* */ /* *INDENT-ON* */
priv->v_info.size = 0; priv->v_info.size = 0;
@ -324,10 +341,12 @@ prepare_buffer (GstVulkanImageBufferPool * vk_pool, GstBuffer * buffer)
#if defined(VK_KHR_synchronization2) #if defined(VK_KHR_synchronization2)
GstVulkanImageBufferPoolPrivate *priv = GET_PRIV (vk_pool); GstVulkanImageBufferPoolPrivate *priv = GET_PRIV (vk_pool);
GArray *barriers = NULL; GArray *barriers = NULL;
VkImageLayout new_layout;
guint64 new_access;
GError *error = NULL; GError *error = NULL;
if (priv->initial_layout == VK_IMAGE_LAYOUT_UNDEFINED ||
priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED)
return TRUE;
if (!priv->exec) { if (!priv->exec) {
GstVulkanCommandPool *cmd_pool; GstVulkanCommandPool *cmd_pool;
GstVulkanQueue *queue = NULL; GstVulkanQueue *queue = NULL;
@ -349,31 +368,12 @@ prepare_buffer (GstVulkanImageBufferPool * vk_pool, GstBuffer * buffer)
VK_PIPELINE_STAGE_NONE_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT)) VK_PIPELINE_STAGE_NONE_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT))
return FALSE; return FALSE;
#if GST_VULKAN_HAVE_VIDEO_EXTENSIONS
if ((priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) != 0 &&
(priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR) == 0) {
new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
} else if ((priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR) != 0) {
new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
#ifdef VK_ENABLE_BETA_EXTENSIONS
} else if ((priv->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) != 0) {
new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
#endif
} else
#endif
{
new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
new_access = VK_ACCESS_MEMORY_WRITE_BIT;
}
if (!gst_vulkan_operation_begin (priv->exec, &error)) if (!gst_vulkan_operation_begin (priv->exec, &error))
goto error; goto error;
if (!gst_vulkan_operation_add_frame_barrier (priv->exec, buffer, if (!gst_vulkan_operation_add_frame_barrier (priv->exec, buffer,
VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, new_access, new_layout, NULL)) VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, priv->initial_access,
priv->initial_layout, NULL))
goto error; goto error;
barriers = gst_vulkan_operation_retrieve_image_barriers (priv->exec); barriers = gst_vulkan_operation_retrieve_image_barriers (priv->exec);
@ -442,7 +442,9 @@ gst_vulkan_image_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE, .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0, .queueFamilyIndexCount = 0,
.pQueueFamilyIndices = NULL, .pQueueFamilyIndices = NULL,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, .initialLayout = priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED
? VK_IMAGE_LAYOUT_PREINITIALIZED
: VK_IMAGE_LAYOUT_UNDEFINED,
}; };
/* *INDENT-ON* */ /* *INDENT-ON* */

View file

@ -84,7 +84,9 @@ GST_VULKAN_API
void gst_vulkan_image_buffer_pool_config_set_allocation_params void gst_vulkan_image_buffer_pool_config_set_allocation_params
(GstStructure * config, (GstStructure * config,
VkImageUsageFlags usage, VkImageUsageFlags usage,
VkMemoryPropertyFlags mem_properties); VkMemoryPropertyFlags mem_properties,
VkImageLayout initial_layout,
guint64 initial_access);
GST_VULKAN_API GST_VULKAN_API
void gst_vulkan_image_buffer_pool_config_set_decode_caps void gst_vulkan_image_buffer_pool_config_set_decode_caps

View file

@ -71,8 +71,9 @@ _create_info_from_args (VkImageCreateInfo * info, VkFormat format, gsize width,
gboolean gboolean
gst_vulkan_image_memory_init (GstVulkanImageMemory * mem, gst_vulkan_image_memory_init (GstVulkanImageMemory * mem,
GstAllocator * allocator, GstMemory * parent, GstVulkanDevice * device, GstAllocator * allocator, GstMemory * parent, GstVulkanDevice * device,
VkFormat format, VkImageUsageFlags usage, GstAllocationParams * params, VkFormat format, VkImageUsageFlags usage, VkImageLayout layout,
gsize size, gpointer user_data, GDestroyNotify notify) GstAllocationParams * params, gsize size, gpointer user_data,
GDestroyNotify notify)
{ {
gsize align = gst_memory_alignment, offset = 0, maxsize = size; gsize align = gst_memory_alignment, offset = 0, maxsize = size;
GstMemoryFlags flags = 0; GstMemoryFlags flags = 0;
@ -93,7 +94,7 @@ gst_vulkan_image_memory_init (GstVulkanImageMemory * mem,
mem->barrier.parent.access_flags = 0; mem->barrier.parent.access_flags = 0;
mem->barrier.parent.semaphore = VK_NULL_HANDLE; mem->barrier.parent.semaphore = VK_NULL_HANDLE;
mem->barrier.parent.semaphore_value = 0; mem->barrier.parent.semaphore_value = 0;
mem->barrier.image_layout = VK_IMAGE_LAYOUT_UNDEFINED; mem->barrier.image_layout = layout;
/* *INDENT-OFF* */ /* *INDENT-OFF* */
mem->barrier.subresource_range = (VkImageSubresourceRange) { mem->barrier.subresource_range = (VkImageSubresourceRange) {
.aspectMask = gst_vulkan_format_get_aspect (format), .aspectMask = gst_vulkan_format_get_aspect (format),
@ -158,8 +159,8 @@ _vk_image_mem_new_alloc_with_image_info (GstAllocator * allocator,
vkGetImageMemoryRequirements (device->device, image, &mem->requirements); vkGetImageMemoryRequirements (device->device, image, &mem->requirements);
gst_vulkan_image_memory_init (mem, allocator, parent, device, gst_vulkan_image_memory_init (mem, allocator, parent, device,
image_info->format, image_info->usage, &params, mem->requirements.size, image_info->format, image_info->usage, image_info->initialLayout, &params,
user_data, notify); mem->requirements.size, user_data, notify);
mem->create_info = *image_info; mem->create_info = *image_info;
/* XXX: to avoid handling pNext lifetime */ /* XXX: to avoid handling pNext lifetime */
mem->create_info.pNext = NULL; mem->create_info.pNext = NULL;
@ -270,7 +271,8 @@ _vk_image_mem_new_wrapped (GstAllocator * allocator, GstMemory * parent,
params.align = mem->requirements.alignment - 1; params.align = mem->requirements.alignment - 1;
params.flags = GST_MEMORY_FLAG_NOT_MAPPABLE; params.flags = GST_MEMORY_FLAG_NOT_MAPPABLE;
gst_vulkan_image_memory_init (mem, allocator, parent, device, format, usage, gst_vulkan_image_memory_init (mem, allocator, parent, device, format, usage,
&params, mem->requirements.size, user_data, notify); VK_IMAGE_LAYOUT_UNDEFINED, &params, mem->requirements.size, user_data,
notify);
mem->wrapped = TRUE; mem->wrapped = TRUE;
if (!_create_info_from_args (&mem->create_info, format, width, height, tiling, if (!_create_info_from_args (&mem->create_info, format, width, height, tiling,

View file

@ -172,6 +172,7 @@ gboolean gst_vulkan_image_memory_init (GstVulkanImageMemory *
GstVulkanDevice * device, GstVulkanDevice * device,
VkFormat format, VkFormat format,
VkImageUsageFlags usage, VkImageUsageFlags usage,
VkImageLayout initial_layout,
GstAllocationParams * params, GstAllocationParams * params,
gsize size, gsize size,
gpointer user_data, gpointer user_data,

View file

@ -195,7 +195,8 @@ _io_surface_vulkan_memory_new (GstVulkanDevice * device, IOSurfaceRef surface,
gst_vulkan_image_memory_init (&mem->vulkan_mem, gst_vulkan_image_memory_init (&mem->vulkan_mem,
_io_surface_vulkan_memory_allocator, NULL, device, vk_format, usage, _io_surface_vulkan_memory_allocator, NULL, device, vk_format, usage,
&params, mem->vulkan_mem.requirements.size, user_data, notify); VK_IMAGE_LAYOUT_UNDEFINED, &params, mem->vulkan_mem.requirements.size,
user_data, notify);
mem->vulkan_mem.create_info = image_info; mem->vulkan_mem.create_info = image_info;
mem->vulkan_mem.image = image; mem->vulkan_mem.image = image;
mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL; mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL;

View file

@ -60,7 +60,7 @@ setup_queue (guint expected_flags)
static GstBufferPool * static GstBufferPool *
create_buffer_pool (const char *format, VkImageUsageFlags usage, create_buffer_pool (const char *format, VkImageUsageFlags usage,
GstCaps * dec_caps) VkImageLayout initial_layout, guint64 initial_access, GstCaps * dec_caps)
{ {
GstCaps *caps; GstCaps *caps;
GstBufferPool *pool; GstBufferPool *pool;
@ -79,7 +79,8 @@ create_buffer_pool (const char *format, VkImageUsageFlags usage,
gst_caps_unref (caps); gst_caps_unref (caps);
gst_vulkan_image_buffer_pool_config_set_allocation_params (config, gst_vulkan_image_buffer_pool_config_set_allocation_params (config,
usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, initial_layout,
initial_access);
if (dec_caps) if (dec_caps)
gst_vulkan_image_buffer_pool_config_set_decode_caps (config, dec_caps); gst_vulkan_image_buffer_pool_config_set_decode_caps (config, dec_caps);
@ -97,7 +98,7 @@ GST_START_TEST (test_image)
GstBuffer *buffer = NULL; GstBuffer *buffer = NULL;
setup_queue (VK_QUEUE_COMPUTE_BIT); setup_queue (VK_QUEUE_COMPUTE_BIT);
pool = create_buffer_pool ("NV12", 0, NULL); pool = create_buffer_pool ("NV12", 0, VK_IMAGE_LAYOUT_UNDEFINED, 0, NULL);
ret = gst_buffer_pool_acquire_buffer (pool, &buffer, NULL); ret = gst_buffer_pool_acquire_buffer (pool, &buffer, NULL);
fail_unless (ret == GST_FLOW_OK); fail_unless (ret == GST_FLOW_OK);
@ -191,6 +192,7 @@ GST_START_TEST (test_decoding_image)
fail_unless (dec_caps); fail_unless (dec_caps);
pool = create_buffer_pool ("NV12", VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR, pool = create_buffer_pool ("NV12", VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR,
VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR, VK_ACCESS_TRANSFER_WRITE_BIT,
dec_caps); dec_caps);
gst_caps_unref (dec_caps); gst_caps_unref (dec_caps);

View file

@ -305,7 +305,8 @@ GST_START_TEST (test_decoder)
gst_caps_unref (caps); gst_caps_unref (caps);
gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage, gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR, VK_ACCESS_TRANSFER_WRITE_BIT);
gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps); gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps);
gst_caps_unref (profile_caps); gst_caps_unref (profile_caps);