vkimagebufferpool: allow specifying initial image layout

The pool currently defaults to performing a layout transition to
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, with some special exceptions for
video usages. This may not be a legal transition depending on the usage.
Provide an API to explicitly control the initial image layout.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5881>

girs/GstVulkan-1.0.gir

@@ -3501,6 +3501,15 @@ inside @handle.</doc>
             <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Vulkan memory property flags.</doc>
             <type name="Vulkan.MemoryPropertyFlags" c:type="VkMemoryPropertyFlags"/>
           </parameter>
+          <parameter name="initial_layout" transfer-ownership="none">
+            <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Initial Vulkan image layout.</doc>
+            <type name="Vulkan.ImageLayout" c:type="VkImageLayout"/>
+          </parameter>
+          <parameter name="initial_access" transfer-ownership="none">
+            <doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c">Access flags for the layout transition if @initial_layout is
+not VK_IMAGE_LAYOUT_UNDEFINED or VK_IMAGE_LAYOUT_PREINITIALIZED.</doc>
+            <type name="guint64" c:type="guint64"/>
+          </parameter>
         </parameters>
       </function>
       <function name="config_set_decode_caps" c:identifier="gst_vulkan_image_buffer_pool_config_set_decode_caps" version="1.24">
@@ -3704,6 +3713,9 @@ dest or DPB images.</doc>
           <parameter name="usage" transfer-ownership="none">
             <type name="Vulkan.ImageUsageFlags" c:type="VkImageUsageFlags"/>
           </parameter>
+          <parameter name="initial_layout" transfer-ownership="none">
+            <type name="Vulkan.ImageLayout" c:type="VkImageLayout"/>
+          </parameter>
           <parameter name="params" transfer-ownership="none">
             <type name="Gst.AllocationParams" c:type="GstAllocationParams*"/>
           </parameter>

subprojects/gst-plugins-bad/ext/vulkan/vkh264dec.c

@@ -370,7 +370,8 @@ gst_vulkan_h264_decoder_decide_allocation (GstVideoDecoder * decoder,
   gst_buffer_pool_config_set_params (config, new_caps, size, min, max);
 
   gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
-      VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+      VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR,
+      VK_ACCESS_TRANSFER_WRITE_BIT);
   gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps);
 
   gst_caps_unref (profile_caps);

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkdecoder.c

@@ -588,7 +588,8 @@ gst_vulkan_decoder_create_dpb_pool (GstVulkanDecoder * self, GstCaps * caps)
   gst_buffer_pool_config_set_params (config, caps, 1024, min_buffers,
       max_buffers);
   gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
-      VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+      VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
+      VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT);
 
   if (self->layered_dpb) {
     gst_structure_set (config, "num-layers", G_TYPE_UINT,

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.c

@@ -43,6 +43,8 @@ struct _GstVulkanImageBufferPoolPrivate
   GstVideoInfo v_info;
   VkImageUsageFlags usage;
   VkMemoryPropertyFlags mem_props;
+  VkImageLayout initial_layout;
+  guint64 initial_access;
   VkFormat vk_fmts[GST_VIDEO_MAX_PLANES];
   int n_imgs;
   guint32 n_layers;
@@ -69,6 +71,9 @@ G_DEFINE_TYPE_WITH_CODE (GstVulkanImageBufferPool, gst_vulkan_image_buffer_pool,
  * @config: the #GstStructure with the pool's configuration.
  * @usage: The Vulkan image usage flags.
  * @mem_properties: Vulkan memory property flags.
+ * @initial_layout: Initial Vulkan image layout.
+ * @initial_access: Access flags for the layout transition if @initial_layout is
+ * not VK_IMAGE_LAYOUT_UNDEFINED or VK_IMAGE_LAYOUT_PREINITIALIZED.
  *
  * Sets the @usage and @mem_properties of the images to setup.
  *
@@ -76,11 +81,13 @@ G_DEFINE_TYPE_WITH_CODE (GstVulkanImageBufferPool, gst_vulkan_image_buffer_pool,
  */
 void
 gst_vulkan_image_buffer_pool_config_set_allocation_params (GstStructure *
-    config, VkImageUsageFlags usage, VkMemoryPropertyFlags mem_properties)
+    config, VkImageUsageFlags usage, VkMemoryPropertyFlags mem_properties,
+    VkImageLayout initial_layout, guint64 initial_access)
 {
   /* assumption: G_TYPE_UINT is compatible with uint32_t (VkFlags) */
   gst_structure_set (config, "usage", G_TYPE_UINT, usage, "memory-properties",
-      G_TYPE_UINT, mem_properties, NULL);
+      G_TYPE_UINT, mem_properties, "initial-layout", G_TYPE_UINT,
+      initial_layout, "initial-access", G_TYPE_UINT64, initial_access, NULL);
 }
 
 /**
@@ -105,6 +112,7 @@ gst_vulkan_image_buffer_pool_config_set_decode_caps (GstStructure * config,
 static inline gboolean
 gst_vulkan_image_buffer_pool_config_get_allocation_params (GstStructure *
     config, VkImageUsageFlags * usage, VkMemoryPropertyFlags * mem_props,
+    VkImageLayout * initial_layout, guint64 * initial_access,
     guint32 * n_layers, GstCaps ** decode_caps)
 {
   if (!gst_structure_get_uint (config, "usage", usage)) {
@@ -116,6 +124,12 @@ gst_vulkan_image_buffer_pool_config_get_allocation_params (GstStructure *
   if (!gst_structure_get_uint (config, "memory-properties", mem_props))
     *mem_props = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
 
+  if (!gst_structure_get_uint (config, "initial-layout", initial_layout))
+    *initial_layout = VK_IMAGE_LAYOUT_UNDEFINED;
+
+  if (!gst_structure_get_uint64 (config, "initial-access", initial_access))
+    *initial_access = 0;
+
   if (!gst_structure_get_uint (config, "num-layers", n_layers))
     *n_layers = 1;
 
@@ -161,7 +175,8 @@ gst_vulkan_image_buffer_pool_set_config (GstBufferPool * pool,
       GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY);
 
   gst_vulkan_image_buffer_pool_config_get_allocation_params (config,
-      &priv->usage, &priv->mem_props, &priv->n_layers, &decode_caps);
+      &priv->usage, &priv->mem_props, &priv->initial_layout,
+      &priv->initial_access, &priv->n_layers, &decode_caps);
 
   priv->has_profile = FALSE;
 #if GST_VULKAN_HAVE_VIDEO_EXTENSIONS
@@ -224,7 +239,9 @@ gst_vulkan_image_buffer_pool_set_config (GstBufferPool * pool,
     .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
     .queueFamilyIndexCount = 0,
     .pQueueFamilyIndices = NULL,
-    .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+    .initialLayout = priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED
+                     ? VK_IMAGE_LAYOUT_PREINITIALIZED
+                     : VK_IMAGE_LAYOUT_UNDEFINED,
   };
   /* *INDENT-ON* */
   priv->v_info.size = 0;
@@ -324,10 +341,12 @@ prepare_buffer (GstVulkanImageBufferPool * vk_pool, GstBuffer * buffer)
 #if defined(VK_KHR_synchronization2)
   GstVulkanImageBufferPoolPrivate *priv = GET_PRIV (vk_pool);
   GArray *barriers = NULL;
-  VkImageLayout new_layout;
-  guint64 new_access;
   GError *error = NULL;
 
+  if (priv->initial_layout == VK_IMAGE_LAYOUT_UNDEFINED ||
+      priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED)
+    return TRUE;
+
   if (!priv->exec) {
     GstVulkanCommandPool *cmd_pool;
     GstVulkanQueue *queue = NULL;
@@ -349,31 +368,12 @@ prepare_buffer (GstVulkanImageBufferPool * vk_pool, GstBuffer * buffer)
           VK_PIPELINE_STAGE_NONE_KHR, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT))
     return FALSE;
 
-#if GST_VULKAN_HAVE_VIDEO_EXTENSIONS
-  if ((priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) != 0 &&
-      (priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR) == 0) {
-    new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
-    new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
-  } else if ((priv->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR) != 0) {
-    new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
-    new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
-#ifdef VK_ENABLE_BETA_EXTENSIONS
-  } else if ((priv->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) != 0) {
-    new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
-    new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
-#endif
-  } else
-#endif
-  {
-    new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
-    new_access = VK_ACCESS_MEMORY_WRITE_BIT;
-  }
-
   if (!gst_vulkan_operation_begin (priv->exec, &error))
     goto error;
 
   if (!gst_vulkan_operation_add_frame_barrier (priv->exec, buffer,
-          VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, new_access, new_layout, NULL))
+          VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, priv->initial_access,
+          priv->initial_layout, NULL))
     goto error;
 
   barriers = gst_vulkan_operation_retrieve_image_barriers (priv->exec);
@@ -442,7 +442,9 @@ gst_vulkan_image_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
     .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
     .queueFamilyIndexCount = 0,
     .pQueueFamilyIndices = NULL,
-    .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+    .initialLayout = priv->initial_layout == VK_IMAGE_LAYOUT_PREINITIALIZED
+                     ? VK_IMAGE_LAYOUT_PREINITIALIZED
+                     : VK_IMAGE_LAYOUT_UNDEFINED,
   };
   /* *INDENT-ON* */
 

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagebufferpool.h

@@ -84,7 +84,9 @@ GST_VULKAN_API
 void            gst_vulkan_image_buffer_pool_config_set_allocation_params
                                                         (GstStructure * config,
                                                          VkImageUsageFlags usage,
-                                                         VkMemoryPropertyFlags mem_properties);
+                                                         VkMemoryPropertyFlags mem_properties,
+                                                         VkImageLayout initial_layout,
+                                                         guint64 initial_access);
 
 GST_VULKAN_API
 void            gst_vulkan_image_buffer_pool_config_set_decode_caps

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagememory.c

@@ -71,8 +71,9 @@ _create_info_from_args (VkImageCreateInfo * info, VkFormat format, gsize width,
 gboolean
 gst_vulkan_image_memory_init (GstVulkanImageMemory * mem,
     GstAllocator * allocator, GstMemory * parent, GstVulkanDevice * device,
-    VkFormat format, VkImageUsageFlags usage, GstAllocationParams * params,
+    VkFormat format, VkImageUsageFlags usage, VkImageLayout layout,
-    gsize size, gpointer user_data, GDestroyNotify notify)
+    GstAllocationParams * params, gsize size, gpointer user_data,
+    GDestroyNotify notify)
 {
   gsize align = gst_memory_alignment, offset = 0, maxsize = size;
   GstMemoryFlags flags = 0;
@@ -93,7 +94,7 @@ gst_vulkan_image_memory_init (GstVulkanImageMemory * mem,
   mem->barrier.parent.access_flags = 0;
   mem->barrier.parent.semaphore = VK_NULL_HANDLE;
   mem->barrier.parent.semaphore_value = 0;
-  mem->barrier.image_layout = VK_IMAGE_LAYOUT_UNDEFINED;
+  mem->barrier.image_layout = layout;
   /* *INDENT-OFF* */
   mem->barrier.subresource_range = (VkImageSubresourceRange) {
           .aspectMask = gst_vulkan_format_get_aspect (format),
@@ -158,8 +159,8 @@ _vk_image_mem_new_alloc_with_image_info (GstAllocator * allocator,
   vkGetImageMemoryRequirements (device->device, image, &mem->requirements);
 
   gst_vulkan_image_memory_init (mem, allocator, parent, device,
-      image_info->format, image_info->usage, &params, mem->requirements.size,
+      image_info->format, image_info->usage, image_info->initialLayout, &params,
-      user_data, notify);
+      mem->requirements.size, user_data, notify);
   mem->create_info = *image_info;
   /* XXX: to avoid handling pNext lifetime  */
   mem->create_info.pNext = NULL;
@@ -270,7 +271,8 @@ _vk_image_mem_new_wrapped (GstAllocator * allocator, GstMemory * parent,
   params.align = mem->requirements.alignment - 1;
   params.flags = GST_MEMORY_FLAG_NOT_MAPPABLE;
   gst_vulkan_image_memory_init (mem, allocator, parent, device, format, usage,
-      &params, mem->requirements.size, user_data, notify);
+      VK_IMAGE_LAYOUT_UNDEFINED, &params, mem->requirements.size, user_data,
+      notify);
   mem->wrapped = TRUE;
 
   if (!_create_info_from_args (&mem->create_info, format, width, height, tiling,

subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkimagememory.h

@@ -172,6 +172,7 @@ gboolean        gst_vulkan_image_memory_init            (GstVulkanImageMemory *
                                                          GstVulkanDevice * device,
                                                          VkFormat format,
                                                          VkImageUsageFlags usage,
+                                                         VkImageLayout initial_layout,
                                                          GstAllocationParams * params,
                                                          gsize size,
                                                          gpointer user_data,

subprojects/gst-plugins-bad/sys/applemedia/iosurfacevulkanmemory.c

@@ -195,7 +195,8 @@ _io_surface_vulkan_memory_new (GstVulkanDevice * device, IOSurfaceRef surface,
 
   gst_vulkan_image_memory_init (&mem->vulkan_mem,
       _io_surface_vulkan_memory_allocator, NULL, device, vk_format, usage,
-      &params, mem->vulkan_mem.requirements.size, user_data, notify);
+      VK_IMAGE_LAYOUT_UNDEFINED, &params, mem->vulkan_mem.requirements.size,
+      user_data, notify);
   mem->vulkan_mem.create_info = image_info;
   mem->vulkan_mem.image = image;
   mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL;

subprojects/gst-plugins-bad/tests/check/libs/vkimagebufferpool.c

@@ -60,7 +60,7 @@ setup_queue (guint expected_flags)
 
 static GstBufferPool *
 create_buffer_pool (const char *format, VkImageUsageFlags usage,
-    GstCaps * dec_caps)
+    VkImageLayout initial_layout, guint64 initial_access, GstCaps * dec_caps)
 {
   GstCaps *caps;
   GstBufferPool *pool;
@@ -79,7 +79,8 @@ create_buffer_pool (const char *format, VkImageUsageFlags usage,
   gst_caps_unref (caps);
 
   gst_vulkan_image_buffer_pool_config_set_allocation_params (config,
-      usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+      usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, initial_layout,
+      initial_access);
 
   if (dec_caps)
     gst_vulkan_image_buffer_pool_config_set_decode_caps (config, dec_caps);
@@ -97,7 +98,7 @@ GST_START_TEST (test_image)
   GstBuffer *buffer = NULL;
 
   setup_queue (VK_QUEUE_COMPUTE_BIT);
-  pool = create_buffer_pool ("NV12", 0, NULL);
+  pool = create_buffer_pool ("NV12", 0, VK_IMAGE_LAYOUT_UNDEFINED, 0, NULL);
 
   ret = gst_buffer_pool_acquire_buffer (pool, &buffer, NULL);
   fail_unless (ret == GST_FLOW_OK);
@@ -191,6 +192,7 @@ GST_START_TEST (test_decoding_image)
   fail_unless (dec_caps);
 
   pool = create_buffer_pool ("NV12", VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR,
+      VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR, VK_ACCESS_TRANSFER_WRITE_BIT,
       dec_caps);
 
   gst_caps_unref (dec_caps);

subprojects/gst-plugins-bad/tests/check/libs/vkvideodecode.c

@@ -305,7 +305,8 @@ GST_START_TEST (test_decoder)
     gst_caps_unref (caps);
 
     gst_vulkan_image_buffer_pool_config_set_allocation_params (config, usage,
-        VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
+        VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
+        VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR, VK_ACCESS_TRANSFER_WRITE_BIT);
     gst_vulkan_image_buffer_pool_config_set_decode_caps (config, profile_caps);
 
     gst_caps_unref (profile_caps);