vulkan: provide a custom VK_DEFINE_NON_DISPATCHABLE_HANDLE

If the application did not define one yet, define our own
VK_DEFINE_NON_DISPATCHABLE_HANDLE that is independent of the
architecture.

Vulkan, by default, provides a define that depends on the architecture,
which causes the symbol type to be different. This causes an
architecture dependent .gir file, which then causes multilib
installation problems because the .gir files can't be shared.

Make it possible to override the format specifier and provide
a default one that is compatible with the default non dispatchable
handle.

Return VK_NULL_HANDLE from functions that return a non-dispatchable
handle.

Fixes #1566

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2130>

gst-libs/gst/vulkan/android/gstvkwindow_android.c

@@ -163,14 +163,14 @@ gst_vulkan_window_android_get_surface (GstVulkanWindow * window,
   if (!window_android->CreateAndroidSurface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateAndroidSurfaceKHR\" function pointer");
-    return 0;
+    return VK_NULL_HANDLE;
   }
 
   err =
       window_android->CreateAndroidSurface (window->display->instance->instance,
       &info, NULL, &ret);
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateAndroidSurfaceKHR") < 0)
-    return 0;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

gst-libs/gst/vulkan/cocoa/gstvkwindow_cocoa.m

@@ -226,14 +226,14 @@ gst_vulkan_window_cocoa_get_surface (GstVulkanWindow * window, GError ** error)
   if (!window_cocoa->CreateMacOSSurface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateMacOSSurfaceMVK\" function pointer");
-    return NULL;
+    return VK_NULL_HANDLE;
   }
 
   err =
       window_cocoa->CreateMacOSSurface (window->display->instance->instance, &info,
       NULL, &ret);
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateMacOSSurfaceMVK") < 0)
-    return NULL;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

gst-libs/gst/vulkan/gstvkapi.h

@@ -31,6 +31,24 @@
 #include <gst/vulkan/vulkan_fwd.h>
 #include <gst/vulkan/vulkan-enumtypes.h>
 
+/**
+ * VK_DEFINE_NON_DISPATCHABLE_HANDLE:
+ *
+ * Allow applications to override the VK_DEFINE_NON_DISPATCHABLE_HANDLE
+ * but provide our own version otherwise. The default vulkan define
+ * provides a different symbol type depending on the architecture and
+ * this causes multilib problems because the generated .gir files are
+ * different.
+ *
+ * Also make sure to provide a suitable GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT
+ * implementation when redefining VK_DEFINE_NON_DISPATCHABLE_HANDLE.
+ *
+ * Since: 1.20
+ */
+#if !defined(VK_DEFINE_NON_DISPATCHABLE_HANDLE)
+#define VK_DEFINE_NON_DISPATCHABLE_HANDLE(object) typedef uint64_t object;
+#endif
+
 #include <vulkan/vulkan_core.h>
 
 #endif /* __GST_VULKAN_API_H__ */

gst-libs/gst/vulkan/gstvkhandle.h

@@ -54,11 +54,12 @@ VK_DEFINE_NON_DISPATCHABLE_HANDLE(GstVulkanHandleTypedef)
  *
  * The printf format specifier for raw Vulkan non dispatchable handles.
  *
+ * When redefining VK_DEFINE_NON_DISPATCHABLE_HANDLE, also make sure
+ * to redefine a suitable printf format specifier.
+ *
  * Since: 1.18
  */
-#if GLIB_SIZEOF_VOID_P == 8
-# define GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT "p"
-#else
+#if !defined(GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT)
 # define GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT G_GUINT64_FORMAT
 #endif
 

gst-libs/gst/vulkan/ios/gstvkwindow_ios.m

@@ -203,7 +203,7 @@ gst_vulkan_window_ios_get_surface (GstVulkanWindow * window, GError ** error)
     g_set_error_literal (error, GST_VULKAN_ERROR,
         VK_ERROR_INITIALIZATION_FAILED,
         "No layer to retrieve surface for. Has create_window() been called?");
-    return 0;
+    return VK_NULL_HANDLE;
   }
 
   info.sType = VK_STRUCTURE_TYPE_IOS_SURFACE_CREATE_INFO_MVK;
@@ -218,14 +218,14 @@ gst_vulkan_window_ios_get_surface (GstVulkanWindow * window, GError ** error)
   if (!window_ios->CreateIOSSurface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateIOSSurfaceMVK\" function pointer");
-    return 0;
+    return VK_NULL_HANDLE;
   }
 
   err =
       window_ios->CreateIOSSurface (window->display->instance->instance, &info,
       NULL, &ret);
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateIOSSurfaceMVK") < 0)
-    return 0;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

gst-libs/gst/vulkan/wayland/gstvkwindow_wayland.c

@@ -268,14 +268,14 @@ gst_vulkan_window_wayland_get_surface (GstVulkanWindow * window,
   if (!window_wl->CreateWaylandSurface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateWaylandSurfaceKHR\" function pointer");
-    return NULL;
+    return VK_NULL_HANDLE;
   }
 
   err =
       window_wl->CreateWaylandSurface (window->display->instance->instance,
       &info, NULL, &ret);
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateWaylandSurfaceKHR") < 0)
-    return NULL;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

gst-libs/gst/vulkan/win32/gstvkwindow_win32.c

@@ -393,7 +393,7 @@ gst_vulkan_window_win32_get_surface (GstVulkanWindow * window, GError ** error)
   if (!window_win32->CreateWin32Surface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateWin32SurfaceKHR\" function pointer");
-    return NULL;
+    return VK_NULL_HANDLE;
   }
 
   err =
@@ -401,7 +401,7 @@ gst_vulkan_window_win32_get_surface (GstVulkanWindow * window, GError ** error)
       &info, NULL, &ret);
 
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateWin32SurfaceKHR") < 0)
-    return NULL;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

gst-libs/gst/vulkan/xcb/gstvkwindow_xcb.c

@@ -283,14 +283,14 @@ gst_vulkan_window_xcb_get_surface (GstVulkanWindow * window, GError ** error)
   if (!window_xcb->CreateXcbSurface) {
     g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
         "Could not retrieve \"vkCreateXcbSurfaceKHR\" function pointer");
-    return NULL;
+    return VK_NULL_HANDLE;
   }
 
   err =
       window_xcb->CreateXcbSurface (window->display->instance->instance, &info,
       NULL, &ret);
   if (gst_vulkan_error_to_g_error (err, error, "vkCreateXcbSurfaceKHR") < 0)
-    return NULL;
+    return VK_NULL_HANDLE;
 
   return ret;
 }

sys/applemedia/videotexturecache-vulkan.mm

@@ -292,8 +292,8 @@ gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
     texture_data->texture = (__bridge_retained gpointer) texture;
 
     VkResult err = vkSetMTLTextureMVK (memory->vulkan_mem.image, texture);
-    GST_DEBUG ("bound texture %p to image %p: 0x%x", texture, memory->vulkan_mem.image,
-        err);
+    GST_DEBUG ("bound texture %p to image %"GST_VULKAN_NON_DISPATCHABLE_HANDLE_FORMAT": 0x%x",
+               texture, memory->vulkan_mem.image, err);
 
     vk_mem->user_data = texture_data;
     vk_mem->notify = (GDestroyNotify) free_texture_wrapper;