vulkan: Add iOS window implementation

ext/vulkan/ios/vkdisplay_ios.h

@@ -0,0 +1,67 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <ystreet00@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_VULKAN_DISPLAY_IOS_H__
+#define __GST_VULKAN_DISPLAY_IOS_H__
+
+#include <gst/gst.h>
+
+#include <vk.h>
+#ifndef VK_USE_PLATFORM_IOS_MVK
+#error "VK_USE_PLATFORM_IOS_MVK not defined before including this header"
+#error "Either include vkapi.h or define VK_USE_PLATFORM_IOS_MVK before including this header"
+#endif
+#include <vulkan/vulkan.h>
+
+G_BEGIN_DECLS
+
+GType gst_vulkan_display_ios_get_type (void);
+
+#define GST_TYPE_VULKAN_DISPLAY_IOS             (gst_vulkan_display_ios_get_type())
+#define GST_VULKAN_DISPLAY_IOS(obj)             (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VULKAN_DISPLAY_IOS,GstVulkanDisplayIos))
+#define GST_VULKAN_DISPLAY_IOS_CLASS(klass)     (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_VULKAN_DISPLAY_IOS,GstVulkanDisplayIosClass))
+#define GST_IS_VULKAN_DISPLAY_IOS(obj)          (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VULKAN_DISPLAY_IOS))
+#define GST_IS_VULKAN_DISPLAY_IOS_CLASS(klass)  (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_VULKAN_DISPLAY_IOS))
+#define GST_VULKAN_DISPLAY_IOS_CAST(obj)        ((GstVulkanDisplayIos*)(obj))
+
+typedef struct _GstVulkanDisplayIos GstVulkanDisplayIos;
+typedef struct _GstVulkanDisplayIosClass GstVulkanDisplayIosClass;
+
+/**
+ * GstVulkanDisplayIos:
+ *
+ * the contents of a #GstVulkanDisplayIos are private and should only be accessed
+ * through the provided API
+ */
+struct _GstVulkanDisplayIos
+{
+  GstVulkanDisplay          parent;
+};
+
+struct _GstVulkanDisplayIosClass
+{
+  GstVulkanDisplayClass object_class;
+};
+
+GstVulkanDisplayIos * gst_vulkan_display_ios_new                    (void);
+
+G_END_DECLS
+
+#endif /* __GST_VULKAN_DISPLAY_IOS_H__ */

ext/vulkan/ios/vkdisplay_ios.m

@@ -0,0 +1,85 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <UIKit/UIKit.h>
+
+#include "vkdisplay_ios.h"
+
+#define GST_CAT_DEFAULT gst_vulkan_display_debug
+GST_DEBUG_CATEGORY_STATIC (gst_vulkan_display_debug);
+
+G_DEFINE_TYPE (GstVulkanDisplayIos, gst_vulkan_display_ios,
+    GST_TYPE_VULKAN_DISPLAY);
+
+static void gst_vulkan_display_ios_finalize (GObject * object);
+static gpointer gst_vulkan_display_ios_get_handle (GstVulkanDisplay * display);
+
+static void
+gst_vulkan_display_ios_class_init (GstVulkanDisplayIosClass * klass)
+{
+  GST_VULKAN_DISPLAY_CLASS (klass)->get_handle =
+      GST_DEBUG_FUNCPTR (gst_vulkan_display_ios_get_handle);
+
+  G_OBJECT_CLASS (klass)->finalize = gst_vulkan_display_ios_finalize;
+}
+
+static void
+gst_vulkan_display_ios_init (GstVulkanDisplayIos * display_ios)
+{
+  GstVulkanDisplay *display = (GstVulkanDisplay *) display_ios;
+
+  display->type = GST_VULKAN_DISPLAY_TYPE_IOS;
+}
+
+static void
+gst_vulkan_display_ios_finalize (GObject * object)
+{
+  G_OBJECT_CLASS (gst_vulkan_display_ios_parent_class)->finalize (object);
+}
+
+/**
+ * gst_vulkan_display_ios_new:
+ *
+ * Create a new #GstVulkanDisplayIos.
+ *
+ * Returns: (transfer full): a new #GstVulkanDisplayIos or %NULL
+ */
+GstVulkanDisplayIos *
+gst_vulkan_display_ios_new (void)
+{
+  GstVulkanDisplayIos *ret;
+
+  GST_DEBUG_CATEGORY_GET (gst_vulkan_display_debug, "vulkandisplay");
+
+  ret = g_object_new (GST_TYPE_VULKAN_DISPLAY_IOS, NULL);
+  gst_object_ref_sink (ret);
+
+  return ret;
+}
+
+static gpointer
+gst_vulkan_display_ios_get_handle (GstVulkanDisplay * display)
+{
+  return (gpointer) (__bridge gpointer) [UIApplication sharedApplication];
+}

ext/vulkan/ios/vkios_utils.h

@@ -0,0 +1,40 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __VULKAN_IOS_UTILS_H__
+#define __VULKAN_IOS_UTILS_H__
+
+#include <gst/gst.h>
+#include <UIKit/UIKit.h>
+
+#include "vkwindow_ios.h"
+
+G_BEGIN_DECLS
+
+@interface GstVulkanUIView : UIView
+@end
+
+typedef void (*GstVulkanWindowFunc) (gpointer data);
+
+void _invoke_on_main (GstVulkanWindowFunc func, gpointer data, GDestroyNotify notify);
+
+G_END_DECLS
+
+#endif

ext/vulkan/ios/vkwindow_ios.h

@@ -0,0 +1,82 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <ystreet00@gmail.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_VULKAN_WINDOW_IOS_H__
+#define __GST_VULKAN_WINDOW_IOS_H__
+
+#include <vk.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VULKAN_WINDOW_IOS         (gst_vulkan_window_ios_get_type())
+#define GST_VULKAN_WINDOW_IOS(o)           (G_TYPE_CHECK_INSTANCE_CAST((o), GST_TYPE_VULKAN_WINDOW_IOS, GstVulkanWindowIos))
+#define GST_VULKAN_WINDOW_IOS_CLASS(k)     (G_TYPE_CHECK_CLASS((k), GST_TYPE_VULKAN_WINDOW_IOS, GstVulkanWindowIosClass))
+#define GST_IS_VULKAN_WINDOW_IOS(o)        (G_TYPE_CHECK_INSTANCE_TYPE((o), GST_TYPE_VULKAN_WINDOW_IOS))
+#define GST_IS_VULKAN_WINDOW_IOS_CLASS(k)  (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_VULKAN_WINDOW_IOS))
+#define GST_VULKAN_WINDOW_IOS_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_VULKAN_WINDOW_IOS, GstVulkanWindowIosClass))
+
+typedef struct _GstVulkanWindowIos        GstVulkanWindowIos;
+typedef struct _GstVulkanWindowIosPrivate GstVulkanWindowIosPrivate;
+typedef struct _GstVulkanWindowIosClass   GstVulkanWindowIosClass;
+
+/**
+ * GstVulkanWindowIos:
+ *
+ * Opaque #GstVulkanWindowIos object
+ */
+struct _GstVulkanWindowIos
+{
+  /*< private >*/
+  GstVulkanWindow parent;
+
+  gpointer view;
+
+  gint          visible :1;
+
+  PFN_vkCreateIOSSurfaceMVK CreateIOSSurface;
+
+  /*< private >*/
+  GstVulkanWindowIosPrivate *priv;
+  
+  gpointer _reserved[GST_PADDING];
+};
+
+/**
+ * GstVulkanWindowIosClass:
+ *
+ * Opaque #GstVulkanWindowIosClass object
+ */
+struct _GstVulkanWindowIosClass {
+  /*< private >*/
+  GstVulkanWindowClass parent_class;
+
+  /*< private >*/
+  gpointer _reserved[GST_PADDING_LARGE];
+};
+
+GType gst_vulkan_window_ios_get_type     (void);
+
+GstVulkanWindowIos * gst_vulkan_window_ios_new (GstVulkanDisplay * display);
+
+gboolean gst_vulkan_window_ios_create_window (GstVulkanWindowIos * window_ios);
+
+G_END_DECLS
+
+#endif /* __GST_VULKAN_WINDOW_IOS_H__ */

ext/vulkan/ios/vkwindow_ios.m

@@ -0,0 +1,265 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <QuartzCore/QuartzCore.h>
+
+#include <gst/gst.h>
+
+#include "vkwindow_ios.h"
+#include "vkdisplay_ios.h"
+
+#include "vkios_utils.h"
+
+#define GST_VULKAN_WINDOW_IOS_GET_PRIVATE(o)  \
+  (G_TYPE_INSTANCE_GET_PRIVATE((o), GST_TYPE_VULKAN_WINDOW_IOS, GstVulkanWindowIosPrivate))
+
+#define GST_CAT_DEFAULT gst_vulkan_window_ios_debug
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT);
+
+static void
+_init_debug (void)
+{
+  static volatile gsize _init = 0;
+
+  if (g_once_init_enter (&_init)) {
+    GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "vulkanwindowios", 0,
+        "Vulkan iOS Window");
+    g_once_init_leave (&_init, 1);
+  }
+}
+
+enum
+{
+  PROP_0,
+};
+
+struct _GstVulkanWindowIosPrivate
+{
+  gpointer internal_view;
+  gpointer external_view;
+
+  gint preferred_width;
+  gint preferred_height;
+};
+
+#define gst_vulkan_window_ios_parent_class parent_class
+G_DEFINE_TYPE_WITH_CODE (GstVulkanWindowIos, gst_vulkan_window_ios,
+    GST_TYPE_VULKAN_WINDOW, G_ADD_PRIVATE (GstVulkanWindowIos) _init_debug ());
+
+static VkSurfaceKHR gst_vulkan_window_ios_get_surface (GstVulkanWindow * window,
+    GError ** error);
+static gboolean gst_vulkan_window_ios_get_presentation_support (GstVulkanWindow
+    * window, GstVulkanDevice * device, guint32 queue_family_idx);
+static gboolean gst_vulkan_window_ios_open (GstVulkanWindow * window,
+    GError ** error);
+static void gst_vulkan_window_ios_close (GstVulkanWindow * window);
+static void gst_vulkan_window_ios_set_window_handle (GstVulkanWindow * window,
+    guintptr window_handle);
+
+static void
+gst_vulkan_window_ios_finalize (GObject * object)
+{
+  G_OBJECT_CLASS (parent_class)->finalize (object);
+}
+
+static void
+gst_vulkan_window_ios_class_init (GstVulkanWindowIosClass * klass)
+{
+  GObjectClass *obj_class = G_OBJECT_CLASS (klass);
+  GstVulkanWindowClass *window_class = (GstVulkanWindowClass *) klass;
+
+  obj_class->finalize = gst_vulkan_window_ios_finalize;
+
+  window_class->open = GST_DEBUG_FUNCPTR (gst_vulkan_window_ios_open);
+  window_class->close = GST_DEBUG_FUNCPTR (gst_vulkan_window_ios_close);
+  window_class->get_surface = gst_vulkan_window_ios_get_surface;
+  window_class->get_presentation_support =
+      gst_vulkan_window_ios_get_presentation_support;
+  window_class->set_window_handle =
+      gst_vulkan_window_ios_set_window_handle;
+}
+
+static void
+gst_vulkan_window_ios_init (GstVulkanWindowIos * window)
+{
+  window->priv = gst_vulkan_window_ios_get_instance_private (window);
+
+  window->priv->preferred_width = 320;
+  window->priv->preferred_height = 240;
+}
+
+/* Must be called in the gl thread */
+GstVulkanWindowIos *
+gst_vulkan_window_ios_new (GstVulkanDisplay * display)
+{
+  GstVulkanWindowIos *window;
+
+  _init_debug ();
+
+  if ((gst_vulkan_display_get_handle_type (display) &
+          GST_VULKAN_DISPLAY_TYPE_IOS)
+      == GST_VULKAN_DISPLAY_TYPE_NONE) {
+    GST_INFO ("Wrong display type %u for this window type %u", display->type,
+        GST_VULKAN_DISPLAY_TYPE_IOS);
+    return NULL;
+  }
+
+  window = g_object_new (GST_TYPE_VULKAN_WINDOW_IOS, NULL);
+  gst_object_ref_sink (window);
+
+  return window;
+}
+
+static void
+_create_window (GstVulkanWindowIos * window_ios)
+{
+  GstVulkanWindowIosPrivate *priv = window_ios->priv;
+  CGRect rect = CGRectMake (0, 0, priv->preferred_width, priv->preferred_height);
+  UIView *external_view = (__bridge UIView *) priv->external_view;
+  GstVulkanUIView *view;
+
+  view = [[GstVulkanUIView alloc] initWithFrame:rect];
+  view.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight;
+
+  priv->internal_view = (__bridge_retained gpointer)view;
+  [external_view addSubview:view];
+}
+
+gboolean
+gst_vulkan_window_ios_create_window (GstVulkanWindowIos * window_ios)
+{
+  if (!window_ios->priv->external_view) {
+    GST_WARNING_OBJECT(window_ios, "No external UIView provided");
+    return FALSE;
+  }
+
+  _invoke_on_main ((GstVulkanWindowFunc) _create_window,
+      gst_object_ref (window_ios), gst_object_unref);
+
+  return TRUE;
+}
+
+static VkSurfaceKHR
+gst_vulkan_window_ios_get_surface (GstVulkanWindow * window, GError ** error)
+{
+  GstVulkanWindowIos *window_ios = GST_VULKAN_WINDOW_IOS (window);
+  VkIOSSurfaceCreateInfoMVK info = { 0, };
+  VkSurfaceKHR ret;
+  VkResult err;
+
+  info.sType = VK_STRUCTURE_TYPE_IOS_SURFACE_CREATE_INFO_MVK;
+  info.pNext = NULL;
+  info.flags = 0;
+  info.pView = window_ios->priv->internal_view;
+
+  if (!window_ios->CreateIOSSurface)
+    window_ios->CreateIOSSurface =
+        gst_vulkan_instance_get_proc_address (window->display->instance,
+        "vkCreateIOSSurfaceMVK");
+  if (!window_ios->CreateIOSSurface) {
+    g_set_error_literal (error, GST_VULKAN_ERROR, VK_ERROR_FEATURE_NOT_PRESENT,
+        "Could not retrieve \"vkCreateIOSSurfaceMVK\" function pointer");
+    return 0;
+  }
+
+  err =
+      window_ios->CreateIOSSurface (window->display->instance->instance, &info,
+      NULL, &ret);
+  if (gst_vulkan_error_to_g_error (err, error, "vkCreateIOSSurfaceMVK") < 0)
+    return 0;
+
+  return ret;
+}
+
+static gboolean
+gst_vulkan_window_ios_get_presentation_support (GstVulkanWindow * window,
+    GstVulkanDevice * device, guint32 queue_family_idx)
+{
+  return TRUE;
+}
+
+static gboolean
+gst_vulkan_window_ios_open (GstVulkanWindow * window, GError ** error)
+{
+  GstVulkanWindowIos *window_ios = GST_VULKAN_WINDOW_IOS (window);
+
+  if (!GST_VULKAN_WINDOW_CLASS (parent_class)->open (window, error))
+    return FALSE;
+
+  return gst_vulkan_window_ios_create_window (window_ios);
+}
+
+static void
+gst_vulkan_window_ios_close (GstVulkanWindow * window)
+{
+  GstVulkanWindowIos *window_ios = GST_VULKAN_WINDOW_IOS (window);
+
+  CFBridgingRelease (window_ios->priv->internal_view);
+  window_ios->priv->internal_view = NULL;
+
+  GST_VULKAN_WINDOW_CLASS (parent_class)->close (window);
+}
+
+static void
+gst_vulkan_window_ios_set_window_handle (GstVulkanWindow * window,
+    guintptr window_handle)
+{
+  GstVulkanWindowIos *window_ios = GST_VULKAN_WINDOW_IOS (window);
+  gpointer view = (gpointer) window_handle;
+
+  g_return_if_fail (view != NULL);
+
+  if (window_ios->priv->external_view && window_ios->priv->external_view != view) {
+    GST_FIXME_OBJECT (window_ios, "View changes are not implemented");
+    return;
+  }
+
+  window_ios->priv->external_view = view;
+}
+
+@implementation GstVulkanUIView
+
++(Class) layerClass
+{
+  return [CAMetalLayer class];
+}
+
+@end
+
+void
+_invoke_on_main (GstVulkanWindowFunc func, gpointer data, GDestroyNotify notify)
+{
+  if ([NSThread isMainThread]) {
+    func (data);
+    if (notify)
+      notify (data);
+  } else {
+    dispatch_async (dispatch_get_main_queue (), ^{
+      func (data);
+      if (notify)
+        notify (data);
+    });
+  }
+}
+

ext/vulkan/meson.build

@@ -25,7 +25,11 @@ if get_option('vulkan').disabled()
   subdir_done()
 endif
 
-vulkan_dep = cc.find_library('vulkan', required : get_option('vulkan'))
+if host_system == 'ios'
+  vulkan_dep = cc.find_library('MoltenVK', required : get_option('vulkan'))
+else
+  vulkan_dep = cc.find_library('vulkan', required : get_option('vulkan'))
+endif
 has_vulkan_header = cc.has_header('vulkan/vulkan.h')
 if not has_vulkan_header and get_option('vulkan').enabled()
   error('vulkan plugin enabled, but vulkan.h not found')
@@ -69,25 +73,43 @@ if vulkan_dep.found() and has_vulkan_header
     endif
 
     vulkan_objc_args += ['-fobjc-arc']
-  endif
 
-  if host_system == 'darwin'
     foundation_dep = dependency('appleframeworks', modules : ['Foundation'], required : get_option('vulkan'))
     quartzcore_dep = dependency('appleframeworks', modules : ['QuartzCore'], required : get_option('vulkan'))
     corefoundation_dep = dependency('appleframeworks', modules : ['CoreFoundation'], required : get_option('vulkan'))
+    if foundation_dep.found() and quartzcore_dep.found() and corefoundation_dep.found()
+      optional_deps += [foundation_dep, corefoundation_dep, quartzcore_dep]
+    endif
+  endif
+
+  if host_system == 'darwin'
     cocoa_dep = dependency('appleframeworks', modules : ['Cocoa'], required : get_option('vulkan'))
 
-    if foundation_dep.found() and quartzcore_dep.found() and corefoundation_dep.found() and cocoa_dep.found()
+    if cocoa_dep.found()
       vulkan_sources += [
         'cocoa/vkdisplay_cocoa.m',
         'cocoa/vkwindow_cocoa.m',
       ]
-      optional_deps += [foundation_dep, corefoundation_dep, quartzcore_dep, cocoa_dep]
+      optional_deps += [cocoa_dep]
       have_vulkan_windowing = true
       vkconf.set10('GST_VULKAN_HAVE_WINDOW_COCOA', 1)
     endif
   endif
 
+  if host_system == 'ios'
+    uikit_dep = dependency('appleframeworks', modules : ['UIKit'], required : get_option('vulkan'))
+
+    if uikit_dep.found()
+      vulkan_sources += [
+        'ios/vkdisplay_ios.m',
+        'ios/vkwindow_ios.m',
+      ]
+      optional_deps += [uikit_dep]
+      have_vulkan_windowing = true
+      vkconf.set10('GST_VULKAN_HAVE_WINDOW_IOS', 1)
+    endif
+  endif
+
   if have_vulkan_windowing
     configure_file(input : 'vkconfig.h.meson',
       output : 'vkconfig.h',
@@ -99,7 +121,7 @@ if vulkan_dep.found() and has_vulkan_header
       objc_args : gst_plugins_bad_args + vulkan_defines + vulkan_objc_args,
       link_args : noseh_link_args,
       include_directories : [configinc],
-      dependencies : [vulkan_dep, gstvideo_dep, gstbase_dep] + optional_deps,
+      dependencies : [gstvideo_dep, gstbase_dep, vulkan_dep] + optional_deps,
       install : true,
       install_dir : plugins_install_dir,
     )

ext/vulkan/vkapi.h

@@ -46,6 +46,12 @@
 #endif
 #endif
 
+#if GST_VULKAN_HAVE_WINDOW_IOS
+#ifndef VK_USE_PLATFORM_IOS_MVK
+#define VK_USE_PLATFORM_IOS_MVK
+#endif
+#endif
+
 #include <vulkan/vulkan.h>
 
 #endif /* _VK_H_ */

ext/vulkan/vkconfig.h.meson

@@ -13,6 +13,7 @@ G_BEGIN_DECLS
 #mesondefine GST_VULKAN_HAVE_WINDOW_XCB
 #mesondefine GST_VULKAN_HAVE_WINDOW_WAYLAND
 #mesondefine GST_VULKAN_HAVE_WINDOW_COCOA
+#mesondefine GST_VULKAN_HAVE_WINDOW_IOS
 
 G_END_DECLS
 

ext/vulkan/vkdisplay.c

@@ -39,6 +39,9 @@
 #if GST_VULKAN_HAVE_WINDOW_COCOA
 #include "cocoa/vkdisplay_cocoa.h"
 #endif
+#if GST_VULKAN_HAVE_WINDOW_IOS
+#include "ios/vkdisplay_ios.h"
+#endif
 
 GST_DEBUG_CATEGORY_STATIC (GST_CAT_CONTEXT);
 #define GST_CAT_DEFAULT gst_vulkan_display_debug
@@ -432,6 +435,9 @@ gst_vulkan_display_choose_type (GstVulkanInstance * instance)
 #if GST_VULKAN_HAVE_WINDOW_COCOA
   CHOOSE_WINSYS (cocoa, COCOA);
 #endif
+#if GST_VULKAN_HAVE_WINDOW_IOS
+  CHOOSE_WINSYS (ios, IOS);
+#endif
 
 #undef CHOOSE_WINSYS
 
@@ -463,6 +469,10 @@ gst_vulkan_display_type_to_extension_string (GstVulkanDisplayType type)
   if (type & GST_VULKAN_DISPLAY_TYPE_COCOA)
     return VK_MVK_MACOS_SURFACE_EXTENSION_NAME;
 #endif
+#if GST_VULKAN_HAVE_WINDOW_IOS
+  if (type & GST_VULKAN_DISPLAY_TYPE_IOS)
+    return VK_MVK_IOS_SURFACE_EXTENSION_NAME;
+#endif
 
   return NULL;
 }

ext/vulkan/vkdisplay.h

@@ -50,6 +50,7 @@ enum _GstVulkanDisplayType
   GST_VULKAN_DISPLAY_TYPE_MIR = (1 << 3),
   GST_VULKAN_DISPLAY_TYPE_WIN32 = (1 << 4),
   GST_VULKAN_DISPLAY_TYPE_COCOA = (1 << 5),
+  GST_VULKAN_DISPLAY_TYPE_IOS = (1 << 6),
 
   GST_VULKAN_DISPLAY_TYPE_ANY = G_MAXUINT32
 };

ext/vulkan/vksink.c

@@ -65,6 +65,10 @@ static GstFlowReturn gst_vulkan_sink_prepare (GstBaseSink * bsink,
 static GstFlowReturn gst_vulkan_sink_show_frame (GstVideoSink * bsink,
     GstBuffer * buf);
 
+static void gst_vulkan_sink_video_overlay_init (GstVideoOverlayInterface *
+    iface);
+
+
 static GstStaticPadTemplate gst_vulkan_sink_template =
 GST_STATIC_PAD_TEMPLATE ("sink",
     GST_PAD_SINK,
@@ -91,7 +95,9 @@ enum
 #define gst_vulkan_sink_parent_class parent_class
 G_DEFINE_TYPE_WITH_CODE (GstVulkanSink, gst_vulkan_sink,
     GST_TYPE_VIDEO_SINK, GST_DEBUG_CATEGORY_INIT (gst_debug_vulkan_sink,
-        "vulkansink", 0, "Vulkan Video Sink"));
+        "vulkansink", 0, "Vulkan Video Sink");
+    G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY,
+        gst_vulkan_sink_video_overlay_init));
 
 static void
 gst_vulkan_sink_class_init (GstVulkanSinkClass * klass)
@@ -261,6 +267,7 @@ gst_vulkan_sink_change_state (GstElement * element, GstStateChange transition)
         return GST_STATE_CHANGE_FAILURE;
       }
 
+      /* FIXME: this probably doesn't need to be so early in the setup process */
       if (!(vk_sink->window =
               gst_vulkan_display_create_window (vk_sink->display))) {
         GST_ELEMENT_ERROR (vk_sink, RESOURCE, NOT_FOUND,
@@ -268,6 +275,13 @@ gst_vulkan_sink_change_state (GstElement * element, GstStateChange transition)
         return GST_STATE_CHANGE_FAILURE;
       }
 
+      if (!vk_sink->set_window_handle)
+        gst_video_overlay_prepare_window_handle (GST_VIDEO_OVERLAY (vk_sink));
+
+      if (vk_sink->set_window_handle)
+        gst_vulkan_window_set_window_handle (vk_sink->window,
+            vk_sink->set_window_handle);
+
       if (!gst_vulkan_window_open (vk_sink->window, &error)) {
         GST_ELEMENT_ERROR (vk_sink, RESOURCE, NOT_FOUND,
             ("Failed to open window"), ("%s", error->message));
@@ -501,3 +515,17 @@ gst_vulkan_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
 
   return GST_FLOW_OK;
 }
+
+static void
+gst_vulkan_sink_set_window_handle (GstVideoOverlay * voverlay, guintptr handle)
+{
+  GstVulkanSink *vk_sink = GST_VULKAN_SINK (voverlay);
+
+  vk_sink->set_window_handle = handle;
+}
+
+static void
+gst_vulkan_sink_video_overlay_init (GstVideoOverlayInterface * iface)
+{
+  iface->set_window_handle = gst_vulkan_sink_set_window_handle;
+}

ext/vulkan/vksink.h

@@ -56,6 +56,9 @@ struct _GstVulkanSink
 
   /* stream configuration */
   GstVideoInfo v_info;
+
+  /* the currently set window handle */
+  guintptr set_window_handle;
 };
 
 struct _GstVulkanSinkClass

ext/vulkan/vkwindow.c

@@ -49,6 +49,9 @@
 #if GST_VULKAN_HAVE_WINDOW_COCOA
 #include "cocoa/vkwindow_cocoa.h"
 #endif
+#if GST_VULKAN_HAVE_WINDOW_IOS
+#include "ios/vkwindow_ios.h"
+#endif
 
 #define GST_CAT_DEFAULT gst_vulkan_window_debug
 GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
@@ -190,6 +193,10 @@ gst_vulkan_window_new (GstVulkanDisplay * display)
 #if GST_VULKAN_HAVE_WINDOW_COCOA
   if (!window && (!user_choice || g_strstr_len (user_choice, 5, "cocoa")))
     window = GST_VULKAN_WINDOW (gst_vulkan_window_cocoa_new (display));
+#endif
+#if GST_VULKAN_HAVE_WINDOW_IOS
+  if (!window && (!user_choice || g_strstr_len (user_choice, 3, "ios")))
+    window = GST_VULKAN_WINDOW (gst_vulkan_window_ios_new (display));
 #endif
   if (!window) {
     /* subclass returned a NULL window */
@@ -294,6 +301,24 @@ gst_vulkan_window_redraw (GstVulkanWindow * window)
   g_signal_emit (window, gst_vulkan_window_signals[SIGNAL_DRAW], 0);
 }
 
+void
+gst_vulkan_window_set_window_handle (GstVulkanWindow * window, guintptr handle)
+{
+  GstVulkanWindowClass *klass;
+
+  g_return_if_fail (GST_IS_VULKAN_WINDOW (window));
+  klass = GST_VULKAN_WINDOW_GET_CLASS (window);
+
+  if (!klass->set_window_handle) {
+    if (handle)
+      g_warning ("%s does not implement the set_window_handle vfunc. "
+          "Output will not be embedded into the specified surface.",
+          GST_OBJECT_NAME (window));
+  } else {
+    klass->set_window_handle (window, handle);
+  }
+}
+
 GType gst_vulkan_dummy_window_get_type (void);
 G_DEFINE_TYPE (GstVulkanDummyWindow, gst_vulkan_dummy_window,
     GST_TYPE_VULKAN_WINDOW);

ext/vulkan/vkwindow.h

@@ -87,6 +87,8 @@ struct _GstVulkanWindowClass {
   gboolean      (*get_presentation_support)     (GstVulkanWindow *window,
                                                  GstVulkanDevice *device,
                                                  guint32 queue_family_idx);
+  void          (*set_window_handle)            (GstVulkanWindow *window,
+                                                 guintptr handle);
 
   /*< private >*/
   gpointer _reserved[GST_PADDING];
@@ -100,6 +102,8 @@ VkSurfaceKHR       gst_vulkan_window_get_surface                    (GstVulkanWi
 gboolean           gst_vulkan_window_get_presentation_support       (GstVulkanWindow *window,
                                                                      GstVulkanDevice *device,
                                                                      guint32 queue_family_idx);
+void               gst_vulkan_window_set_window_handle              (GstVulkanWindow *window,
+                                                                     guintptr handle);
 
 gboolean           gst_vulkan_window_open                           (GstVulkanWindow *window,
                                                                      GError ** error);