vtdec: add support for outputing vulkan images

meson.build

@@ -369,6 +369,7 @@ if ['darwin', 'ios'].contains(host_system)
 #    cdata.set('HAVE_VIDEOTOOLBOX_10_9_6', 1)
 #  endif
 endif
+have_objcpp = add_languages('objcpp', required : false)
 
 have_orcc = false
 orcc_args = []

sys/applemedia/corevideobuffer.c

@@ -26,6 +26,12 @@
 #include "iosurfaceglmemory.h"
 #endif
 #include "videotexturecache-gl.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include "videotexturecache-vulkan.h"
+#if !HAVE_IOS
+#include "iosurfacevulkanmemory.h"
+#endif
+#endif
 
 static const GstMetaInfo *gst_core_video_meta_get_info (void);
 
@@ -123,6 +129,13 @@ _create_glmem (GstAppleCoreVideoPixelBuffer * gpixbuf,
 #endif
 }
 
+#if defined(APPLEMEDIA_MOLTENVK)
+/* in videotexturecache-vulkan.m to avoid objc-ism from Metal being included
+  * in a non-objc file */
+extern GstMemory *_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
+    GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache);
+#endif
+
 void
 gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
     GstVideoInfo * info,
@@ -136,6 +149,9 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
   GstAppleCoreVideoPixelBuffer *gpixbuf;
   GstMemory *mem = NULL;
   gboolean do_gl = GST_IS_VIDEO_TEXTURE_CACHE_GL (cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+  gboolean do_vulkan = GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (cache);
+#endif
 
   gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf);
 
@@ -158,6 +174,10 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
 
       if (do_gl)
         mem = _create_glmem (gpixbuf, info, i, size, cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+      else if (do_vulkan)
+        mem = _create_vulkan_memory (gpixbuf, info, i, size, cache);
+#endif
       else
         mem =
             GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf,
@@ -172,6 +192,10 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
 
     if (do_gl)
       mem = _create_glmem (gpixbuf, info, 0, size, cache);
+#if defined(APPLEMEDIA_MOLTENVK)
+    else if (do_vulkan)
+      mem = _create_vulkan_memory (gpixbuf, info, 0, size, cache);
+#endif
     else
       mem =
           GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf, 0,

sys/applemedia/iosurfacevulkanmemory.c

@@ -0,0 +1,241 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "iosurfacevulkanmemory.h"
+#include "metal-helpers.h"
+
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_IO_SURFACE_VULKAN_MEMORY);
+#define GST_CAT_DEFAULT GST_CAT_IO_SURFACE_VULKAN_MEMORY
+
+G_DEFINE_TYPE (GstIOSurfaceVulkanMemoryAllocator,
+    gst_io_surface_vulkan_memory_allocator,
+    GST_TYPE_VULKAN_IMAGE_MEMORY_ALLOCATOR);
+
+typedef struct
+{
+  GstIOSurfaceVulkanMemory *memory;
+  IOSurfaceRef surface;
+} ContextThreadData;
+
+static GstAllocator *_io_surface_vulkan_memory_allocator;
+
+static void
+_mem_free (GstAllocator * allocator, GstMemory * mem)
+{
+  gst_io_surface_vulkan_memory_set_surface ((GstIOSurfaceVulkanMemory *) mem,
+      NULL);
+
+  GST_ALLOCATOR_CLASS
+      (gst_io_surface_vulkan_memory_allocator_parent_class)->free (allocator,
+      mem);
+}
+
+static gpointer
+_io_surface_vulkan_memory_allocator_map (GstMemory * bmem,
+    GstMapInfo * info, gsize size)
+{
+  GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
+
+  GST_LOG ("mapping surface %p flags %d", mem->surface, info->flags);
+
+  if (!(info->flags & GST_MAP_WRITE)) {
+    IOSurfaceLock (mem->surface, kIOSurfaceLockReadOnly, NULL);
+    return IOSurfaceGetBaseAddressOfPlane (mem->surface, mem->plane);
+  } else {
+    GST_ERROR ("couldn't map IOSurface %p flags %d", mem->surface, info->flags);
+    return NULL;
+  }
+}
+
+static void
+_io_surface_vulkan_memory_allocator_unmap (GstMemory * bmem, GstMapInfo * info)
+{
+  GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
+
+  GST_LOG ("unmapping surface %p flags %d", mem->surface, info->flags);
+
+  IOSurfaceUnlock (mem->surface, kIOSurfaceLockReadOnly, NULL);
+}
+
+static GstMemory *
+_mem_alloc (GstAllocator * allocator, gsize size, GstAllocationParams * params)
+{
+  g_warning
+      ("use gst_io_surface_vulkan_memory_wrapped () to allocate from this "
+      "IOSurface allocator");
+
+  return NULL;
+}
+
+static void
+    gst_io_surface_vulkan_memory_allocator_class_init
+    (GstIOSurfaceVulkanMemoryAllocatorClass * klass)
+{
+  GstAllocatorClass *allocator_class = (GstAllocatorClass *) klass;
+
+  allocator_class->alloc = _mem_alloc;
+  allocator_class->free = _mem_free;
+}
+
+static void
+gst_io_surface_vulkan_memory_allocator_init (GstIOSurfaceVulkanMemoryAllocator *
+    allocator)
+{
+  GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
+
+  alloc->mem_type = GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME;
+  GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
+
+  alloc->mem_map_full = _io_surface_vulkan_memory_allocator_map;
+  alloc->mem_unmap_full = _io_surface_vulkan_memory_allocator_unmap;
+}
+
+void
+gst_io_surface_vulkan_memory_init (void)
+{
+  static volatile gsize _init = 0;
+
+  if (g_once_init_enter (&_init)) {
+    GST_DEBUG_CATEGORY_INIT (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
+        "iosurfacevulkan", 0, "IOSurface Vulkan Buffer");
+
+    _io_surface_vulkan_memory_allocator =
+        g_object_new (GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, NULL);
+    gst_object_ref_sink (_io_surface_vulkan_memory_allocator);
+
+    gst_allocator_register (GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME,
+        gst_object_ref (_io_surface_vulkan_memory_allocator));
+    g_once_init_leave (&_init, 1);
+  }
+}
+
+gboolean
+gst_is_io_surface_vulkan_memory (GstMemory * mem)
+{
+  return mem != NULL && mem->allocator != NULL &&
+      g_type_is_a (G_OBJECT_TYPE (mem->allocator),
+      GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR);
+}
+
+static GstIOSurfaceVulkanMemory *
+_io_surface_vulkan_memory_new (GstVulkanDevice * device, IOSurfaceRef surface,
+    unsigned int /* MTLPixelFormat */ fmt, GstVideoInfo * info, guint plane,
+    gpointer user_data, GDestroyNotify notify)
+{
+  GstIOSurfaceVulkanMemory *mem;
+  GstAllocationParams params = { 0, };
+  VkImageCreateInfo image_info;
+  VkPhysicalDevice gpu;
+  VkImageUsageFlags usage;
+  VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL;
+  VkFormat vk_format;
+  VkImage image;
+  GError *error = NULL;
+  VkResult err;
+
+  mem = g_new0 (GstIOSurfaceVulkanMemory, 1);
+
+  vk_format = metal_format_to_vulkan (fmt);
+  /* FIXME: choose from outside */
+  usage =
+      VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
+      VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
+
+  /* *INDENT-OFF* */
+  image_info = (VkImageCreateInfo) {
+      .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
+      .pNext = NULL,
+      .flags = 0,
+      .imageType = VK_IMAGE_TYPE_2D,
+      .format = vk_format,
+      /* MoltenVK double checks these against the IOSurface in vkUseIOSurface()
+       * and will fail if they do not match */
+      .extent = (VkExtent3D) { GST_VIDEO_INFO_COMP_WIDTH (info, plane), GST_VIDEO_INFO_COMP_HEIGHT (info, plane), 1 },
+      .mipLevels = 1,
+      .arrayLayers = 1,
+      .samples = VK_SAMPLE_COUNT_1_BIT,
+      .tiling = tiling,
+      .usage = usage,
+      .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
+      .queueFamilyIndexCount = 0,
+      .pQueueFamilyIndices = NULL,
+      .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
+  };
+  /* *INDENT-ON* */
+
+  gpu = gst_vulkan_device_get_physical_device (device);
+
+  err = vkCreateImage (device->device, &image_info, NULL, &image);
+  if (gst_vulkan_error_to_g_error (err, &error, "vkCreateImage") < 0)
+    goto vk_error;
+
+  vkGetImageMemoryRequirements (device->device, image,
+      &mem->vulkan_mem.requirements);
+
+  gst_vulkan_image_memory_init (&mem->vulkan_mem,
+      _io_surface_vulkan_memory_allocator, NULL, device, usage, &params,
+      mem->vulkan_mem.requirements.size, user_data, notify);
+  mem->vulkan_mem.create_info = image_info;
+  mem->vulkan_mem.image = image;
+  mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL;
+
+  err =
+      vkGetPhysicalDeviceImageFormatProperties (gpu, vk_format,
+      VK_IMAGE_TYPE_2D, tiling, usage, 0, &mem->vulkan_mem.format_properties);
+  if (gst_vulkan_error_to_g_error (err, &error,
+          "vkGetPhysicalDeviceImageFormatProperties") < 0)
+    goto vk_error;
+
+  GST_MINI_OBJECT_FLAG_SET (mem, GST_MEMORY_FLAG_READONLY);
+
+  mem->surface = NULL;
+  mem->plane = plane;
+  gst_io_surface_vulkan_memory_set_surface (mem, surface);
+
+  return mem;
+
+vk_error:
+  {
+    GST_CAT_ERROR (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
+        "Failed to allocate image memory %s", error->message);
+    g_clear_error (&error);
+    goto error;
+  }
+
+error:
+  {
+    if (mem)
+      gst_memory_unref ((GstMemory *) mem);
+    return NULL;
+  }
+}
+
+GstIOSurfaceVulkanMemory *
+gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
+    IOSurfaceRef surface, unsigned int /* MTLPixelFormat */ fmt,
+    GstVideoInfo * info, guint plane, gpointer user_data, GDestroyNotify notify)
+{
+  return _io_surface_vulkan_memory_new (device, surface, fmt, info, plane,
+      user_data, notify);
+}

sys/applemedia/iosurfacevulkanmemory.h

@@ -0,0 +1,77 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_IO_SURFACE_VULKAN_MEMORY_H_
+#define _GST_IO_SURFACE_VULKAN_MEMORY_H_
+
+#include <IOSurface/IOSurfaceRef.h>
+#include <IOSurface/IOSurfaceObjc.h>
+#include <gst/gst.h>
+#include <gst/gstallocator.h>
+#include <gst/video/video.h>
+#include <gst/vulkan/vulkan.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR (gst_io_surface_vulkan_memory_allocator_get_type())
+GType gst_io_surface_vulkan_memory_allocator_get_type(void);
+
+#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj)              (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
+#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass)      (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_GET_CLASS(obj)       (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj)                 (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocator))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass)         (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CAST(obj)            ((GstIOSurfaceVulkanMemoryAllocator *)(obj))
+
+typedef struct _GstIOSurfaceVulkanMemory
+{
+  GstVulkanImageMemory vulkan_mem;
+  IOSurfaceRef surface;
+  guint plane;
+} GstIOSurfaceVulkanMemory;
+
+#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME   "IOSurfaceVulkanMemory"
+
+void gst_io_surface_vulkan_memory_init (void);
+
+GstIOSurfaceVulkanMemory *
+gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
+    IOSurfaceRef surface,
+    unsigned int fmt, /* MTLPixelFormat */
+    GstVideoInfo * info,
+    guint plane,
+    gpointer user_data,
+    GDestroyNotify notify);
+
+gboolean gst_is_io_surface_vulkan_memory (GstMemory * mem);
+
+typedef struct _GstIOSurfaceVulkanMemoryAllocator
+{
+  GstVulkanMemoryAllocator allocator;
+} GstIOSurfaceVulkanMemoryAllocator;
+
+typedef struct _GstIOSurfaceVulkanMemoryAllocatorClass
+{
+  GstVulkanMemoryAllocatorClass parent_class;
+} GstIOSurfaceVulkanMemoryAllocatorClass;
+
+G_END_DECLS
+
+#endif /* _GST_IO_SURFACE_MEMORY_H_ */

sys/applemedia/meson.build

@@ -68,12 +68,12 @@ if host_system == 'ios'
     applemedia_objc_args += ['-fobjc-abi-version=2', '-fobjc-legacy-dispatch']
 
     ios_media_dep = dependency('appleframeworks', modules : ['Foundation', 'AssetsLibrary'], required : applemedia_option)
-    applemedia_frameworks += [ios_media_dep]
+    iosurface_dep = dependency('IOSurface', required : applemedia_option)
+    applemedia_frameworks += [ios_media_dep, iosurface_dep]
 else
     applemedia_sources += [
         'iosurfaceglmemory.c'
     ]
-    applemedia_objc_args += ['-mmacosx-version-min=10.8']
     cocoa_dep = dependency('Cocoa', required : applemedia_option)
     iosurface_dep = dependency('IOSurface', required : applemedia_option)
     applemedia_opengl_dep = dependency('appleframeworks', modules : ['OpenGL'], required : applemedia_option)
@@ -87,11 +87,25 @@ foreach framework : applemedia_frameworks
     endif
 endforeach
 
+if gstvulkan_dep.found() and have_objcpp
+  moltenvk_dep = cc.find_library('MoltenVK', required : false)
+  metal_dep = dependency('appleframeworks', modules : ['Metal'], required : false)
+  if metal_dep.found() and moltenvk_dep.found() and cc.has_header ('MoltenVK/vk_mvk_moltenvk.h')
+    applemedia_frameworks += [moltenvk_dep, gstvulkan_dep, metal_dep]
+    applemedia_sources += [
+      'videotexturecache-vulkan.mm',
+      'iosurfacevulkanmemory.c',
+    ]
+    applemedia_args += ['-DAPPLEMEDIA_MOLTENVK']
+  endif
+endif
+
 if applemedia_found_deps
     gstapplemedia = library('gstapplemedia',
         applemedia_sources,
         c_args : gst_plugins_bad_args + applemedia_args,
         objc_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
+        objcpp_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
         link_args : noseh_link_args,
         include_directories : [configinc, libsinc],
         dependencies : [gstvideo_dep, gstaudio_dep, gstpbutils_dep, gst_dep, gstbase_dep, gstgl_dep] + applemedia_frameworks,

sys/applemedia/metal-helpers.h

@@ -0,0 +1,44 @@
+/*
+ * GStreamer
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _APPLEMEDIA_METAL_HELPERS_H_
+#define _APPLEMEDIA_METAL_HELPERS_H_
+
+#include <gst/gst.h>
+#include "corevideomemory.h"
+#include "videotexturecache.h"
+
+G_BEGIN_DECLS
+
+VkFormat                metal_format_to_vulkan                          (unsigned int fmt);
+unsigned int            video_info_to_metal_format                      (GstVideoInfo * info,
+                                                                         guint plane);
+
+GstMemory *             _create_vulkan_memory                           (GstAppleCoreVideoPixelBuffer * gpixbuf,
+                                                                         GstVideoInfo * info,
+                                                                         guint plane,
+                                                                         gsize size,
+                                                                         GstVideoTextureCache * cache);
+
+void                    gst_io_surface_vulkan_memory_set_surface        (GstIOSurfaceVulkanMemory * memory,
+                                                                         IOSurfaceRef surface);
+
+G_END_DECLS
+#endif /* _APPLEMEDIA_METAL_HELPERS_H_ */

sys/applemedia/videotexturecache-vulkan.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
+#define __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
+
+#include <gst/vulkan/vulkan.h>
+#include "videotexturecache.h"
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN         (gst_video_texture_cache_vulkan_get_type())
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN(o)           (G_TYPE_CHECK_INSTANCE_CAST((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkan))
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k)     (G_TYPE_CHECK_CLASS_CAST((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
+#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN(o)        (G_TYPE_CHECK_INSTANCE_TYPE((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
+#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k)  (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
+#define GST_VIDEO_TEXTURE_CACHE_VULKAN_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
+GType gst_video_texture_cache_vulkan_get_type     (void);
+
+typedef struct _GstVideoTextureCacheVulkan
+{
+  GstVideoTextureCache parent;
+
+  GstVulkanDevice *device;
+} GstVideoTextureCacheVulkan;
+
+typedef struct _GstVideoTextureCacheVulkanClass
+{
+  GstVideoTextureCacheClass parent_class;
+} GstVideoTextureCacheVulkanClass;
+
+GstVideoTextureCache *  gst_video_texture_cache_vulkan_new             (GstVulkanDevice * device);
+
+G_END_DECLS
+
+#endif /* __GST_CORE_VIDEO_TEXTURE_CACHE_H__ */

sys/applemedia/videotexturecache-vulkan.mm

@@ -0,0 +1,301 @@
+/*
+ * Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include "config.h"
+#endif
+
+#include <Metal/Metal.h>
+
+#if !HAVE_IOS
+#import <AppKit/AppKit.h>
+#endif
+#include "iosurfacevulkanmemory.h"
+
+/* silence macor redefinition warnings */
+#undef VK_USE_PLATFORM_MACOS_MVK
+#undef VK_USE_PLATFORM_IOS_MVK
+#include <MoltenVK/vk_mvk_moltenvk.h>
+#include <MoltenVK/mvk_datatypes.h>
+/* silence macro redefinition warnings */
+#undef VK_USE_PLATFORM_MACOS_MVK
+#undef VK_USE_PLATFORM_IOS_MVK
+
+#include "coremediabuffer.h"
+#include "corevideobuffer.h"
+#include "vtutil.h"
+
+#include "videotexturecache-vulkan.h"
+#include "metal-helpers.h"
+
+typedef struct _GstVideoTextureCacheVulkanPrivate
+{
+  GstBufferPool *pool;
+} GstVideoTextureCacheVulkanPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE (GstVideoTextureCacheVulkan, gst_video_texture_cache_vulkan, GST_TYPE_VIDEO_TEXTURE_CACHE);
+
+#define GET_PRIV(instance) \
+    G_TYPE_INSTANCE_GET_PRIVATE (instance, GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanPrivate)
+
+typedef struct _IOSTextureWrapper
+{
+  CVMetalTextureCacheRef cache;
+  CVMetalTextureRef texture;
+} IOSTextureWrapper;
+
+enum
+{
+  PROP_0,
+  PROP_DEVICE,
+};
+
+static GstMemory * gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
+      GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size);
+
+GstVideoTextureCache *
+gst_video_texture_cache_vulkan_new (GstVulkanDevice * device)
+{
+  g_return_val_if_fail (GST_IS_VULKAN_DEVICE (device), NULL);
+
+  return (GstVideoTextureCache *) g_object_new (GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN,
+      "device", device, NULL);
+}
+
+static void
+gst_video_texture_cache_vulkan_finalize (GObject * object)
+{
+  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+#if 0
+  gst_buffer_pool_set_active (cache->pool, FALSE);
+  gst_object_unref (cache->pool);
+#endif
+  gst_object_unref (cache_vulkan->device);
+
+  G_OBJECT_CLASS (gst_video_texture_cache_vulkan_parent_class)->finalize (object);
+}
+
+static void
+gst_video_texture_cache_vulkan_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+  switch (prop_id) {
+    case PROP_DEVICE:
+      cache_vulkan->device = (GstVulkanDevice *) g_value_dup_object (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_video_texture_cache_vulkan_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * pspec)
+{
+  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+  switch (prop_id) {
+    case PROP_DEVICE:
+      g_value_set_object (value, cache_vulkan->device);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_video_texture_cache_vulkan_constructed (GObject * object)
+{
+  GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
+
+  g_return_if_fail (GST_IS_VULKAN_DEVICE (cache_vulkan->device));
+
+  gst_io_surface_vulkan_memory_init ();
+#if 0
+  cache->pool = GST_BUFFER_POOL (gst_vulkan_buffer_pool_new (ctx));
+#endif
+}
+
+static void
+gst_video_texture_cache_vulkan_init (GstVideoTextureCacheVulkan * cache_vulkan)
+{
+}
+
+static void
+gst_video_texture_cache_vulkan_class_init (GstVideoTextureCacheVulkanClass *klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstVideoTextureCacheClass *cache_class = (GstVideoTextureCacheClass *) klass;
+
+  gobject_class->set_property = gst_video_texture_cache_vulkan_set_property;
+  gobject_class->get_property = gst_video_texture_cache_vulkan_get_property;
+  gobject_class->constructed = gst_video_texture_cache_vulkan_constructed;
+  gobject_class->finalize = gst_video_texture_cache_vulkan_finalize;
+
+  g_object_class_install_property (gobject_class, PROP_DEVICE,
+      g_param_spec_object ("device", "device",
+          "Associated Vulkan device", GST_TYPE_VULKAN_DEVICE,
+          (GParamFlags) (G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)));
+
+  cache_class->create_memory = gst_video_texture_cache_vulkan_create_memory;
+}
+
+static GstMemory *
+gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
+      GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size)
+{
+  return (GstMemory *) gpixbuf;
+}
+
+VkFormat
+metal_format_to_vulkan (unsigned int fmt)
+{
+  MTLPixelFormat mtl_fmt = (MTLPixelFormat) fmt;
+
+  switch (mtl_fmt) {
+    case MTLPixelFormatRGBA8Unorm:
+      return VK_FORMAT_R8G8B8A8_UNORM;
+    case MTLPixelFormatRG8Unorm:
+      return VK_FORMAT_R8G8_UNORM;
+    case MTLPixelFormatR8Unorm:
+      return VK_FORMAT_R8_UNORM;
+    default:
+      g_assert_not_reached ();
+  }
+}
+
+unsigned int
+video_info_to_metal_format (GstVideoInfo * info, guint plane)
+{
+  switch (GST_VIDEO_INFO_FORMAT (info)) {
+    case GST_VIDEO_FORMAT_BGRA:
+      return (unsigned int) MTLPixelFormatRGBA8Unorm;
+    case GST_VIDEO_FORMAT_NV12:
+      if (plane == 0)
+        return (unsigned int) MTLPixelFormatR8Unorm;
+      else
+        return (unsigned int) MTLPixelFormatRG8Unorm;
+    default:
+      g_assert_not_reached ();
+  }
+}
+
+GstMemory *
+_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
+    GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache)
+{
+  GstIOSurfaceVulkanMemory *mem;
+  CVPixelBufferRef pixel_buf = gpixbuf->buf;
+  IOSurfaceRef surface = CVPixelBufferGetIOSurface (pixel_buf);
+  GstVideoTextureCacheVulkan *cache_vulkan =
+      GST_VIDEO_TEXTURE_CACHE_VULKAN (cache);
+  MTLPixelFormat fmt = video_info_to_metal_format (info, plane);
+
+  CFRetain (pixel_buf);
+  mem = gst_io_surface_vulkan_memory_wrapped (cache_vulkan->device,
+      surface, fmt, info, plane, pixel_buf, (GDestroyNotify) CFRelease);
+
+  if (!mem)
+    return NULL;
+
+  return GST_MEMORY_CAST (mem);
+}
+
+typedef struct _IOSurfaceTextureWrapper
+{
+  CVPixelBufferRef pixbuf;
+  gpointer texture; /* id<MTLTexture> */
+} IOSurfaceTextureWrapper;
+
+static void
+free_texture_wrapper (IOSurfaceTextureWrapper * wrapper)
+{
+  CFRelease (wrapper->pixbuf);
+  id<MTLTexture> tex = (__bridge_transfer id<MTLTexture>) wrapper->texture;
+  (void) tex;
+  g_free (wrapper);
+}
+
+static MTLTextureDescriptor *
+gst_new_mtl_tex_descripter_from_memory (GstIOSurfaceVulkanMemory * memory)
+{
+  GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
+  MTLTextureDescriptor* tex_desc = [MTLTextureDescriptor new];
+  tex_desc.pixelFormat = mvkMTLPixelFormatFromVkFormat (vk_mem->create_info.format);
+  tex_desc.textureType = mvkMTLTextureTypeFromVkImageType (vk_mem->create_info.imageType, 0, false);
+  tex_desc.width = vk_mem->create_info.extent.width;
+  tex_desc.height = vk_mem->create_info.extent.height;
+  tex_desc.depth = vk_mem->create_info.extent.depth;
+  tex_desc.mipmapLevelCount = vk_mem->create_info.mipLevels;
+  tex_desc.sampleCount = mvkSampleCountFromVkSampleCountFlagBits(vk_mem->create_info.samples);
+  tex_desc.arrayLength = vk_mem->create_info.arrayLayers;
+  tex_desc.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView;//mvkMTLTextureUsageFromVkImageUsageFlags(vk_mem->create_info.usage);
+  tex_desc.storageMode = MTLStorageModePrivate;//MTLStorageModeManaged; /* IOSurface == shared/managed */
+  tex_desc.cpuCacheMode = MTLCPUCacheModeDefaultCache;//mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(vk_mem->vk_mem->properties);
+
+  return tex_desc;
+}
+
+void
+gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
+    IOSurfaceRef surface)
+{
+  GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
+
+  if (memory->surface) {
+     IOSurfaceDecrementUseCount (memory->surface);
+  } else {
+    g_assert (vk_mem->notify == (GDestroyNotify) CFRelease);
+    g_assert (vk_mem->user_data != NULL);
+  }
+
+  memory->surface = surface;
+  if (surface) {
+    id<MTLDevice> mtl_dev = nil;
+    id<MTLTexture> texture = nil;
+    VkPhysicalDevice gpu;
+
+    IOSurfaceIncrementUseCount (surface);
+
+    gpu = gst_vulkan_device_get_physical_device (vk_mem->device);
+    vkGetMTLDeviceMVK (gpu, &mtl_dev);
+
+    /* We cannot use vkUseIOSurfaceMVK() for multi-planer as MoltenVK does not
+     * support them. */
+
+    MTLTextureDescriptor *tex_desc = gst_new_mtl_tex_descripter_from_memory (memory);
+    texture = [mtl_dev newTextureWithDescriptor:tex_desc iosurface:surface plane:memory->plane];
+
+    IOSurfaceTextureWrapper *texture_data = g_new0 (IOSurfaceTextureWrapper, 1);
+    texture_data->pixbuf = (CVPixelBufferRef) vk_mem->user_data;
+    texture_data->texture = (__bridge_retained gpointer) texture;
+
+    VkResult err = vkSetMTLTextureMVK (memory->vulkan_mem.image, texture);
+    GST_DEBUG ("bound texture %p to image %p: 0x%x", texture, memory->vulkan_mem.image,
+        err);
+
+    vk_mem->user_data = texture_data;
+    vk_mem->notify = (GDestroyNotify) free_texture_wrapper;
+  }
+}

sys/applemedia/vtdec.c

@@ -45,6 +45,9 @@
 #include "corevideobuffer.h"
 #include "coremediabuffer.h"
 #include "videotexturecache-gl.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include "videotexturecache-vulkan.h"
+#endif
 
 GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
 #define GST_CAT_DEFAULT gst_vtdec_debug_category
@@ -111,11 +114,21 @@ const CFStringRef
 CFSTR ("RequireHardwareAcceleratedVideoDecoder");
 #endif
 
+#if defined(APPLEMEDIA_MOLTENVK)
 #define VIDEO_SRC_CAPS \
     GST_VIDEO_CAPS_MAKE("NV12") ";"                                     \
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
         "NV12") ", "                                                    \
-    "texture-target = (string) rectangle;"
+    "texture-target = (string) rectangle ; "                            \
+    GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\
+        "NV12")
+#else
+#define VIDEO_SRC_CAPS \
+    GST_VIDEO_CAPS_MAKE("NV12") ";"                                     \
+    GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
+        "NV12") ", "                                                    \
+    "texture-target = (string) rectangle "
+#endif
 
 G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
 
@@ -210,20 +223,22 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
     CFRelease (vtdec->format_description);
   vtdec->format_description = NULL;
 
+#if defined(APPLEMEDIA_MOLTENVK)
+  gst_clear_object (&vtdec->device);
+  gst_clear_object (&vtdec->instance);
+#endif
+
   GST_DEBUG_OBJECT (vtdec, "stop");
 
   return TRUE;
 }
 
 static void
-setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
+setup_texture_cache (GstVtdec * vtdec)
 {
   GstVideoCodecState *output_state;
 
-  g_return_if_fail (vtdec->texture_cache == NULL);
-
   output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
-  vtdec->texture_cache = gst_video_texture_cache_gl_new (context);
   gst_video_texture_cache_set_format (vtdec->texture_cache,
       GST_VIDEO_FORMAT_NV12, output_state->caps);
   gst_video_codec_state_unref (output_state);
@@ -240,7 +255,10 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
   GstVtdec *vtdec;
   OSStatus err = noErr;
   GstCapsFeatures *features = NULL;
-  gboolean output_textures;
+  gboolean output_textures = FALSE;
+#if defined(APPLEMEDIA_MOLTENVK)
+  gboolean output_vulkan = FALSE;
+#endif
 
   vtdec = GST_VTDEC (decoder);
   if (vtdec->session)
@@ -296,6 +314,12 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
           GST_GL_TEXTURE_TARGET_2D_STR,
 #endif
           NULL);
+
+#if defined(APPLEMEDIA_MOLTENVK)
+    output_vulkan =
+        gst_caps_features_contains (features,
+        GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE);
+#endif
   }
   gst_caps_unref (caps);
 
@@ -323,33 +347,83 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
     }
   }
 
-  if (vtdec->texture_cache != NULL && !output_textures) {
+  if (vtdec->texture_cache != NULL
+      && ((GST_IS_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache)
+              && !output_textures)
+#if defined(APPLEMEDIA_MOLTENVK)
+          || (GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache)
+              && !output_vulkan)
+#endif
+      )) {
     g_object_unref (vtdec->texture_cache);
     vtdec->texture_cache = NULL;
   }
 
-  if (err == noErr && output_textures) {
+  if (err == noErr) {
-    GstVideoTextureCacheGL *cache_gl = NULL;
+    if (output_textures) {
+      GstVideoTextureCacheGL *cache_gl = NULL;
 
-    if (vtdec->texture_cache)
+      if (vtdec->texture_cache)
-      cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
+        cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
 
-    /* call this regardless of whether caps have changed or not since a new
+      /* call this regardless of whether caps have changed or not since a new
-     * local context could have become available
+       * local context could have become available
-     */
+       */
-    if (!vtdec->ctxh)
+      if (!vtdec->ctxh)
-      vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
+        vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
-    gst_gl_context_helper_ensure_context (vtdec->ctxh);
+      gst_gl_context_helper_ensure_context (vtdec->ctxh);
 
-    GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
+      GST_INFO_OBJECT (vtdec, "pushing GL textures, context %p old context %p",
-        vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
+          vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
 
-    if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
+      if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
-      g_object_unref (vtdec->texture_cache);
+        g_object_unref (vtdec->texture_cache);
-      vtdec->texture_cache = NULL;
+        vtdec->texture_cache = NULL;
+      }
+      if (!vtdec->texture_cache) {
+        vtdec->texture_cache =
+            gst_video_texture_cache_gl_new (vtdec->ctxh->context);
+        setup_texture_cache (vtdec);
+      }
     }
-    if (!vtdec->texture_cache)
+#if defined(APPLEMEDIA_MOLTENVK)
-      setup_texture_cache (vtdec, vtdec->ctxh->context);
+    if (output_vulkan) {
+      GstVideoTextureCacheVulkan *cache_vulkan = NULL;
+
+      if (vtdec->texture_cache)
+        cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache);
+
+      gst_vulkan_ensure_element_data (GST_ELEMENT (vtdec), NULL,
+          &vtdec->instance);
+
+      if (!gst_vulkan_device_run_context_query (GST_ELEMENT (vtdec),
+              &vtdec->device)) {
+        GError *error = NULL;
+        GST_DEBUG_OBJECT (vtdec, "No device retrieved from peer elements");
+        if (!(vtdec->device =
+                gst_vulkan_instance_create_device (vtdec->instance, &error))) {
+          GST_ELEMENT_ERROR (vtdec, RESOURCE, NOT_FOUND,
+              ("Failed to create vulkan device"), ("%s", error->message));
+          g_clear_error (&error);
+          return FALSE;
+        }
+      }
+
+      GST_INFO_OBJECT (vtdec, "pushing vulkan images, device %" GST_PTR_FORMAT
+          " old device %" GST_PTR_FORMAT, vtdec->device,
+          cache_vulkan ? cache_vulkan->device : NULL);
+
+      if (cache_vulkan && cache_vulkan->device != vtdec->device) {
+        g_object_unref (vtdec->texture_cache);
+        vtdec->texture_cache = NULL;
+      }
+      if (!vtdec->texture_cache) {
+        vtdec->texture_cache =
+            gst_video_texture_cache_vulkan_new (vtdec->device);
+        setup_texture_cache (vtdec);
+      }
+    }
+#endif
   }
 
   if (prevcaps)
@@ -1043,6 +1117,11 @@ gst_vtdec_set_context (GstElement * element, GstContext * context)
     vtdec->ctxh = gst_gl_context_helper_new (element);
   gst_gl_handle_set_context (element, context,
       &vtdec->ctxh->display, &vtdec->ctxh->other_context);
+
+#if defined (APPLEMEDIA_MOLTENVK)
+  gst_vulkan_handle_set_context (element, context, NULL, &vtdec->instance);
+#endif
+
   GST_ELEMENT_CLASS (gst_vtdec_parent_class)->set_context (element, context);
 }
 

sys/applemedia/vtdec.h

@@ -27,6 +27,9 @@
 #include <VideoToolbox/VideoToolbox.h>
 #include "videotexturecache.h"
 #include "glcontexthelper.h"
+#if defined(APPLEMEDIA_MOLTENVK)
+#include <gst/vulkan/vulkan.h>
+#endif
 
 G_BEGIN_DECLS
 
@@ -51,6 +54,11 @@ struct _GstVtdec
   GstVideoTextureCache *texture_cache;
   GstGLContextHelper *ctxh;
 
+#if defined(APPLEMEDIA_MOLTENVK)
+  GstVulkanInstance *instance;
+  GstVulkanDevice *device;
+#endif
+
   gboolean require_hardware;
 };