mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-23 10:11:08 +00:00
Code Issues 1 Projects Releases Wiki Activity

vtdec: add support for outputing vulkan images

Browse source
This commit is contained in:
Matthew Waters 2019-09-03 13:56:22 +10:00 committed by GStreamer Merge Bot
parent 52ff97d085
commit 526afac736
10 changed files with 867 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
meson.build
View file

@ -369,6 +369,7 @@ if ['darwin', 'ios'].contains(host_system)
# cdata.set('HAVE_VIDEOTOOLBOX_10_9_6', 1)
# endif
endif
have_objcpp = add_languages('objcpp', required : false)
have_orcc = false
orcc_args = []

24
sys/applemedia/corevideobuffer.c
View file

@ -26,6 +26,12 @@
#include "iosurfaceglmemory.h"
#endif
#include "videotexturecache-gl.h"
#if defined(APPLEMEDIA_MOLTENVK)
#include "videotexturecache-vulkan.h"
#if !HAVE_IOS
#include "iosurfacevulkanmemory.h"
#endif
#endif
static const GstMetaInfo *gst_core_video_meta_get_info (void);
@ -123,6 +129,13 @@ _create_glmem (GstAppleCoreVideoPixelBuffer * gpixbuf,
#endif
}
#if defined(APPLEMEDIA_MOLTENVK)
/* in videotexturecache-vulkan.m to avoid objc-ism from Metal being included
* in a non-objc file */
extern GstMemory *_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache);
#endif
void
gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
GstVideoInfo * info,
@ -136,6 +149,9 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
GstAppleCoreVideoPixelBuffer *gpixbuf;
GstMemory *mem = NULL;
gboolean do_gl = GST_IS_VIDEO_TEXTURE_CACHE_GL (cache);
#if defined(APPLEMEDIA_MOLTENVK)
gboolean do_vulkan = GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (cache);
#endif
gpixbuf = gst_apple_core_video_pixel_buffer_new (pixel_buf);
@ -158,6 +174,10 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
if (do_gl)
mem = _create_glmem (gpixbuf, info, i, size, cache);
#if defined(APPLEMEDIA_MOLTENVK)
else if (do_vulkan)
mem = _create_vulkan_memory (gpixbuf, info, i, size, cache);
#endif
else
mem =
GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf,
@ -172,6 +192,10 @@ gst_core_video_wrap_pixel_buffer (GstBuffer * buf,
if (do_gl)
mem = _create_glmem (gpixbuf, info, 0, size, cache);
#if defined(APPLEMEDIA_MOLTENVK)
else if (do_vulkan)
mem = _create_vulkan_memory (gpixbuf, info, 0, size, cache);
#endif
else
mem =
GST_MEMORY_CAST (gst_apple_core_video_memory_new_wrapped (gpixbuf, 0,

241
sys/applemedia/iosurfacevulkanmemory.c Normal file
View file

@ -0,0 +1,241 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "iosurfacevulkanmemory.h"
#include "metal-helpers.h"
GST_DEBUG_CATEGORY_STATIC (GST_CAT_IO_SURFACE_VULKAN_MEMORY);
#define GST_CAT_DEFAULT GST_CAT_IO_SURFACE_VULKAN_MEMORY
G_DEFINE_TYPE (GstIOSurfaceVulkanMemoryAllocator,
gst_io_surface_vulkan_memory_allocator,
GST_TYPE_VULKAN_IMAGE_MEMORY_ALLOCATOR);
typedef struct
{
GstIOSurfaceVulkanMemory *memory;
IOSurfaceRef surface;
} ContextThreadData;
static GstAllocator *_io_surface_vulkan_memory_allocator;
static void
_mem_free (GstAllocator * allocator, GstMemory * mem)
{
gst_io_surface_vulkan_memory_set_surface ((GstIOSurfaceVulkanMemory *) mem,
NULL);
GST_ALLOCATOR_CLASS
(gst_io_surface_vulkan_memory_allocator_parent_class)->free (allocator,
mem);
}
static gpointer
_io_surface_vulkan_memory_allocator_map (GstMemory * bmem,
GstMapInfo * info, gsize size)
{
GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
GST_LOG ("mapping surface %p flags %d", mem->surface, info->flags);
if (!(info->flags & GST_MAP_WRITE)) {
IOSurfaceLock (mem->surface, kIOSurfaceLockReadOnly, NULL);
return IOSurfaceGetBaseAddressOfPlane (mem->surface, mem->plane);
} else {
GST_ERROR ("couldn't map IOSurface %p flags %d", mem->surface, info->flags);
return NULL;
}
}
static void
_io_surface_vulkan_memory_allocator_unmap (GstMemory * bmem, GstMapInfo * info)
{
GstIOSurfaceVulkanMemory *mem = (GstIOSurfaceVulkanMemory *) bmem;
GST_LOG ("unmapping surface %p flags %d", mem->surface, info->flags);
IOSurfaceUnlock (mem->surface, kIOSurfaceLockReadOnly, NULL);
}
static GstMemory *
_mem_alloc (GstAllocator * allocator, gsize size, GstAllocationParams * params)
{
g_warning
("use gst_io_surface_vulkan_memory_wrapped () to allocate from this "
"IOSurface allocator");
return NULL;
}
static void
gst_io_surface_vulkan_memory_allocator_class_init
(GstIOSurfaceVulkanMemoryAllocatorClass * klass)
{
GstAllocatorClass *allocator_class = (GstAllocatorClass *) klass;
allocator_class->alloc = _mem_alloc;
allocator_class->free = _mem_free;
}
static void
gst_io_surface_vulkan_memory_allocator_init (GstIOSurfaceVulkanMemoryAllocator *
allocator)
{
GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
alloc->mem_type = GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME;
GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
alloc->mem_map_full = _io_surface_vulkan_memory_allocator_map;
alloc->mem_unmap_full = _io_surface_vulkan_memory_allocator_unmap;
}
void
gst_io_surface_vulkan_memory_init (void)
{
static volatile gsize _init = 0;
if (g_once_init_enter (&_init)) {
GST_DEBUG_CATEGORY_INIT (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
"iosurfacevulkan", 0, "IOSurface Vulkan Buffer");
_io_surface_vulkan_memory_allocator =
g_object_new (GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, NULL);
gst_object_ref_sink (_io_surface_vulkan_memory_allocator);
gst_allocator_register (GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME,
gst_object_ref (_io_surface_vulkan_memory_allocator));
g_once_init_leave (&_init, 1);
}
}
gboolean
gst_is_io_surface_vulkan_memory (GstMemory * mem)
{
return mem != NULL && mem->allocator != NULL &&
g_type_is_a (G_OBJECT_TYPE (mem->allocator),
GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR);
}
static GstIOSurfaceVulkanMemory *
_io_surface_vulkan_memory_new (GstVulkanDevice * device, IOSurfaceRef surface,
unsigned int /* MTLPixelFormat */ fmt, GstVideoInfo * info, guint plane,
gpointer user_data, GDestroyNotify notify)
{
GstIOSurfaceVulkanMemory *mem;
GstAllocationParams params = { 0, };
VkImageCreateInfo image_info;
VkPhysicalDevice gpu;
VkImageUsageFlags usage;
VkImageTiling tiling = VK_IMAGE_TILING_OPTIMAL;
VkFormat vk_format;
VkImage image;
GError *error = NULL;
VkResult err;
mem = g_new0 (GstIOSurfaceVulkanMemory, 1);
vk_format = metal_format_to_vulkan (fmt);
/* FIXME: choose from outside */
usage =
VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_SAMPLED_BIT |
VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT;
/* *INDENT-OFF* */
image_info = (VkImageCreateInfo) {
.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
.pNext = NULL,
.flags = 0,
.imageType = VK_IMAGE_TYPE_2D,
.format = vk_format,
/* MoltenVK double checks these against the IOSurface in vkUseIOSurface()
* and will fail if they do not match */
.extent = (VkExtent3D) { GST_VIDEO_INFO_COMP_WIDTH (info, plane), GST_VIDEO_INFO_COMP_HEIGHT (info, plane), 1 },
.mipLevels = 1,
.arrayLayers = 1,
.samples = VK_SAMPLE_COUNT_1_BIT,
.tiling = tiling,
.usage = usage,
.sharingMode = VK_SHARING_MODE_EXCLUSIVE,
.queueFamilyIndexCount = 0,
.pQueueFamilyIndices = NULL,
.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
};
/* *INDENT-ON* */
gpu = gst_vulkan_device_get_physical_device (device);
err = vkCreateImage (device->device, &image_info, NULL, &image);
if (gst_vulkan_error_to_g_error (err, &error, "vkCreateImage") < 0)
goto vk_error;
vkGetImageMemoryRequirements (device->device, image,
&mem->vulkan_mem.requirements);
gst_vulkan_image_memory_init (&mem->vulkan_mem,
_io_surface_vulkan_memory_allocator, NULL, device, usage, &params,
mem->vulkan_mem.requirements.size, user_data, notify);
mem->vulkan_mem.create_info = image_info;
mem->vulkan_mem.image = image;
mem->vulkan_mem.barrier.image_layout = VK_IMAGE_LAYOUT_GENERAL;
err =
vkGetPhysicalDeviceImageFormatProperties (gpu, vk_format,
VK_IMAGE_TYPE_2D, tiling, usage, 0, &mem->vulkan_mem.format_properties);
if (gst_vulkan_error_to_g_error (err, &error,
"vkGetPhysicalDeviceImageFormatProperties") < 0)
goto vk_error;
GST_MINI_OBJECT_FLAG_SET (mem, GST_MEMORY_FLAG_READONLY);
mem->surface = NULL;
mem->plane = plane;
gst_io_surface_vulkan_memory_set_surface (mem, surface);
return mem;
vk_error:
{
GST_CAT_ERROR (GST_CAT_IO_SURFACE_VULKAN_MEMORY,
"Failed to allocate image memory %s", error->message);
g_clear_error (&error);
goto error;
}
error:
{
if (mem)
gst_memory_unref ((GstMemory *) mem);
return NULL;
}
}
GstIOSurfaceVulkanMemory *
gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
IOSurfaceRef surface, unsigned int /* MTLPixelFormat */ fmt,
GstVideoInfo * info, guint plane, gpointer user_data, GDestroyNotify notify)
{
return _io_surface_vulkan_memory_new (device, surface, fmt, info, plane,
user_data, notify);
}

77
sys/applemedia/iosurfacevulkanmemory.h Normal file
View file

@ -0,0 +1,77 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _GST_IO_SURFACE_VULKAN_MEMORY_H_
#define _GST_IO_SURFACE_VULKAN_MEMORY_H_
#include <IOSurface/IOSurfaceRef.h>
#include <IOSurface/IOSurfaceObjc.h>
#include <gst/gst.h>
#include <gst/gstallocator.h>
#include <gst/video/video.h>
#include <gst/vulkan/vulkan.h>
G_BEGIN_DECLS
#define GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR (gst_io_surface_vulkan_memory_allocator_get_type())
GType gst_io_surface_vulkan_memory_allocator_get_type(void);
#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
#define GST_IS_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR))
#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocator))
#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR, GstIOSurfaceVulkanMemoryAllocatorClass))
#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_CAST(obj) ((GstIOSurfaceVulkanMemoryAllocator *)(obj))
typedef struct _GstIOSurfaceVulkanMemory
{
GstVulkanImageMemory vulkan_mem;
IOSurfaceRef surface;
guint plane;
} GstIOSurfaceVulkanMemory;
#define GST_IO_SURFACE_VULKAN_MEMORY_ALLOCATOR_NAME "IOSurfaceVulkanMemory"
void gst_io_surface_vulkan_memory_init (void);
GstIOSurfaceVulkanMemory *
gst_io_surface_vulkan_memory_wrapped (GstVulkanDevice * device,
IOSurfaceRef surface,
unsigned int fmt, /* MTLPixelFormat */
GstVideoInfo * info,
guint plane,
gpointer user_data,
GDestroyNotify notify);
gboolean gst_is_io_surface_vulkan_memory (GstMemory * mem);
typedef struct _GstIOSurfaceVulkanMemoryAllocator
{
GstVulkanMemoryAllocator allocator;
} GstIOSurfaceVulkanMemoryAllocator;
typedef struct _GstIOSurfaceVulkanMemoryAllocatorClass
{
GstVulkanMemoryAllocatorClass parent_class;
} GstIOSurfaceVulkanMemoryAllocatorClass;
G_END_DECLS
#endif /* _GST_IO_SURFACE_MEMORY_H_ */

18
sys/applemedia/meson.build
View file

@ -68,12 +68,12 @@ if host_system == 'ios'
applemedia_objc_args += ['-fobjc-abi-version=2', '-fobjc-legacy-dispatch']
ios_media_dep = dependency('appleframeworks', modules : ['Foundation', 'AssetsLibrary'], required : applemedia_option)
applemedia_frameworks += [ios_media_dep]
iosurface_dep = dependency('IOSurface', required : applemedia_option)
applemedia_frameworks += [ios_media_dep, iosurface_dep]
else
applemedia_sources += [
'iosurfaceglmemory.c'
]
applemedia_objc_args += ['-mmacosx-version-min=10.8']
cocoa_dep = dependency('Cocoa', required : applemedia_option)
iosurface_dep = dependency('IOSurface', required : applemedia_option)
applemedia_opengl_dep = dependency('appleframeworks', modules : ['OpenGL'], required : applemedia_option)
@ -87,11 +87,25 @@ foreach framework : applemedia_frameworks
endif
endforeach
if gstvulkan_dep.found() and have_objcpp
moltenvk_dep = cc.find_library('MoltenVK', required : false)
metal_dep = dependency('appleframeworks', modules : ['Metal'], required : false)
if metal_dep.found() and moltenvk_dep.found() and cc.has_header ('MoltenVK/vk_mvk_moltenvk.h')
applemedia_frameworks += [moltenvk_dep, gstvulkan_dep, metal_dep]
applemedia_sources += [
'videotexturecache-vulkan.mm',
'iosurfacevulkanmemory.c',
]
applemedia_args += ['-DAPPLEMEDIA_MOLTENVK']
endif
endif
if applemedia_found_deps
gstapplemedia = library('gstapplemedia',
applemedia_sources,
c_args : gst_plugins_bad_args + applemedia_args,
objc_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
objcpp_args : gst_plugins_bad_args + applemedia_args + applemedia_objc_args,
link_args : noseh_link_args,
include_directories : [configinc, libsinc],
dependencies : [gstvideo_dep, gstaudio_dep, gstpbutils_dep, gst_dep, gstbase_dep, gstgl_dep] + applemedia_frameworks,

44
sys/applemedia/metal-helpers.h Normal file
View file

@ -0,0 +1,44 @@
/*
* GStreamer
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _APPLEMEDIA_METAL_HELPERS_H_
#define _APPLEMEDIA_METAL_HELPERS_H_
#include <gst/gst.h>
#include "corevideomemory.h"
#include "videotexturecache.h"
G_BEGIN_DECLS
VkFormat metal_format_to_vulkan (unsigned int fmt);
unsigned int video_info_to_metal_format (GstVideoInfo * info,
guint plane);
GstMemory * _create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
GstVideoInfo * info,
guint plane,
gsize size,
GstVideoTextureCache * cache);
void gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
IOSurfaceRef surface);
G_END_DECLS
#endif /* _APPLEMEDIA_METAL_HELPERS_H_ */

52
sys/applemedia/videotexturecache-vulkan.h Normal file
View file

@ -0,0 +1,52 @@
/*
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
#define __GST_CORE_VIDEO_TEXTURE_CACHE_VULKAN_H__
#include <gst/vulkan/vulkan.h>
#include "videotexturecache.h"
G_BEGIN_DECLS
#define GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN (gst_video_texture_cache_vulkan_get_type())
#define GST_VIDEO_TEXTURE_CACHE_VULKAN(o) (G_TYPE_CHECK_INSTANCE_CAST((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkan))
#define GST_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN(o) (G_TYPE_CHECK_INSTANCE_TYPE((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
#define GST_IS_VIDEO_TEXTURE_CACHE_VULKAN_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE((k), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN))
#define GST_VIDEO_TEXTURE_CACHE_VULKAN_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS((o), GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanClass))
GType gst_video_texture_cache_vulkan_get_type (void);
typedef struct _GstVideoTextureCacheVulkan
{
GstVideoTextureCache parent;
GstVulkanDevice *device;
} GstVideoTextureCacheVulkan;
typedef struct _GstVideoTextureCacheVulkanClass
{
GstVideoTextureCacheClass parent_class;
} GstVideoTextureCacheVulkanClass;
GstVideoTextureCache * gst_video_texture_cache_vulkan_new (GstVulkanDevice * device);
G_END_DECLS
#endif /* __GST_CORE_VIDEO_TEXTURE_CACHE_H__ */

301
sys/applemedia/videotexturecache-vulkan.mm Normal file
View file

@ -0,0 +1,301 @@
/*
* Copyright (C) 2019 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include <Metal/Metal.h>
#if !HAVE_IOS
#import <AppKit/AppKit.h>
#endif
#include "iosurfacevulkanmemory.h"
/* silence macor redefinition warnings */
#undef VK_USE_PLATFORM_MACOS_MVK
#undef VK_USE_PLATFORM_IOS_MVK
#include <MoltenVK/vk_mvk_moltenvk.h>
#include <MoltenVK/mvk_datatypes.h>
/* silence macro redefinition warnings */
#undef VK_USE_PLATFORM_MACOS_MVK
#undef VK_USE_PLATFORM_IOS_MVK
#include "coremediabuffer.h"
#include "corevideobuffer.h"
#include "vtutil.h"
#include "videotexturecache-vulkan.h"
#include "metal-helpers.h"
typedef struct _GstVideoTextureCacheVulkanPrivate
{
GstBufferPool *pool;
} GstVideoTextureCacheVulkanPrivate;
G_DEFINE_TYPE_WITH_PRIVATE (GstVideoTextureCacheVulkan, gst_video_texture_cache_vulkan, GST_TYPE_VIDEO_TEXTURE_CACHE);
#define GET_PRIV(instance) \
G_TYPE_INSTANCE_GET_PRIVATE (instance, GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN, GstVideoTextureCacheVulkanPrivate)
typedef struct _IOSTextureWrapper
{
CVMetalTextureCacheRef cache;
CVMetalTextureRef texture;
} IOSTextureWrapper;
enum
{
PROP_0,
PROP_DEVICE,
};
static GstMemory * gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size);
GstVideoTextureCache *
gst_video_texture_cache_vulkan_new (GstVulkanDevice * device)
{
g_return_val_if_fail (GST_IS_VULKAN_DEVICE (device), NULL);
return (GstVideoTextureCache *) g_object_new (GST_TYPE_VIDEO_TEXTURE_CACHE_VULKAN,
"device", device, NULL);
}
static void
gst_video_texture_cache_vulkan_finalize (GObject * object)
{
GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
#if 0
gst_buffer_pool_set_active (cache->pool, FALSE);
gst_object_unref (cache->pool);
#endif
gst_object_unref (cache_vulkan->device);
G_OBJECT_CLASS (gst_video_texture_cache_vulkan_parent_class)->finalize (object);
}
static void
gst_video_texture_cache_vulkan_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
switch (prop_id) {
case PROP_DEVICE:
cache_vulkan->device = (GstVulkanDevice *) g_value_dup_object (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_video_texture_cache_vulkan_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec)
{
GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
switch (prop_id) {
case PROP_DEVICE:
g_value_set_object (value, cache_vulkan->device);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_video_texture_cache_vulkan_constructed (GObject * object)
{
GstVideoTextureCacheVulkan *cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (object);
g_return_if_fail (GST_IS_VULKAN_DEVICE (cache_vulkan->device));
gst_io_surface_vulkan_memory_init ();
#if 0
cache->pool = GST_BUFFER_POOL (gst_vulkan_buffer_pool_new (ctx));
#endif
}
static void
gst_video_texture_cache_vulkan_init (GstVideoTextureCacheVulkan * cache_vulkan)
{
}
static void
gst_video_texture_cache_vulkan_class_init (GstVideoTextureCacheVulkanClass *klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstVideoTextureCacheClass *cache_class = (GstVideoTextureCacheClass *) klass;
gobject_class->set_property = gst_video_texture_cache_vulkan_set_property;
gobject_class->get_property = gst_video_texture_cache_vulkan_get_property;
gobject_class->constructed = gst_video_texture_cache_vulkan_constructed;
gobject_class->finalize = gst_video_texture_cache_vulkan_finalize;
g_object_class_install_property (gobject_class, PROP_DEVICE,
g_param_spec_object ("device", "device",
"Associated Vulkan device", GST_TYPE_VULKAN_DEVICE,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)));
cache_class->create_memory = gst_video_texture_cache_vulkan_create_memory;
}
static GstMemory *
gst_video_texture_cache_vulkan_create_memory (GstVideoTextureCache * cache,
GstAppleCoreVideoPixelBuffer *gpixbuf, guint plane, gsize size)
{
return (GstMemory *) gpixbuf;
}
VkFormat
metal_format_to_vulkan (unsigned int fmt)
{
MTLPixelFormat mtl_fmt = (MTLPixelFormat) fmt;
switch (mtl_fmt) {
case MTLPixelFormatRGBA8Unorm:
return VK_FORMAT_R8G8B8A8_UNORM;
case MTLPixelFormatRG8Unorm:
return VK_FORMAT_R8G8_UNORM;
case MTLPixelFormatR8Unorm:
return VK_FORMAT_R8_UNORM;
default:
g_assert_not_reached ();
}
}
unsigned int
video_info_to_metal_format (GstVideoInfo * info, guint plane)
{
switch (GST_VIDEO_INFO_FORMAT (info)) {
case GST_VIDEO_FORMAT_BGRA:
return (unsigned int) MTLPixelFormatRGBA8Unorm;
case GST_VIDEO_FORMAT_NV12:
if (plane == 0)
return (unsigned int) MTLPixelFormatR8Unorm;
else
return (unsigned int) MTLPixelFormatRG8Unorm;
default:
g_assert_not_reached ();
}
}
GstMemory *
_create_vulkan_memory (GstAppleCoreVideoPixelBuffer * gpixbuf,
GstVideoInfo * info, guint plane, gsize size, GstVideoTextureCache * cache)
{
GstIOSurfaceVulkanMemory *mem;
CVPixelBufferRef pixel_buf = gpixbuf->buf;
IOSurfaceRef surface = CVPixelBufferGetIOSurface (pixel_buf);
GstVideoTextureCacheVulkan *cache_vulkan =
GST_VIDEO_TEXTURE_CACHE_VULKAN (cache);
MTLPixelFormat fmt = video_info_to_metal_format (info, plane);
CFRetain (pixel_buf);
mem = gst_io_surface_vulkan_memory_wrapped (cache_vulkan->device,
surface, fmt, info, plane, pixel_buf, (GDestroyNotify) CFRelease);
if (!mem)
return NULL;
return GST_MEMORY_CAST (mem);
}
typedef struct _IOSurfaceTextureWrapper
{
CVPixelBufferRef pixbuf;
gpointer texture; /* id<MTLTexture> */
} IOSurfaceTextureWrapper;
static void
free_texture_wrapper (IOSurfaceTextureWrapper * wrapper)
{
CFRelease (wrapper->pixbuf);
id<MTLTexture> tex = (__bridge_transfer id<MTLTexture>) wrapper->texture;
(void) tex;
g_free (wrapper);
}
static MTLTextureDescriptor *
gst_new_mtl_tex_descripter_from_memory (GstIOSurfaceVulkanMemory * memory)
{
GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
MTLTextureDescriptor* tex_desc = [MTLTextureDescriptor new];
tex_desc.pixelFormat = mvkMTLPixelFormatFromVkFormat (vk_mem->create_info.format);
tex_desc.textureType = mvkMTLTextureTypeFromVkImageType (vk_mem->create_info.imageType, 0, false);
tex_desc.width = vk_mem->create_info.extent.width;
tex_desc.height = vk_mem->create_info.extent.height;
tex_desc.depth = vk_mem->create_info.extent.depth;
tex_desc.mipmapLevelCount = vk_mem->create_info.mipLevels;
tex_desc.sampleCount = mvkSampleCountFromVkSampleCountFlagBits(vk_mem->create_info.samples);
tex_desc.arrayLength = vk_mem->create_info.arrayLayers;
tex_desc.usage = MTLTextureUsageShaderRead | MTLTextureUsagePixelFormatView;//mvkMTLTextureUsageFromVkImageUsageFlags(vk_mem->create_info.usage);
tex_desc.storageMode = MTLStorageModePrivate;//MTLStorageModeManaged; /* IOSurface == shared/managed */
tex_desc.cpuCacheMode = MTLCPUCacheModeDefaultCache;//mvkMTLCPUCacheModeFromVkMemoryPropertyFlags(vk_mem->vk_mem->properties);
return tex_desc;
}
void
gst_io_surface_vulkan_memory_set_surface (GstIOSurfaceVulkanMemory * memory,
IOSurfaceRef surface)
{
GstVulkanImageMemory *vk_mem = (GstVulkanImageMemory *) memory;
if (memory->surface) {
IOSurfaceDecrementUseCount (memory->surface);
} else {
g_assert (vk_mem->notify == (GDestroyNotify) CFRelease);
g_assert (vk_mem->user_data != NULL);
}
memory->surface = surface;
if (surface) {
id<MTLDevice> mtl_dev = nil;
id<MTLTexture> texture = nil;
VkPhysicalDevice gpu;
IOSurfaceIncrementUseCount (surface);
gpu = gst_vulkan_device_get_physical_device (vk_mem->device);
vkGetMTLDeviceMVK (gpu, &mtl_dev);
/* We cannot use vkUseIOSurfaceMVK() for multi-planer as MoltenVK does not
* support them. */
MTLTextureDescriptor *tex_desc = gst_new_mtl_tex_descripter_from_memory (memory);
texture = [mtl_dev newTextureWithDescriptor:tex_desc iosurface:surface plane:memory->plane];
IOSurfaceTextureWrapper *texture_data = g_new0 (IOSurfaceTextureWrapper, 1);
texture_data->pixbuf = (CVPixelBufferRef) vk_mem->user_data;
texture_data->texture = (__bridge_retained gpointer) texture;
VkResult err = vkSetMTLTextureMVK (memory->vulkan_mem.image, texture);
GST_DEBUG ("bound texture %p to image %p: 0x%x", texture, memory->vulkan_mem.image,
err);
vk_mem->user_data = texture_data;
vk_mem->notify = (GDestroyNotify) free_texture_wrapper;
}
}

127
sys/applemedia/vtdec.c
View file

@ -45,6 +45,9 @@
#include "corevideobuffer.h"
#include "coremediabuffer.h"
#include "videotexturecache-gl.h"
#if defined(APPLEMEDIA_MOLTENVK)
#include "videotexturecache-vulkan.h"
#endif
GST_DEBUG_CATEGORY_STATIC (gst_vtdec_debug_category);
#define GST_CAT_DEFAULT gst_vtdec_debug_category
@ -111,11 +114,21 @@ const CFStringRef
CFSTR ("RequireHardwareAcceleratedVideoDecoder");
#endif
#if defined(APPLEMEDIA_MOLTENVK)
#define VIDEO_SRC_CAPS \
GST_VIDEO_CAPS_MAKE("NV12") ";" \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
"NV12") ", " \
"texture-target = (string) rectangle;"
"texture-target = (string) rectangle ; " \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE,\
"NV12")
#else
#define VIDEO_SRC_CAPS \
GST_VIDEO_CAPS_MAKE("NV12") ";" \
GST_VIDEO_CAPS_MAKE_WITH_FEATURES(GST_CAPS_FEATURE_MEMORY_GL_MEMORY,\
"NV12") ", " \
"texture-target = (string) rectangle "
#endif
G_DEFINE_TYPE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER);
@ -210,20 +223,22 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
CFRelease (vtdec->format_description);
vtdec->format_description = NULL;
#if defined(APPLEMEDIA_MOLTENVK)
gst_clear_object (&vtdec->device);
gst_clear_object (&vtdec->instance);
#endif
GST_DEBUG_OBJECT (vtdec, "stop");
return TRUE;
}
static void
setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
setup_texture_cache (GstVtdec * vtdec)
{
GstVideoCodecState *output_state;
g_return_if_fail (vtdec->texture_cache == NULL);
output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
vtdec->texture_cache = gst_video_texture_cache_gl_new (context);
gst_video_texture_cache_set_format (vtdec->texture_cache,
GST_VIDEO_FORMAT_NV12, output_state->caps);
gst_video_codec_state_unref (output_state);
@ -240,7 +255,10 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
GstVtdec *vtdec;
OSStatus err = noErr;
GstCapsFeatures *features = NULL;
gboolean output_textures;
gboolean output_textures = FALSE;
#if defined(APPLEMEDIA_MOLTENVK)
gboolean output_vulkan = FALSE;
#endif
vtdec = GST_VTDEC (decoder);
if (vtdec->session)
@ -296,6 +314,12 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
GST_GL_TEXTURE_TARGET_2D_STR,
#endif
NULL);
#if defined(APPLEMEDIA_MOLTENVK)
output_vulkan =
gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_VULKAN_IMAGE);
#endif
}
gst_caps_unref (caps);
@ -323,33 +347,83 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
}
}
if (vtdec->texture_cache != NULL && !output_textures) {
if (vtdec->texture_cache != NULL
&& ((GST_IS_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache)
&& !output_textures)
#if defined(APPLEMEDIA_MOLTENVK)
|| (GST_IS_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache)
&& !output_vulkan)
#endif
)) {
g_object_unref (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
if (err == noErr && output_textures) {
GstVideoTextureCacheGL *cache_gl = NULL;
if (err == noErr) {
if (output_textures) {
GstVideoTextureCacheGL *cache_gl = NULL;
if (vtdec->texture_cache)
cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
if (vtdec->texture_cache)
cache_gl = GST_VIDEO_TEXTURE_CACHE_GL (vtdec->texture_cache);
/* call this regardless of whether caps have changed or not since a new
* local context could have become available
*/
if (!vtdec->ctxh)
vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
gst_gl_context_helper_ensure_context (vtdec->ctxh);
/* call this regardless of whether caps have changed or not since a new
* local context could have become available
*/
if (!vtdec->ctxh)
vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
gst_gl_context_helper_ensure_context (vtdec->ctxh);
GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
GST_INFO_OBJECT (vtdec, "pushing GL textures, context %p old context %p",
vtdec->ctxh->context, cache_gl ? cache_gl->ctx : NULL);
if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
g_object_unref (vtdec->texture_cache);
vtdec->texture_cache = NULL;
if (cache_gl && cache_gl->ctx != vtdec->ctxh->context) {
g_object_unref (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
if (!vtdec->texture_cache) {
vtdec->texture_cache =
gst_video_texture_cache_gl_new (vtdec->ctxh->context);
setup_texture_cache (vtdec);
}
}
if (!vtdec->texture_cache)
setup_texture_cache (vtdec, vtdec->ctxh->context);
#if defined(APPLEMEDIA_MOLTENVK)
if (output_vulkan) {
GstVideoTextureCacheVulkan *cache_vulkan = NULL;
if (vtdec->texture_cache)
cache_vulkan = GST_VIDEO_TEXTURE_CACHE_VULKAN (vtdec->texture_cache);
gst_vulkan_ensure_element_data (GST_ELEMENT (vtdec), NULL,
&vtdec->instance);
if (!gst_vulkan_device_run_context_query (GST_ELEMENT (vtdec),
&vtdec->device)) {
GError *error = NULL;
GST_DEBUG_OBJECT (vtdec, "No device retrieved from peer elements");
if (!(vtdec->device =
gst_vulkan_instance_create_device (vtdec->instance, &error))) {
GST_ELEMENT_ERROR (vtdec, RESOURCE, NOT_FOUND,
("Failed to create vulkan device"), ("%s", error->message));
g_clear_error (&error);
return FALSE;
}
}
GST_INFO_OBJECT (vtdec, "pushing vulkan images, device %" GST_PTR_FORMAT
" old device %" GST_PTR_FORMAT, vtdec->device,
cache_vulkan ? cache_vulkan->device : NULL);
if (cache_vulkan && cache_vulkan->device != vtdec->device) {
g_object_unref (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
if (!vtdec->texture_cache) {
vtdec->texture_cache =
gst_video_texture_cache_vulkan_new (vtdec->device);
setup_texture_cache (vtdec);
}
}
#endif
}
if (prevcaps)
@ -1043,6 +1117,11 @@ gst_vtdec_set_context (GstElement * element, GstContext * context)
vtdec->ctxh = gst_gl_context_helper_new (element);
gst_gl_handle_set_context (element, context,
&vtdec->ctxh->display, &vtdec->ctxh->other_context);
#if defined (APPLEMEDIA_MOLTENVK)
gst_vulkan_handle_set_context (element, context, NULL, &vtdec->instance);
#endif
GST_ELEMENT_CLASS (gst_vtdec_parent_class)->set_context (element, context);
}

8
sys/applemedia/vtdec.h
View file

@ -27,6 +27,9 @@
#include <VideoToolbox/VideoToolbox.h>
#include "videotexturecache.h"
#include "glcontexthelper.h"
#if defined(APPLEMEDIA_MOLTENVK)
#include <gst/vulkan/vulkan.h>
#endif
G_BEGIN_DECLS
@ -51,6 +54,11 @@ struct _GstVtdec
GstVideoTextureCache *texture_cache;
GstGLContextHelper *ctxh;
#if defined(APPLEMEDIA_MOLTENVK)
GstVulkanInstance *instance;
GstVulkanDevice *device;
#endif
gboolean require_hardware;
};