From cdfe90aaba18949ed5f5ae962a1d89aa2ef196a6 Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Fri, 9 Jan 2015 00:16:17 +1100 Subject: [PATCH] avfvideosrc: implement the GL texture upload meta --- sys/applemedia/Makefile.am | 18 ++-- sys/applemedia/avfvideosrc.m | 57 +++++++++++++ sys/applemedia/coremediabuffer.c | 2 + sys/applemedia/corevideotexturecache.h | 45 ++++++++++ sys/applemedia/corevideotexturecache.m | 113 +++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 sys/applemedia/corevideotexturecache.h create mode 100644 sys/applemedia/corevideotexturecache.m diff --git a/sys/applemedia/Makefile.am b/sys/applemedia/Makefile.am index e05c092f2a..9fd1fb2c7c 100644 --- a/sys/applemedia/Makefile.am +++ b/sys/applemedia/Makefile.am @@ -5,6 +5,7 @@ libgstapplemedia_la_SOURCES = \ vtutil.c \ corevideobuffer.c \ coremediabuffer.c \ + corevideotexturecache.m \ atdec.c libgstapplemedia_la_CPPFLAGS = \ @@ -15,11 +16,15 @@ libgstapplemedia_la_CPPFLAGS = \ -Dgst_core_video_buffer_get_type=gst_core_video_buffer_priv_get_type libgstapplemedia_la_CFLAGS = \ + -I$(top_srcdir)/gst-libs \ + -I$(top_builddir)/gst-libs \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) libgstapplemedia_la_OBJCFLAGS = \ + -I$(top_srcdir)/gst-libs \ + -I$(top_builddir)/gst-libs \ $(GST_OBJCFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) @@ -28,12 +33,13 @@ if HAVE_IOS libgstapplemedia_la_OBJCFLAGS += -fobjc-abi-version=2 -fobjc-legacy-dispatch endif -libgstapplemedia_la_LIBADD = \ - $(GST_BASE_LIBS) \ - $(GST_PLUGINS_BASE_LIBS) \ - -lgstvideo-$(GST_API_VERSION) \ - -lgstaudio-$(GST_API_VERSION) \ - -lgstpbutils-$(GST_API_VERSION) \ +libgstapplemedia_la_LIBADD = \ + $(top_builddir)/gst-libs/gst/gl/libgstgl-$(GST_API_VERSION).la \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + -lgstvideo-$(GST_API_VERSION) \ + -lgstaudio-$(GST_API_VERSION) \ + -lgstpbutils-$(GST_API_VERSION) \ $(GMODULE_NO_EXPORT_LIBS) libgstapplemedia_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) --tag=CC diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m index f83c8c06f1..0f375c26e2 100644 --- a/sys/applemedia/avfvideosrc.m +++ b/sys/applemedia/avfvideosrc.m @@ -25,7 +25,9 @@ #import #include +#include #include "coremediabuffer.h" +#include "corevideotexturecache.h" #define DEFAULT_DEVICE_INDEX -1 #define DEFAULT_DO_STATS FALSE @@ -52,6 +54,10 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", "framerate = " GST_VIDEO_FPS_RANGE ", " "width = " GST_VIDEO_SIZE_RANGE ", " "height = " GST_VIDEO_SIZE_RANGE "; " + + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + "RGBA") "; " )); typedef enum _QueueState { @@ -101,6 +107,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC); BOOL captureScreenMouseClicks; BOOL useVideoMeta; + GstCoreVideoTextureCache *textureCache; } - (id)init; @@ -161,6 +168,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer captureScreenCursor = NO; captureScreenMouseClicks = NO; useVideoMeta = NO; + textureCache = NULL; #if !HAVE_IOS displayId = kCGDirectMainDisplay; #endif @@ -321,6 +329,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer }); } + #define GST_AVF_CAPS_NEW(format, w, h, fps_n, fps_d) \ (gst_caps_new_simple ("video/x-raw", \ "width", G_TYPE_INT, w, \ @@ -343,6 +352,9 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer case kCVPixelFormatType_32BGRA: /* BGRA */ gst_format = GST_VIDEO_FORMAT_BGRA; break; + case kCVPixelFormatType_32RGBA: /* RGBA */ + gst_format = GST_VIDEO_FORMAT_RGBA; + break; case kCVPixelFormatType_422YpCbCr8_yuvs: /* yuvs */ gst_format = GST_VIDEO_FORMAT_YUY2; break; @@ -383,6 +395,12 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer GstVideoFormat gst_format = [self getGstVideoFormat:pixel_format]; if (gst_format != GST_VIDEO_FORMAT_UNKNOWN) gst_caps_append (result, GST_AVF_CAPS_NEW (gst_format, dimensions.width, dimensions.height, fps_n, fps_d)); + + if (gst_format == GST_VIDEO_FORMAT_BGRA) { + GstCaps *rgba_caps = GST_AVF_CAPS_NEW (GST_VIDEO_FORMAT_RGBA, dimensions.width, dimensions.height, fps_n, fps_d); + gst_caps_set_features (rgba_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL)); + gst_caps_append (result, rgba_caps); + } } } } @@ -622,6 +640,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer case GST_VIDEO_FORMAT_YUY2: newformat = kCVPixelFormatType_422YpCbCr8_yuvs; break; + case GST_VIDEO_FORMAT_RGBA: + /* In order to do RGBA, we negotiate BGRA (since RGBA is not supported + * if not in textures) and then we get RGBA textures via + * CVOpenGL*TextureCacheCreateTextureFromImage. Computers. */ case GST_VIDEO_FORMAT_BGRA: newformat = kCVPixelFormatType_32BGRA; break; @@ -682,6 +704,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer bufQueue = nil; inputClock = nil; + if (textureCache) + gst_core_video_texture_cache_free (textureCache); + textureCache = NULL; + return YES; } @@ -713,6 +739,20 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer useVideoMeta = gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + guint idx; + if (gst_query_find_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { + GstGLContext *context; + const GstStructure *upload_meta_params; + + gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); + if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, &context, NULL) && context) { + textureCache = gst_core_video_texture_cache_new (context); + gst_object_unref (context); + } + } + return YES; } @@ -824,8 +864,25 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer } *buf = gst_core_media_buffer_new (sbuf, useVideoMeta); + if (format == GST_VIDEO_FORMAT_RGBA) { + /* So now buf contains BGRA data (!) . Since downstream is actually going to + * use the GL upload meta to get RGBA textures (??), we need to override the + * VideoMeta format (!!!). Yes this is confusing, see setCaps: */ + GstVideoMeta *video_meta = gst_buffer_get_video_meta (*buf); + if (video_meta) { + video_meta->format = format; + } + } CFRelease (sbuf); + if (textureCache != NULL) { + GstVideoGLTextureType texture_types[4] = {GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0}; + gst_buffer_add_video_gl_texture_upload_meta (*buf, + GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, + 1, texture_types, + gst_core_video_texture_cache_upload, textureCache, NULL, NULL); + } + GST_BUFFER_OFFSET (*buf) = offset++; GST_BUFFER_OFFSET_END (*buf) = GST_BUFFER_OFFSET (buf) + 1; GST_BUFFER_TIMESTAMP (*buf) = timestamp; diff --git a/sys/applemedia/coremediabuffer.c b/sys/applemedia/coremediabuffer.c index 74b814b9d3..ac9703dfaf 100644 --- a/sys/applemedia/coremediabuffer.c +++ b/sys/applemedia/coremediabuffer.c @@ -76,6 +76,8 @@ gst_core_media_buffer_get_video_format (OSType format) return GST_VIDEO_FORMAT_UYVY; case kCVPixelFormatType_32BGRA: return GST_VIDEO_FORMAT_BGRA; + case kCVPixelFormatType_32RGBA: + return GST_VIDEO_FORMAT_RGBA; default: GST_WARNING ("Unknown OSType format: %d", (gint) format); return GST_VIDEO_FORMAT_UNKNOWN; diff --git a/sys/applemedia/corevideotexturecache.h b/sys/applemedia/corevideotexturecache.h new file mode 100644 index 0000000000..2e880ee155 --- /dev/null +++ b/sys/applemedia/corevideotexturecache.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2015 Alessandro Decina + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_CORE_VIDEO_TEXTURE_CACHE_H__ +#define __GST_CORE_VIDEO_TEXTURE_CACHE_H__ + +#include +#include +#include "CoreVideo/CoreVideo.h" + +G_BEGIN_DECLS + +typedef struct _GstCoreVideoTextureCache +{ + GstGLContext *ctx; +#if !HAVE_IOS + CVOpenGLTextureCacheRef cache; +#else + CVOpenGLESTextureCacheRef cache; +#endif +} GstCoreVideoTextureCache; + +GstCoreVideoTextureCache *gst_core_video_texture_cache_new (GstGLContext * ctx); +void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache); +gboolean gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]); + +G_END_DECLS + +#endif /* __GST_CORE_VIDEO_TEXTURE_CACHE_H__ */ diff --git a/sys/applemedia/corevideotexturecache.m b/sys/applemedia/corevideotexturecache.m new file mode 100644 index 0000000000..821aca5c96 --- /dev/null +++ b/sys/applemedia/corevideotexturecache.m @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2010 Ole André Vadla Ravnås + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#if !HAVE_IOS +#import +#endif +#include "corevideotexturecache.h" +#include "coremediabuffer.h" + +GstCoreVideoTextureCache * +gst_core_video_texture_cache_new (GstGLContext * ctx) +{ + CVReturn err; + + g_return_val_if_fail (ctx != NULL, NULL); + + GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1); + cache->ctx = gst_object_ref (ctx); + +#if !HAVE_IOS + CGLPixelFormatAttribute attribs[1] = { 0 }; + int numPixelFormats; + CGLPixelFormatObj pixelFormat; + CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5 + NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx); + err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL, + [platform_ctx CGLContextObj], pixelFormat, + NULL, &cache->cache); +#else + err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL, + (CVEAGLContext) gst_gl_context_get_gl_context (ctx), + NULL, &cache->cache); +#endif + + return cache; +} + +void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) +{ + g_return_if_fail (cache != NULL); + +#if !HAVE_IOS + CVOpenGLTextureCacheRelease (cache->cache); +#else + /* FIXME: how do we release ->cache ? */ +#endif + gst_object_unref (cache->ctx); + g_free (cache); +} + +gboolean +gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]) +{ + g_return_val_if_fail (meta != NULL, FALSE); + + GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data; + const GstGLFuncs *gl = cache->ctx->gl_vtable; +#if !HAVE_IOS + CVOpenGLTextureRef texture = NULL; +#else + CVOpenGLESTextureRef texture = NULL; +#endif + GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer); + GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer, + gst_core_media_meta_api_get_type ()); +#if !HAVE_IOS + CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, cv_meta->pixel_buf, NULL, &texture); +#else + CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, + cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D, + GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE, + 0, &texture); +#endif + GLuint fboId; + gl->GenFramebuffers (1, &fboId); + gl->BindFramebuffer (GL_FRAMEBUFFER, fboId); + + gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, +#if !HAVE_IOS + CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture), +#else + CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture), +#endif + 0); + gl->BindTexture (GL_TEXTURE_2D, texture_id[0]); + gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0); + gl->BindTexture (GL_TEXTURE_2D, 0); + gl->BindFramebuffer (GL_FRAMEBUFFER, 0); + gl->DeleteFramebuffers (1, &fboId); + + return TRUE; +}