mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2025-02-17 03:35:21 +00:00
Code Issues 1 Projects Releases Wiki Activity

avfvideosrc: implement the GL texture upload meta

Browse source
This commit is contained in:
Alessandro Decina 2015-01-09 00:16:17 +11:00
parent 39e6f621b3
commit cdfe90aaba
5 changed files with 229 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
sys/applemedia/Makefile.am
View file

@ -5,6 +5,7 @@ libgstapplemedia_la_SOURCES = \
vtutil.c \
corevideobuffer.c \
coremediabuffer.c \
corevideotexturecache.m \
atdec.c
libgstapplemedia_la_CPPFLAGS = \
@ -15,11 +16,15 @@ libgstapplemedia_la_CPPFLAGS = \
-Dgst_core_video_buffer_get_type=gst_core_video_buffer_priv_get_type
libgstapplemedia_la_CFLAGS = \
-I$(top_srcdir)/gst-libs \
-I$(top_builddir)/gst-libs \
$(GST_CFLAGS) \
$(GST_BASE_CFLAGS) \
$(GST_PLUGINS_BASE_CFLAGS)
libgstapplemedia_la_OBJCFLAGS = \
-I$(top_srcdir)/gst-libs \
-I$(top_builddir)/gst-libs \
$(GST_OBJCFLAGS) \
$(GST_BASE_CFLAGS) \
$(GST_PLUGINS_BASE_CFLAGS)
@ -28,12 +33,13 @@ if HAVE_IOS
libgstapplemedia_la_OBJCFLAGS += -fobjc-abi-version=2 -fobjc-legacy-dispatch
endif
libgstapplemedia_la_LIBADD = \
$(GST_BASE_LIBS) \
$(GST_PLUGINS_BASE_LIBS) \
-lgstvideo-$(GST_API_VERSION) \
-lgstaudio-$(GST_API_VERSION) \
-lgstpbutils-$(GST_API_VERSION) \
libgstapplemedia_la_LIBADD = \
$(top_builddir)/gst-libs/gst/gl/libgstgl-$(GST_API_VERSION).la \
$(GST_BASE_LIBS) \
$(GST_PLUGINS_BASE_LIBS) \
-lgstvideo-$(GST_API_VERSION) \
-lgstaudio-$(GST_API_VERSION) \
-lgstpbutils-$(GST_API_VERSION) \
$(GMODULE_NO_EXPORT_LIBS)
libgstapplemedia_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) --tag=CC

57
sys/applemedia/avfvideosrc.m
View file

@ -25,7 +25,9 @@
#import <AVFoundation/AVFoundation.h>
#include <gst/video/video.h>
#include <gst/gl/gstglcontext.h>
#include "coremediabuffer.h"
#include "corevideotexturecache.h"
#define DEFAULT_DEVICE_INDEX -1
#define DEFAULT_DO_STATS FALSE
@ -52,6 +54,10 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
"framerate = " GST_VIDEO_FPS_RANGE ", "
"width = " GST_VIDEO_SIZE_RANGE ", "
"height = " GST_VIDEO_SIZE_RANGE "; "
GST_VIDEO_CAPS_MAKE_WITH_FEATURES
(GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META,
"RGBA") "; "
));
typedef enum _QueueState {
@ -101,6 +107,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
BOOL captureScreenMouseClicks;
BOOL useVideoMeta;
GstCoreVideoTextureCache *textureCache;
}
- (id)init;
@ -161,6 +168,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
captureScreenCursor = NO;
captureScreenMouseClicks = NO;
useVideoMeta = NO;
textureCache = NULL;
#if !HAVE_IOS
displayId = kCGDirectMainDisplay;
#endif
@ -321,6 +329,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
});
}
#define GST_AVF_CAPS_NEW(format, w, h, fps_n, fps_d) \
(gst_caps_new_simple ("video/x-raw", \
"width", G_TYPE_INT, w, \
@ -343,6 +352,9 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
case kCVPixelFormatType_32BGRA: /* BGRA */
gst_format = GST_VIDEO_FORMAT_BGRA;
break;
case kCVPixelFormatType_32RGBA: /* RGBA */
gst_format = GST_VIDEO_FORMAT_RGBA;
break;
case kCVPixelFormatType_422YpCbCr8_yuvs: /* yuvs */
gst_format = GST_VIDEO_FORMAT_YUY2;
break;
@ -383,6 +395,12 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
GstVideoFormat gst_format = [self getGstVideoFormat:pixel_format];
if (gst_format != GST_VIDEO_FORMAT_UNKNOWN)
gst_caps_append (result, GST_AVF_CAPS_NEW (gst_format, dimensions.width, dimensions.height, fps_n, fps_d));
if (gst_format == GST_VIDEO_FORMAT_BGRA) {
GstCaps *rgba_caps = GST_AVF_CAPS_NEW (GST_VIDEO_FORMAT_RGBA, dimensions.width, dimensions.height, fps_n, fps_d);
gst_caps_set_features (rgba_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL));
gst_caps_append (result, rgba_caps);
}
}
}
}
@ -622,6 +640,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
case GST_VIDEO_FORMAT_YUY2:
newformat = kCVPixelFormatType_422YpCbCr8_yuvs;
break;
case GST_VIDEO_FORMAT_RGBA:
/* In order to do RGBA, we negotiate BGRA (since RGBA is not supported
* if not in textures) and then we get RGBA textures via
* CVOpenGL*TextureCacheCreateTextureFromImage. Computers. */
case GST_VIDEO_FORMAT_BGRA:
newformat = kCVPixelFormatType_32BGRA;
break;
@ -682,6 +704,10 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
bufQueue = nil;
inputClock = nil;
if (textureCache)
gst_core_video_texture_cache_free (textureCache);
textureCache = NULL;
return YES;
}
@ -713,6 +739,20 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
useVideoMeta = gst_query_find_allocation_meta (query,
GST_VIDEO_META_API_TYPE, NULL);
guint idx;
if (gst_query_find_allocation_meta (query,
GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) {
GstGLContext *context;
const GstStructure *upload_meta_params;
gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params);
if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext",
GST_GL_TYPE_CONTEXT, &context, NULL) && context) {
textureCache = gst_core_video_texture_cache_new (context);
gst_object_unref (context);
}
}
return YES;
}
@ -824,8 +864,25 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
}
*buf = gst_core_media_buffer_new (sbuf, useVideoMeta);
if (format == GST_VIDEO_FORMAT_RGBA) {
/* So now buf contains BGRA data (!) . Since downstream is actually going to
* use the GL upload meta to get RGBA textures (??), we need to override the
* VideoMeta format (!!!). Yes this is confusing, see setCaps: */
GstVideoMeta *video_meta = gst_buffer_get_video_meta (*buf);
if (video_meta) {
video_meta->format = format;
}
}
CFRelease (sbuf);
if (textureCache != NULL) {
GstVideoGLTextureType texture_types[4] = {GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0};
gst_buffer_add_video_gl_texture_upload_meta (*buf,
GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL,
1, texture_types,
gst_core_video_texture_cache_upload, textureCache, NULL, NULL);
}
GST_BUFFER_OFFSET (*buf) = offset++;
GST_BUFFER_OFFSET_END (*buf) = GST_BUFFER_OFFSET (buf) + 1;
GST_BUFFER_TIMESTAMP (*buf) = timestamp;

2
sys/applemedia/coremediabuffer.c
View file

@ -76,6 +76,8 @@ gst_core_media_buffer_get_video_format (OSType format)
return GST_VIDEO_FORMAT_UYVY;
case kCVPixelFormatType_32BGRA:
return GST_VIDEO_FORMAT_BGRA;
case kCVPixelFormatType_32RGBA:
return GST_VIDEO_FORMAT_RGBA;
default:
GST_WARNING ("Unknown OSType format: %d", (gint) format);
return GST_VIDEO_FORMAT_UNKNOWN;

45
sys/applemedia/corevideotexturecache.h Normal file
View file

@ -0,0 +1,45 @@
/*
* Copyright (C) 2015 Alessandro Decina <twi@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_CORE_VIDEO_TEXTURE_CACHE_H__
#define __GST_CORE_VIDEO_TEXTURE_CACHE_H__
#include <gst/video/gstvideometa.h>
#include <gst/gl/gstglcontext.h>
#include "CoreVideo/CoreVideo.h"
G_BEGIN_DECLS
typedef struct _GstCoreVideoTextureCache
{
GstGLContext *ctx;
#if !HAVE_IOS
CVOpenGLTextureCacheRef cache;
#else
CVOpenGLESTextureCacheRef cache;
#endif
} GstCoreVideoTextureCache;
GstCoreVideoTextureCache *gst_core_video_texture_cache_new (GstGLContext * ctx);
void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache);
gboolean gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]);
G_END_DECLS
#endif /* __GST_CORE_VIDEO_TEXTURE_CACHE_H__ */

113
sys/applemedia/corevideotexturecache.m Normal file
View file

@ -0,0 +1,113 @@
/*
* Copyright (C) 2010 Ole André Vadla Ravnås <oleavr@soundrop.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#if !HAVE_IOS
#import <AppKit/AppKit.h>
#endif
#include "corevideotexturecache.h"
#include "coremediabuffer.h"
GstCoreVideoTextureCache *
gst_core_video_texture_cache_new (GstGLContext * ctx)
{
CVReturn err;
g_return_val_if_fail (ctx != NULL, NULL);
GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1);
cache->ctx = gst_object_ref (ctx);
#if !HAVE_IOS
CGLPixelFormatAttribute attribs[1] = { 0 };
int numPixelFormats;
CGLPixelFormatObj pixelFormat;
CGLChoosePixelFormat (attribs, &pixelFormat, &numPixelFormats); // 5
NSOpenGLContext *platform_ctx = (NSOpenGLContext *) gst_gl_context_get_gl_context (ctx);
err = CVOpenGLTextureCacheCreate (kCFAllocatorDefault, NULL,
[platform_ctx CGLContextObj], pixelFormat,
NULL, &cache->cache);
#else
err = CVOpenGLESTextureCacheCreate (kCFAllocatorDefault, NULL,
(CVEAGLContext) gst_gl_context_get_gl_context (ctx),
NULL, &cache->cache);
#endif
return cache;
}
void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache)
{
g_return_if_fail (cache != NULL);
#if !HAVE_IOS
CVOpenGLTextureCacheRelease (cache->cache);
#else
/* FIXME: how do we release ->cache ? */
#endif
gst_object_unref (cache->ctx);
g_free (cache);
}
gboolean
gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4])
{
g_return_val_if_fail (meta != NULL, FALSE);
GstCoreVideoTextureCache *cache = (GstCoreVideoTextureCache *) meta->user_data;
const GstGLFuncs *gl = cache->ctx->gl_vtable;
#if !HAVE_IOS
CVOpenGLTextureRef texture = NULL;
#else
CVOpenGLESTextureRef texture = NULL;
#endif
GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer);
GstCoreMediaMeta *cv_meta = (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer,
gst_core_media_meta_api_get_type ());
#if !HAVE_IOS
CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
cache->cache, cv_meta->pixel_buf, NULL, &texture);
#else
CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault,
cache->cache, cv_meta->image_buf, NULL, GL_TEXTURE_2D,
GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE,
0, &texture);
#endif
GLuint fboId;
gl->GenFramebuffers (1, &fboId);
gl->BindFramebuffer (GL_FRAMEBUFFER, fboId);
gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
#if !HAVE_IOS
CVOpenGLTextureGetTarget(texture), CVOpenGLTextureGetName(texture),
#else
CVOpenGLESTextureGetTarget(texture), CVOpenGLESTextureGetName(texture),
#endif
0);
gl->BindTexture (GL_TEXTURE_2D, texture_id[0]);
gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, video_meta->height, 0);
gl->BindTexture (GL_TEXTURE_2D, 0);
gl->BindFramebuffer (GL_FRAMEBUFFER, 0);
gl->DeleteFramebuffers (1, &fboId);
return TRUE;
}