mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-07-02 12:45:51 +00:00
Code Issues 1 Projects Releases Wiki Activity

applemedia: refactor GL context code

Browse source 
Rework the GL context code. Now both avfvideosrc and vtdec can create an
internal GL context for pushing textures. Both elements will still try to
use/switch to a local context where available (including after RECONFIGURE
events).
This commit is contained in:
Alessandro Decina 2016-01-29 15:07:59 +11:00
parent 0891b90111
commit 045a935871
6 changed files with 251 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
sys/applemedia/Makefile.am
View file

@ -6,8 +6,9 @@ libgstapplemedia_la_SOURCES = \
corevideomemory.c \
corevideobuffer.c \
coremediabuffer.c \
videotexturecache.m \
atdec.c
videotexturecache.m \
atdec.c \
glcontexthelper.c
libgstapplemedia_la_CPPFLAGS = \
-Dgst_core_media_buffer_new=gst_core_media_buffer_priv_new \
@ -74,7 +75,8 @@ noinst_HEADERS = \
videotexturecache.h \
atdec.h \
iosassetsrc.h \
avfassetsrc.h
avfassetsrc.h \
glcontexthelper.h
if HAVE_IOS

144
sys/applemedia/avfvideosrc.m
View file

@ -22,6 +22,7 @@
#endif
#include "avfvideosrc.h"
#include "glcontexthelper.h"
#import <AVFoundation/AVFoundation.h>
#if !HAVE_IOS
@ -114,6 +115,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
BOOL captureScreenMouseClicks;
BOOL useVideoMeta;
GstGLContextHelper *ctxh;
GstVideoTextureCache *textureCache;
}
@ -150,6 +152,7 @@ G_DEFINE_TYPE (GstAVFVideoSrc, gst_avf_video_src, GST_TYPE_PUSH_SRC);
- (GstStateChangeReturn)changeState:(GstStateChange)transition;
- (GstFlowReturn)create:(GstBuffer **)buf;
- (GstCaps *)fixate:(GstCaps *)caps;
- (BOOL)decideAllocation:(GstQuery *)query;
- (void)updateStatistics;
- (void)captureOutput:(AVCaptureOutput *)captureOutput
didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
@ -180,7 +183,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
captureScreenMouseClicks = NO;
useVideoMeta = NO;
textureCache = NULL;
ctxh = gst_gl_context_helper_new (element);
mainQueue =
dispatch_queue_create ("org.freedesktop.gstreamer.avfvideosrc.main", NULL);
workerQueue =
@ -751,24 +754,8 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
dictionaryWithObject:[NSNumber numberWithInt:newformat]
forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
if (caps)
gst_caps_unref (caps);
caps = gst_caps_copy (new_caps);
if (textureCache)
gst_video_texture_cache_free (textureCache);
textureCache = NULL;
GstCapsFeatures *features = gst_caps_get_features (caps, 0);
if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
GstGLContext *context = find_gl_context (element);
textureCache = gst_video_texture_cache_new (context);
gst_video_texture_cache_set_format (textureCache, format, caps);
gst_object_unref (context);
}
GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT
", pushing textures %d", caps, textureCache != NULL);
gst_caps_replace (&caps, new_caps);
GST_INFO_OBJECT (element, "configured caps %"GST_PTR_FORMAT, caps);
if (![session isRunning])
[session startRunning];
@ -808,9 +795,13 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
bufQueue = nil;
if (textureCache)
gst_video_texture_cache_free (textureCache);
gst_video_texture_cache_free (textureCache);
textureCache = NULL;
if (ctxh)
gst_gl_context_helper_free (ctxh);
ctxh = NULL;
return YES;
}
@ -971,81 +962,11 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
return GST_FLOW_OK;
}
static GstGLContext *
query_gl_local_context (GstPad *srcpad)
{
GstGLContext *gl_context = NULL;
GstContext *context = NULL;
GstQuery *query;
query = gst_query_new_context ("gst.gl.local_context");
if (gst_pad_peer_query (srcpad, query)) {
gst_query_parse_context (query, &context);
if (context) {
const GstStructure *s = gst_context_get_structure (context);
gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
}
}
gst_query_unref (query);
return gl_context;
}
static GstGLContext *
find_gl_app_context (GstElement *element)
{
GstGLContext *gl_context = NULL;
GstContext *context = gst_element_get_context (element, "gst.gl.app_context");
if (context) {
const GstStructure *s = gst_context_get_structure (context);
gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
gst_context_unref (context);
}
return gl_context;
}
static GstGLContext *
find_gl_context (GstElement *element)
{
GstGLContext *gl_context = NULL;
gl_context = query_gl_local_context (GST_BASE_SRC_PAD (element));
if (!gl_context)
gl_context = find_gl_app_context (element);
return gl_context;
}
static gboolean
caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
gpointer user_data)
{
return !gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
}
- (GstCaps *)fixate:(GstCaps *)new_caps
{
GstGLContext *context;
GstStructure *structure;
new_caps = gst_caps_make_writable (new_caps);
context = find_gl_context (element);
if (!context)
gst_caps_filter_and_map_in_place (new_caps, caps_filter_out_gl_memory, NULL);
else
gst_object_unref (context);
/* this can happen if video/x-raw(memory:GLMemory) is forced but a context is
* not available */
if (gst_caps_is_empty (new_caps)) {
GST_WARNING_OBJECT (element, "GLMemory requested but no context available");
return new_caps;
}
new_caps = gst_caps_truncate (new_caps);
structure = gst_caps_get_structure (new_caps, 0);
/* crank up to 11. This is what the presets do, but we don't use the presets
@ -1056,6 +977,33 @@ caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
return gst_caps_fixate (new_caps);
}
- (BOOL)decideAllocation:(GstQuery *)query
{
GstCaps *alloc_caps;
GstCapsFeatures *features;
gboolean ret;
ret = GST_BASE_SRC_CLASS (parent_class)->decide_allocation (baseSrc, query);
if (!ret)
return ret;
gst_query_parse_allocation (query, &alloc_caps, NULL);
features = gst_caps_get_features (alloc_caps, 0);
if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
gst_gl_context_helper_ensure_context (ctxh);
GST_INFO_OBJECT (element, "pushing textures, context %p old context %p",
ctxh->context, textureCache ? textureCache->ctx : NULL);
if (textureCache && textureCache->ctx != ctxh->context) {
gst_video_texture_cache_free (textureCache);
textureCache = NULL;
}
textureCache = gst_video_texture_cache_new (ctxh->context);
gst_video_texture_cache_set_format (textureCache, format, alloc_caps);
}
return TRUE;
}
- (void)getSampleBuffer:(CMSampleBufferRef)sbuf
timestamp:(GstClockTime *)outTimestamp
duration:(GstClockTime *)outDuration
@ -1186,6 +1134,8 @@ static GstFlowReturn gst_avf_video_src_create (GstPushSrc * pushsrc,
GstBuffer ** buf);
static GstCaps * gst_avf_video_src_fixate (GstBaseSrc * bsrc,
GstCaps * caps);
static gboolean gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc,
GstQuery * query);
static void
gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass)
@ -1209,6 +1159,7 @@ gst_avf_video_src_class_init (GstAVFVideoSrcClass * klass)
gstbasesrc_class->unlock = gst_avf_video_src_unlock;
gstbasesrc_class->unlock_stop = gst_avf_video_src_unlock_stop;
gstbasesrc_class->fixate = gst_avf_video_src_fixate;
gstbasesrc_class->decide_allocation = gst_avf_video_src_decide_allocation;
gstpushsrc_class->create = gst_avf_video_src_create;
@ -1463,3 +1414,16 @@ gst_avf_video_src_fixate (GstBaseSrc * bsrc, GstCaps * caps)
return ret;
}
static gboolean
gst_avf_video_src_decide_allocation (GstBaseSrc * bsrc,
GstQuery * query)
{
gboolean ret;
OBJC_CALLOUT_BEGIN ();
ret = [GST_AVF_VIDEO_SRC_IMPL (bsrc) decideAllocation:query];
OBJC_CALLOUT_END ();
return ret;
}

132
sys/applemedia/glcontexthelper.c Normal file
View file

@ -0,0 +1,132 @@
/* GStreamer
* Copyright (C) 2016 Alessandro Decina <alessandro.d@gmail.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#include "glcontexthelper.h"
static GstGLContext *
_find_local_gl_context (GstGLContextHelper * ctxh)
{
GstQuery *query;
GstContext *context;
GstGLContext *gl_context = NULL;
const GstStructure *s;
g_return_val_if_fail (ctxh != NULL, FALSE);
query = gst_query_new_context ("gst.gl.local_context");
if (gst_gl_run_query (ctxh->element, query, GST_PAD_SRC)) {
gst_query_parse_context (query, &context);
if (context) {
s = gst_context_get_structure (context);
gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
}
}
if (!gl_context && gst_gl_run_query (ctxh->element, query, GST_PAD_SINK)) {
gst_query_parse_context (query, &context);
if (context) {
s = gst_context_get_structure (context);
gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
}
}
GST_DEBUG_OBJECT (ctxh->element, "found local context %p", gl_context);
gst_query_unref (query);
return gl_context;
}
GstGLContextHelper *
gst_gl_context_helper_new (GstElement * element)
{
GstGLContextHelper *ctxh = g_new0 (GstGLContextHelper, 1);
ctxh->element = gst_object_ref (element);
return ctxh;
}
void
gst_gl_context_helper_free (GstGLContextHelper * ctxh)
{
g_return_if_fail (ctxh != NULL);
gst_object_unref (ctxh->element);
if (ctxh->display)
gst_object_unref (ctxh->display);
if (ctxh->context)
gst_object_unref (ctxh->context);
if (ctxh->other_context)
gst_object_unref (ctxh->other_context);
g_free (ctxh);
}
void
gst_gl_context_helper_ensure_context (GstGLContextHelper * ctxh)
{
GError *error = NULL;
GstGLContext *context;
g_return_if_fail (ctxh != NULL);
if (!ctxh->display)
gst_gl_ensure_element_data (ctxh->element, &ctxh->display,
&ctxh->other_context);
context = _find_local_gl_context (ctxh);
if (context) {
GST_INFO_OBJECT (ctxh->element, "found local context %p, old context %p",
context, ctxh->context);
if (ctxh->context)
gst_object_unref (ctxh->context);
ctxh->context = context;
}
if (!ctxh->context) {
GST_OBJECT_LOCK (ctxh->display);
do {
if (ctxh->context)
gst_object_unref (ctxh->context);
ctxh->context =
gst_gl_display_get_gl_context_for_thread (ctxh->display, NULL);
if (!ctxh->context) {
if (!gst_gl_display_create_context (ctxh->display,
ctxh->other_context, &ctxh->context, &error)) {
GST_OBJECT_UNLOCK (ctxh->display);
goto context_error;
}
}
} while (!gst_gl_display_add_context (ctxh->display, ctxh->context));
GST_OBJECT_UNLOCK (ctxh->display);
}
return;
context_error:
{
GST_ELEMENT_ERROR (ctxh->element, RESOURCE, NOT_FOUND, ("%s",
error->message), (NULL));
g_clear_error (&error);
return;
}
}

39
sys/applemedia/glcontexthelper.h Normal file
View file

@ -0,0 +1,39 @@
/* GStreamer
* Copyright (C) 2016 Alessandro Decina <alessandro.d@gmail.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef _GST_GL_CONTEXT_HELPER_H_
#define _GST_GL_CONTEXT_HELPER_H_
#include <gst/gst.h>
#include <gst/gl/gl.h>
typedef struct _GstGLContextHelper
{
GstElement *element;
GstGLDisplay *display;
GstGLContext *context;
GstGLContext *other_context;
} GstGLContextHelper;
GstGLContextHelper * gst_gl_context_helper_new (GstElement *element);
void gst_gl_context_helper_free (GstGLContextHelper *ctxh);
void gst_gl_context_helper_ensure_context (GstGLContextHelper *ctxh);
#endif

63
sys/applemedia/vtdec.c
View file

@ -152,6 +152,7 @@ static void
gst_vtdec_init (GstVtdec * vtdec)
{
vtdec->reorder_queue = g_async_queue_new ();
vtdec->ctxh = gst_gl_context_helper_new (GST_ELEMENT (vtdec));
}
void
@ -162,7 +163,7 @@ gst_vtdec_finalize (GObject * object)
GST_DEBUG_OBJECT (vtdec, "finalize");
g_async_queue_unref (vtdec->reorder_queue);
gst_gl_context_helper_free (vtdec->ctxh);
G_OBJECT_CLASS (gst_vtdec_parent_class)->finalize (object);
}
@ -198,26 +199,6 @@ gst_vtdec_stop (GstVideoDecoder * decoder)
return TRUE;
}
static GstGLContext *
query_gl_context (GstVtdec * vtdec)
{
GstGLContext *gl_context = NULL;
GstContext *context = NULL;
GstQuery *query;
query = gst_query_new_context ("gst.gl.local_context");
if (gst_pad_peer_query (GST_VIDEO_DECODER_SRC_PAD (vtdec), query)) {
gst_query_parse_context (query, &context);
if (context) {
const GstStructure *s = gst_context_get_structure (context);
gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &gl_context, NULL);
}
}
gst_query_unref (query);
return gl_context;
}
static void
setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
{
@ -225,8 +206,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
g_return_if_fail (vtdec->texture_cache == NULL);
GST_INFO_OBJECT (vtdec, "Setting up texture cache. GL context %p", context);
output_state = gst_video_decoder_get_output_state (GST_VIDEO_DECODER (vtdec));
vtdec->texture_cache = gst_video_texture_cache_new (context);
gst_video_texture_cache_set_format (vtdec->texture_cache,
@ -234,14 +213,6 @@ setup_texture_cache (GstVtdec * vtdec, GstGLContext * context)
gst_video_codec_state_unref (output_state);
}
static gboolean
caps_filter_out_gl_memory (GstCapsFeatures * features, GstStructure * structure,
gpointer user_data)
{
return !gst_caps_features_contains (features,
GST_CAPS_FEATURE_MEMORY_GL_MEMORY);
}
static gboolean
gst_vtdec_negotiate (GstVideoDecoder * decoder)
{
@ -250,7 +221,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
GstVideoFormat format;
GstStructure *structure;
const gchar *s;
GstGLContext *context;
GstVtdec *vtdec;
gboolean ret = TRUE;
GstCapsFeatures *features = NULL;
@ -263,9 +233,6 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
gst_caps_make_writable (gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD
(vtdec), templcaps));
gst_caps_unref (templcaps);
context = query_gl_context (vtdec);
if (!context)
gst_caps_filter_and_map_in_place (caps, caps_filter_out_gl_memory, NULL);
caps = gst_caps_truncate (caps);
structure = gst_caps_get_structure (caps, 0);
@ -306,21 +273,29 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder)
}
ret = gst_vtdec_create_session (vtdec, format);
if (ret) {
if (vtdec->texture_cache) {
gst_video_texture_cache_free (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
}
if (output_textures)
setup_texture_cache (vtdec, context);
if (ret && output_textures) {
/* call this regardless of whether caps have changed or not since a new
* local context could have become available
*/
gst_gl_context_helper_ensure_context (vtdec->ctxh);
GST_INFO_OBJECT (vtdec, "pushing textures, context %p old context %p",
vtdec->ctxh->context,
vtdec->texture_cache ? vtdec->texture_cache->ctx : NULL);
if (vtdec->texture_cache
&& vtdec->texture_cache->ctx != vtdec->ctxh->context) {
gst_video_texture_cache_free (vtdec->texture_cache);
vtdec->texture_cache = NULL;
}
if (!vtdec->texture_cache)
setup_texture_cache (vtdec, vtdec->ctxh->context);
}
if (prevcaps)
gst_caps_unref (prevcaps);
if (context)
gst_object_unref (context);
if (!ret)
return ret;

2
sys/applemedia/vtdec.h
View file

@ -26,6 +26,7 @@
#include <CoreMedia/CoreMedia.h>
#include <VideoToolbox/VideoToolbox.h>
#include "videotexturecache.h"
#include "glcontexthelper.h"
G_BEGIN_DECLS
@ -48,6 +49,7 @@ struct _GstVtdec
GAsyncQueue *reorder_queue;
gint reorder_queue_length;
GstVideoTextureCache *texture_cache;
GstGLContextHelper *ctxh;
gboolean require_hardware;
};