gl: add a sync meta for synchronizing across GL contexts

A context can create a GLsync object that can be waited on in order
to ensure that GL resources created in one context are able to be
used in another shared context without any chance of reading invalid
data.

This meta would be placed on buffers that are known to cross from
one context to another.  The receiving element would then wait
on the sync object to ensure that the data to be used is complete.
This commit is contained in:
Matthew Waters 2014-10-17 15:22:24 +02:00 committed by Tim-Philipp Müller
parent f4dbfec17b
commit 6bb113f685
12 changed files with 332 additions and 18 deletions

View file

@ -1090,7 +1090,8 @@ gst_glimage_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query)
/* we also support various metadata */
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
if (glimage_sink->context->gl_vtable->FenceSync)
gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
gl_apis =
gst_gl_api_to_string (gst_gl_context_get_gl_api (glimage_sink->context));

View file

@ -223,6 +223,8 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix,
/* we also support various metadata */
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
if (mix->context->gl_vtable->FenceSync)
gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (mix->context));
platform =
@ -778,6 +780,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query)
guint out_width, out_height;
GstGLContext *other_context = NULL;
GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix);
gboolean same_downstream_gl_context = FALSE;
if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context))
return FALSE;
@ -799,6 +802,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query)
mix->context = context;
if (old)
gst_object_unref (old);
same_downstream_gl_context = TRUE;
} else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle",
G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING,
&type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL)
@ -882,13 +886,23 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query)
update_pool = FALSE;
}
if (!pool)
if (!pool || (!same_downstream_gl_context
&& gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE,
NULL)
&& !gst_buffer_pool_has_option (pool,
GST_BUFFER_POOL_OPTION_GL_SYNC_META))) {
/* can't use this pool */
if (pool)
gst_object_unref (pool);
pool = gst_gl_buffer_pool_new (mix->context);
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_GL_SYNC_META);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META);
@ -1075,6 +1089,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
GstCapsFeatures *gl_features;
GstVideoInfo gl_info;
GstVideoFrame gl_frame;
GstGLSyncMeta *sync_meta;
gst_video_info_set_format (&gl_info,
GST_VIDEO_FORMAT_RGBA,
@ -1096,6 +1111,10 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf)
gst_caps_unref (in_caps);
}
sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer);
if (sync_meta)
gst_gl_sync_meta_wait (sync_meta);
if (!gst_gl_upload_perform_with_buffer (pad->upload,
vaggpad->buffer, &gl_buf)) {
++array_index;
@ -1207,12 +1226,17 @@ gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf)
gboolean res = FALSE;
GstGLMixer *mix = GST_GL_MIXER (vagg);
GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (vagg);
GstGLSyncMeta *sync_meta;
if (mix_class->process_buffers)
res = gst_gl_mixer_process_buffers (mix, outbuf);
else if (mix_class->process_textures)
res = gst_gl_mixer_process_textures (mix, outbuf);
sync_meta = gst_buffer_get_gl_sync_meta (outbuf);
if (sync_meta)
gst_gl_sync_meta_set_sync_point (sync_meta, mix->context);
return res ? GST_FLOW_OK : GST_FLOW_ERROR;
}

View file

@ -591,6 +591,7 @@ gst_gl_test_src_fill (GstPushSrc * psrc, GstBuffer * buffer)
GstClockTime next_time;
gint width, height;
GstVideoFrame out_frame;
GstGLSyncMeta *sync_meta;
guint out_tex;
gboolean to_download =
gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY,
@ -662,6 +663,10 @@ gst_gl_test_src_fill (GstPushSrc * psrc, GstBuffer * buffer)
}
gst_video_frame_unmap (&out_frame);
sync_meta = gst_buffer_get_gl_sync_meta (buffer);
if (sync_meta)
gst_gl_sync_meta_set_sync_point (sync_meta, src->context);
GST_BUFFER_TIMESTAMP (buffer) = src->timestamp_offset + src->running_time;
GST_BUFFER_OFFSET (buffer) = src->n_frames;
src->n_frames++;
@ -756,6 +761,7 @@ gst_gl_test_src_decide_allocation (GstBaseSrc * basesrc, GstQuery * query)
guint idx;
guint out_width, out_height;
GstGLContext *other_context = NULL;
gboolean same_downstream_gl_context;
if (!gst_gl_ensure_element_data (src, &src->display, &src->other_context))
return FALSE;
@ -777,6 +783,7 @@ gst_gl_test_src_decide_allocation (GstBaseSrc * basesrc, GstQuery * query)
src->context = context;
if (old)
gst_object_unref (old);
same_downstream_gl_context = TRUE;
} else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle",
G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING,
&type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL)
@ -837,14 +844,26 @@ gst_gl_test_src_decide_allocation (GstBaseSrc * basesrc, GstQuery * query)
update_pool = FALSE;
}
if (!pool)
if (!pool || (!same_downstream_gl_context
&& gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE,
NULL)
&& !gst_buffer_pool_has_option (pool,
GST_BUFFER_POOL_OPTION_GL_SYNC_META))) {
/* can't use this pool */
if (pool)
gst_object_unref (pool);
pool = gst_gl_buffer_pool_new (src->context);
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_GL_SYNC_META);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META);
gst_buffer_pool_set_config (pool, config);
if (update_pool)

View file

@ -24,7 +24,8 @@ libgstgl_@GST_API_VERSION@_la_SOURCES = \
gstglapi.c \
gstglfeature.c \
gstglutils.c \
gstglframebuffer.c
gstglframebuffer.c \
gstglsyncmeta.c
libgstgl_@GST_API_VERSION@includedir = $(includedir)/gstreamer-@GST_API_VERSION@/gst/gl
libgstgl_@GST_API_VERSION@include_HEADERS = \
@ -44,6 +45,7 @@ libgstgl_@GST_API_VERSION@include_HEADERS = \
gstglfeature.h \
gstglutils.h \
gstglframebuffer.h \
gstglsyncmeta.h \
gstgl_fwd.h \
gl.h

View file

@ -44,5 +44,6 @@
#include <gst/gl/gstglframebuffer.h>
#include <gst/gl/gstglfilter.h>
#include <gst/gl/gstglshadervariables.h>
#include <gst/gl/gstglsyncmeta.h>
#endif /* __GST_GL_H__ */

View file

@ -36,6 +36,9 @@ typedef ptrdiff_t GLsizeiptr;
#if !GST_GL_HAVE_GLINTPTR
typedef ptrdiff_t GLintptr;
#endif
#if !GST_GL_HAVE_GLSYNC
typedef gpointer GLsync;
#endif
#if !defined(GST_GL_DEBUG_PROC)
#if defined(GLDEBUGPROC)

View file

@ -22,7 +22,6 @@ GST_GL_EXT_BEGIN (sync,
GST_GL_API_OPENGL3,
3, 2,
3, 0,
/* FIXME: the extension depends on GL 3.1 */
"",
"")
GST_GL_EXT_FUNCTION (GLsync, FenceSync,

View file

@ -52,6 +52,7 @@ struct _GstGLBufferPoolPrivate
GstVideoInfo info;
gboolean add_videometa;
gboolean add_uploadmeta;
gboolean add_glsyncmeta;
gboolean want_eglimage;
GstBuffer *last_buffer;
};
@ -73,7 +74,8 @@ static const gchar **
gst_gl_buffer_pool_get_options (GstBufferPool * pool)
{
static const gchar *options[] = { GST_BUFFER_POOL_OPTION_VIDEO_META,
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META, NULL
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META,
GST_BUFFER_POOL_OPTION_GL_SYNC_META, NULL
};
return options;
@ -132,6 +134,8 @@ gst_gl_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
GST_BUFFER_POOL_OPTION_VIDEO_META);
priv->add_uploadmeta = gst_buffer_pool_config_has_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META);
priv->add_glsyncmeta = gst_buffer_pool_config_has_option (config,
GST_BUFFER_POOL_OPTION_GL_SYNC_META);
#if GST_GL_HAVE_PLATFORM_EGL
g_assert (priv->allocator != NULL);
@ -224,6 +228,9 @@ gst_gl_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
if (priv->add_uploadmeta)
gst_gl_upload_meta_add_to_buffer (glpool->upload, buf);
if (priv->add_glsyncmeta)
gst_buffer_add_gl_sync_meta (glpool->context, buf);
*buffer = buf;
return GST_FLOW_OK;
@ -349,6 +356,7 @@ gst_gl_buffer_pool_init (GstGLBufferPool * pool)
priv->caps = NULL;
priv->im_format = GST_VIDEO_FORMAT_UNKNOWN;
priv->add_videometa = TRUE;
priv->add_glsyncmeta = FALSE;
priv->want_eglimage = FALSE;
priv->last_buffer = FALSE;

View file

@ -1339,13 +1339,6 @@ _do_convert (GstGLContext * context, GstGLColorConvert * convert)
return;
}
gst_buffer_add_video_meta_full (convert->outbuf, 0,
GST_VIDEO_INFO_FORMAT (&convert->out_info),
GST_VIDEO_INFO_WIDTH (&convert->out_info),
GST_VIDEO_INFO_HEIGHT (&convert->out_info),
GST_VIDEO_INFO_N_PLANES (&convert->out_info),
convert->out_info.offset, convert->out_info.stride);
for (i = 0; i < c_info->in_n_textures; i++) {
convert->priv->in_tex[i] =
(GstGLMemory *) gst_buffer_peek_memory (convert->inbuf, i);

View file

@ -944,6 +944,8 @@ gst_gl_filter_propose_allocation (GstBaseTransform * trans,
/* we also support various metadata */
gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0);
if (filter->context->gl_vtable->FenceSync)
gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0);
gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (filter->context));
platform =
@ -1017,6 +1019,7 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query)
GError *error = NULL;
guint in_width, in_height, out_width, out_height;
GstGLContext *other_context = NULL;
gboolean same_downstream_gl_context = FALSE;
if (!gst_gl_ensure_element_data (filter, &filter->display,
&filter->other_context))
@ -1039,6 +1042,7 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query)
filter->context = context;
if (old)
gst_object_unref (old);
same_downstream_gl_context = TRUE;
} else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle",
G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING,
&type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL)
@ -1132,13 +1136,23 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query)
update_pool = FALSE;
}
if (!pool)
if (!pool || (!same_downstream_gl_context
&& gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE,
NULL)
&& !gst_buffer_pool_has_option (pool,
GST_BUFFER_POOL_OPTION_GL_SYNC_META))) {
/* can't use this pool */
if (pool)
gst_object_unref (pool);
pool = gst_gl_buffer_pool_new (filter->context);
}
config = gst_buffer_pool_get_config (pool);
gst_buffer_pool_config_set_params (config, caps, size, min, max);
gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL))
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_GL_SYNC_META);
gst_buffer_pool_config_add_option (config,
GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META);
@ -1275,6 +1289,7 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf,
{
GstGLFilter *filter;
GstGLFilterClass *filter_class;
GstGLSyncMeta *out_sync_meta, *in_sync_meta;
filter = GST_GL_FILTER (bt);
filter_class = GST_GL_FILTER_GET_CLASS (bt);
@ -1310,11 +1325,19 @@ gst_gl_filter_transform (GstBaseTransform * bt, GstBuffer * inbuf,
g_assert (filter_class->filter || filter_class->filter_texture);
in_sync_meta = gst_buffer_get_gl_sync_meta (inbuf);
if (in_sync_meta)
gst_gl_sync_meta_wait (in_sync_meta);
if (filter_class->filter)
filter_class->filter (filter, inbuf, outbuf);
else if (filter_class->filter_texture)
gst_gl_filter_filter_texture (filter, inbuf, outbuf);
out_sync_meta = gst_buffer_get_gl_sync_meta (outbuf);
if (out_sync_meta)
gst_gl_sync_meta_set_sync_point (out_sync_meta, filter->context);
return GST_FLOW_OK;
}

View file

@ -0,0 +1,189 @@
/*
* GStreamer
* Copyright (C) 2014 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gl.h"
#include "gstglsyncmeta.h"
#define GST_CAT_DEFAULT gst_gl_sync_meta_debug
GST_DEBUG_CATEGORY (GST_CAT_DEFAULT);
GstGLSyncMeta *
gst_buffer_add_gl_sync_meta (GstGLContext * context, GstBuffer * buffer)
{
GstGLSyncMeta *meta =
(GstGLSyncMeta *) gst_buffer_add_meta ((buffer), GST_GL_SYNC_META_INFO,
NULL);
if (!meta)
return NULL;
meta->context = gst_object_ref (context);
meta->glsync = NULL;
return meta;
}
static void
_set_sync_point (GstGLContext * context, GstGLSyncMeta * sync_meta)
{
const GstGLFuncs *gl = context->gl_vtable;
if (gl->FenceSync) {
if (sync_meta->glsync)
gl->DeleteSync (sync_meta->glsync);
sync_meta->glsync = gl->FenceSync (GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
GST_LOG_OBJECT (sync_meta, "setting sync object %p", sync_meta->glsync);
} else {
gl->Flush ();
}
}
void
gst_gl_sync_meta_set_sync_point (GstGLSyncMeta * sync_meta,
GstGLContext * context)
{
gst_gl_context_thread_add (context,
(GstGLContextThreadFunc) _set_sync_point, sync_meta);
}
static void
_wait (GstGLContext * context, GstGLSyncMeta * sync_meta)
{
const GstGLFuncs *gl = context->gl_vtable;
GLenum res;
if (gl->ClientWaitSync) {
do {
GST_LOG_OBJECT (sync_meta, "waiting on sync object %p",
sync_meta->glsync);
res =
gl->ClientWaitSync (sync_meta->glsync, GL_SYNC_FLUSH_COMMANDS_BIT,
1000000000 /* 1s */ );
} while (res == GL_TIMEOUT_EXPIRED);
}
}
void
gst_gl_sync_meta_wait (GstGLSyncMeta * sync_meta)
{
if (sync_meta->glsync) {
gst_gl_context_thread_add (sync_meta->context,
(GstGLContextThreadFunc) _wait, sync_meta);
}
}
static gboolean
_gst_gl_sync_meta_transform (GstBuffer * dest, GstMeta * meta,
GstBuffer * buffer, GQuark type, gpointer data)
{
GstGLSyncMeta *dmeta, *smeta;
smeta = (GstGLSyncMeta *) meta;
if (GST_META_TRANSFORM_IS_COPY (type)) {
GstMetaTransformCopy *copy = data;
if (!copy->region) {
/* only copy if the complete data is copied as well */
dmeta = gst_buffer_add_gl_sync_meta (smeta->context, dest);
if (!dmeta)
return FALSE;
GST_DEBUG ("copy gl sync metadata");
dmeta->glsync = smeta->glsync;
}
}
return TRUE;
}
static void
_free_gl_sync_meta (GstGLContext * context, GstGLSyncMeta * sync_meta)
{
const GstGLFuncs *gl = context->gl_vtable;
if (sync_meta->glsync)
gl->DeleteSync (sync_meta->glsync);
sync_meta->glsync = NULL;
}
static void
_gst_gl_sync_meta_free (GstGLSyncMeta * sync_meta, GstBuffer * buffer)
{
if (sync_meta->glsync) {
gst_gl_context_thread_add (sync_meta->context,
(GstGLContextThreadFunc) _free_gl_sync_meta, sync_meta);
}
gst_object_unref (sync_meta->context);
}
static gboolean
_gst_gl_sync_meta_init (GstGLSyncMeta * sync_meta, gpointer params,
GstBuffer * buffer)
{
static volatile gsize _init;
if (g_once_init_enter (&_init)) {
GST_DEBUG_CATEGORY_INIT (gst_gl_sync_meta_debug, "glsyncmeta", 0,
"glsyncmeta");
g_once_init_leave (&_init, 1);
}
sync_meta->context = NULL;
sync_meta->glsync = NULL;
return TRUE;
}
GType
gst_gl_sync_meta_api_get_type (void)
{
static volatile GType type = 0;
static const gchar *tags[] = { NULL };
if (g_once_init_enter (&type)) {
GType _type = gst_meta_api_type_register ("GstGLSyncMetaAPI", tags);
g_once_init_leave (&type, _type);
}
return type;
}
const GstMetaInfo *
gst_gl_sync_meta_get_info (void)
{
static const GstMetaInfo *meta_info = NULL;
if (g_once_init_enter (&meta_info)) {
const GstMetaInfo *meta =
gst_meta_register (GST_GL_SYNC_META_API_TYPE, "GstGLSyncMeta",
sizeof (GstVideoMeta), (GstMetaInitFunction) _gst_gl_sync_meta_init,
(GstMetaFreeFunction) _gst_gl_sync_meta_free,
_gst_gl_sync_meta_transform);
g_once_init_leave (&meta_info, meta);
}
return meta_info;
}

View file

@ -0,0 +1,52 @@
/*
* GStreamer
* Copyright (C) 2014 Matthew Waters <matthew@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifndef __GST_GL_SYNC_META_H__
#define __GST_GL_SYNC_META_H__
#include <gst/gl/gstgl_fwd.h>
G_BEGIN_DECLS
#define GST_GL_SYNC_META_API_TYPE (gst_gl_sync_meta_api_get_type())
#define GST_GL_SYNC_META_INFO (gst_gl_sync_meta_get_info())
typedef struct _GstGLSyncMeta GstGLSyncMeta;
#define GST_BUFFER_POOL_OPTION_GL_SYNC_META "GstBufferPoolOptionGLSyncMeta"
struct _GstGLSyncMeta {
/*< private >*/
GstMeta parent;
GstGLContext *context;
GLsync glsync;
};
GType gst_gl_sync_meta_api_get_type (void);
const GstMetaInfo * gst_gl_sync_meta_get_info (void);
#define gst_buffer_get_gl_sync_meta(b) ((GstGLSyncMeta*)gst_buffer_get_meta((b),GST_GL_SYNC_META_API_TYPE))
GstGLSyncMeta * gst_buffer_add_gl_sync_meta (GstGLContext * context, GstBuffer *buffer);
void gst_gl_sync_meta_set_sync_point (GstGLSyncMeta * sync, GstGLContext * context);
void gst_gl_sync_meta_wait (GstGLSyncMeta * sync);
#endif /* __GST_GL_SYNC_META_H__ */