From 343947f383e11897e05701aef3373865be979612 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 30 Apr 2015 11:15:40 +1000 Subject: [PATCH] gl: readd glupload/download onto element pads Allows insertion of gl elements into non-gl pipelines without converter (upload/download) elements. https://bugzilla.gnome.org/show_bug.cgi?id=743974 --- ext/gl/gstglcolorconvertelement.c | 116 ++++++++++++++---- ext/gl/gstglcolorconvertelement.h | 2 + ext/gl/gstglimagesink.c | 92 +++----------- ext/gl/gstglimagesink.h | 1 - ext/gl/gstglmixer.c | 178 ++++++++++++++++++++++------ ext/gl/gstglmixer.h | 8 ++ ext/gl/gstglmosaic.c | 3 + ext/gl/gstgltestsrc.c | 5 +- gst-libs/gst/gl/gstglcolorconvert.c | 12 +- gst-libs/gst/gl/gstglfilter.c | 107 ++++++++++++----- gst-libs/gst/gl/gstglfilter.h | 2 + gst-libs/gst/gl/gstglutils.c | 18 +-- gst-libs/gst/gl/gstglutils.h | 2 +- 13 files changed, 359 insertions(+), 187 deletions(-) diff --git a/ext/gl/gstglcolorconvertelement.c b/ext/gl/gstglcolorconvertelement.c index 833d07e014..f7dc656966 100644 --- a/ext/gl/gstglcolorconvertelement.c +++ b/ext/gl/gstglcolorconvertelement.c @@ -32,8 +32,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_gl_color_convert_element_debug); G_DEFINE_TYPE_WITH_CODE (GstGLColorConvertElement, gst_gl_color_convert_element, GST_TYPE_GL_BASE_FILTER, GST_DEBUG_CATEGORY_INIT (gst_gl_color_convert_element_debug, - "glconvertelement", 0, "convert"); - ); + "glconvertelement", 0, "convert");); static gboolean gst_gl_color_convert_element_set_caps (GstBaseTransform * bt, GstCaps * in_caps, GstCaps * out_caps); @@ -52,22 +51,24 @@ static GstCaps *gst_gl_color_convert_element_fixate_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * othercaps); static GstStaticPadTemplate gst_gl_color_convert_element_src_pad_template = -GST_STATIC_PAD_TEMPLATE ("src", + GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_GL_COLOR_CONVERT_VIDEO_CAPS)); - -static GstStaticPadTemplate gst_gl_color_convert_element_sink_pad_template = -GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_GL_COLOR_CONVERT_VIDEO_CAPS)); + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + GST_GL_COLOR_CONVERT_FORMATS) ";" + GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS))); static gboolean gst_gl_color_convert_element_stop (GstBaseTransform * bt) { GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (bt); + if (convert->upload) { + gst_object_unref (convert->upload); + convert->upload = NULL; + } + if (convert->convert) { gst_object_unref (convert->convert); convert->convert = NULL; @@ -86,6 +87,7 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass) { GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass); GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstCaps *upload_caps; bt_class->transform_caps = gst_gl_color_convert_element_transform_caps; bt_class->set_caps = gst_gl_color_convert_element_set_caps; @@ -102,9 +104,11 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&gst_gl_color_convert_element_src_pad_template)); + + upload_caps = gst_gl_upload_get_input_template_caps (); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get - (&gst_gl_color_convert_element_sink_pad_template)); + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, upload_caps)); + gst_caps_unref (upload_caps); gst_element_class_set_metadata (element_class, "OpenGL color converter", "Filter/Converter/Video", @@ -128,9 +132,6 @@ gst_gl_color_convert_element_set_caps (GstBaseTransform * bt, gst_caps_replace (&convert->in_caps, in_caps); gst_caps_replace (&convert->out_caps, out_caps); - if (convert->convert) - gst_gl_color_convert_set_caps (convert->convert, in_caps, out_caps); - return TRUE; } @@ -139,8 +140,43 @@ gst_gl_color_convert_element_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter) { GstGLContext *context = GST_GL_BASE_FILTER (bt)->context; + GstCaps *tmp, *ret; - return gst_gl_color_convert_transform_caps (context, direction, caps, filter); + if (direction == GST_PAD_SINK) { + ret = gst_gl_upload_transform_caps (context, direction, caps, NULL); + } else { + tmp = + gst_gl_caps_replace_all_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + ret = gst_caps_merge (gst_caps_ref (caps), tmp); + } + + GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, ret); + + tmp = gst_gl_color_convert_transform_caps (context, direction, ret, NULL); + gst_caps_unref (ret); + ret = tmp; + GST_DEBUG_OBJECT (bt, "convert returned %" GST_PTR_FORMAT, ret); + + if (direction == GST_PAD_SRC) { + tmp = gst_gl_upload_transform_caps (context, direction, ret, NULL); + gst_caps_unref (ret); + ret = tmp; + } else { + tmp = + gst_gl_caps_replace_all_caps_features (ret, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + ret = gst_caps_merge (ret, tmp); + } + GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, ret); + + if (filter) { + tmp = gst_caps_intersect_full (filter, ret, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (ret); + ret = tmp; + } + + return ret; } static gboolean @@ -163,6 +199,7 @@ gst_gl_color_convert_element_decide_allocation (GstBaseTransform * trans, { GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (trans); GstGLContext *context; + GstCaps *converted_caps; /* get gl context */ if (!GST_BASE_TRANSFORM_CLASS @@ -172,12 +209,35 @@ gst_gl_color_convert_element_decide_allocation (GstBaseTransform * trans, context = GST_GL_BASE_FILTER (trans)->context; + if (!convert->upload) + convert->upload = gst_gl_upload_new (context); + + if (convert->upload_caps) + gst_caps_unref (convert->upload_caps); + convert->upload_caps = gst_caps_copy (convert->in_caps); + gst_caps_set_features (convert->upload_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + if (!gst_gl_upload_set_caps (convert->upload, convert->in_caps, + convert->upload_caps)) { + GST_ERROR_OBJECT (trans, "failed to set caps for upload"); + return FALSE; + } + if (!convert->convert) convert->convert = gst_gl_color_convert_new (context); - if (!gst_gl_color_convert_set_caps (convert->convert, convert->in_caps, - convert->out_caps)) + converted_caps = gst_caps_copy (convert->out_caps); + gst_caps_set_features (converted_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + if (!gst_gl_color_convert_set_caps (convert->convert, convert->upload_caps, + converted_caps)) { + gst_caps_unref (converted_caps); + GST_ERROR_OBJECT (trans, "failed to set caps for conversion"); return FALSE; + } + gst_caps_unref (converted_caps); return TRUE; } @@ -187,21 +247,35 @@ gst_gl_color_convert_element_prepare_output_buffer (GstBaseTransform * bt, GstBuffer * inbuf, GstBuffer ** outbuf) { GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (bt); + GstBuffer *uploaded_buffer; if (gst_base_transform_is_passthrough (bt)) { *outbuf = inbuf; return GST_FLOW_OK; } - if (!convert->convert) + if (!convert->upload || !convert->convert) return GST_FLOW_NOT_NEGOTIATED; - *outbuf = gst_gl_color_convert_perform (convert->convert, inbuf); + if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (convert->upload, + inbuf, &uploaded_buffer) || !uploaded_buffer) { + GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, ("%s", + "failed to upload buffer"), (NULL)); + return GST_FLOW_ERROR; + } + + *outbuf = gst_gl_color_convert_perform (convert->convert, uploaded_buffer); + gst_buffer_unref (uploaded_buffer); + if (!*outbuf) { + GST_ELEMENT_ERROR (bt, RESOURCE, NOT_FOUND, + ("%s", "failed to convert buffer"), (NULL)); + return GST_FLOW_ERROR; + } /* basetransform doesn't unref if they're the same */ if (inbuf == *outbuf) gst_buffer_unref (*outbuf); - if (*outbuf) + else if (*outbuf) gst_buffer_copy_into (*outbuf, inbuf, GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS, 0, -1); diff --git a/ext/gl/gstglcolorconvertelement.h b/ext/gl/gstglcolorconvertelement.h index 2a0dd1dcaa..61bde0befa 100644 --- a/ext/gl/gstglcolorconvertelement.h +++ b/ext/gl/gstglcolorconvertelement.h @@ -44,8 +44,10 @@ struct _GstGLColorConvertElement { GstGLBaseFilter parent; + GstGLUpload *upload; GstGLColorConvert *convert; GstCaps *in_caps; + GstCaps *upload_caps; GstCaps *out_caps; }; diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c index 6d64703b1e..a7de89fc30 100644 --- a/ext/gl/gstglimagesink.c +++ b/ext/gl/gstglimagesink.c @@ -156,22 +156,6 @@ static void gst_glimage_sink_handle_events (GstVideoOverlay * overlay, gboolean handle_events); -static GstStaticPadTemplate gst_glimage_sink_template = - GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") "; " -#if GST_GL_HAVE_PLATFORM_EGL - GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, "RGBA") "; " -#endif - GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, - "RGBA") "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) - ); - enum { ARG_0, @@ -262,6 +246,7 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) GstBaseSinkClass *gstbasesink_class; GstVideoSinkClass *gstvideosink_class; GstElementClass *element_class; + GstCaps *upload_caps, *elem_caps, *templ; gobject_class = (GObjectClass *) klass; gstelement_class = (GstElementClass *) klass; @@ -343,8 +328,14 @@ gst_glimage_sink_class_init (GstGLImageSinkClass * klass) G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic, G_TYPE_BOOLEAN, 3, GST_GL_TYPE_CONTEXT, G_TYPE_UINT, G_TYPE_UINT); + elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA"); + upload_caps = gst_gl_upload_get_input_template_caps (); + templ = gst_caps_intersect (elem_caps, upload_caps); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&gst_glimage_sink_template)); + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, templ)); + gst_caps_unref (upload_caps); + gst_caps_unref (elem_caps); + gst_caps_unref (templ); gobject_class->finalize = gst_glimage_sink_finalize; @@ -640,19 +631,6 @@ gst_glimage_sink_query (GstBaseSink * bsink, GstQuery * query) return res; } -static gboolean -gst_glimage_sink_stop (GstBaseSink * bsink) -{ - GstGLImageSink *glimage_sink = GST_GLIMAGE_SINK (bsink); - - if (glimage_sink->pool) { - gst_object_unref (glimage_sink->pool); - glimage_sink->pool = NULL; - } - - return TRUE; -} - static void gst_glimage_sink_set_context (GstElement * element, GstContext * context) { @@ -723,11 +701,6 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition) glimage_sink->upload = NULL; } - if (glimage_sink->convert) { - gst_object_unref (glimage_sink->convert); - glimage_sink->convert = NULL; - } - glimage_sink->window_id = 0; /* but do not reset glimage_sink->new_window_id */ @@ -818,13 +791,6 @@ gst_glimage_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) tmp = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (bsink)); - result = - gst_gl_color_convert_transform_caps (gl_sink->context, GST_PAD_SRC, tmp, - NULL); - gst_caps_unref (tmp); - tmp = result; - GST_DEBUG_OBJECT (bsink, "convert returned caps %" GST_PTR_FORMAT, tmp); - result = gst_gl_upload_transform_caps (gl_sink->context, GST_PAD_SRC, tmp, NULL); gst_caps_unref (tmp); @@ -854,8 +820,6 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) gint display_par_n, display_par_d; guint display_ratio_num, display_ratio_den; GstVideoInfo vinfo; - GstCapsFeatures *gl_features; - GstCaps *uploaded_caps; GST_DEBUG_OBJECT (bsink, "set caps with %" GST_PTR_FORMAT, caps); @@ -923,33 +887,17 @@ gst_glimage_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) gst_object_unref (glimage_sink->upload); glimage_sink->upload = gst_gl_upload_new (glimage_sink->context); - gl_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - uploaded_caps = gst_caps_copy (caps); - gst_caps_set_features (uploaded_caps, 0, - gst_caps_features_copy (gl_features)); - gst_gl_upload_set_caps (glimage_sink->upload, caps, uploaded_caps); - if (glimage_sink->gl_caps) gst_caps_unref (glimage_sink->gl_caps); glimage_sink->gl_caps = gst_caps_copy (caps); gst_caps_set_simple (glimage_sink->gl_caps, "format", G_TYPE_STRING, "RGBA", NULL); gst_caps_set_features (glimage_sink->gl_caps, 0, - gst_caps_features_copy (gl_features)); + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); - if (glimage_sink->convert) - gst_object_unref (glimage_sink->convert); - glimage_sink->convert = gst_gl_color_convert_new (glimage_sink->context); - if (!gst_gl_color_convert_set_caps (glimage_sink->convert, uploaded_caps, - glimage_sink->gl_caps)) { - gst_caps_unref (uploaded_caps); - gst_caps_features_free (gl_features); + if (!gst_gl_upload_set_caps (glimage_sink->upload, caps, + glimage_sink->gl_caps)) return FALSE; - } - gst_caps_unref (uploaded_caps); - gst_caps_features_free (gl_features); glimage_sink->caps_change = TRUE; @@ -960,7 +908,7 @@ static GstFlowReturn gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) { GstGLImageSink *glimage_sink; - GstBuffer *uploaded_buffer, *next_buffer = NULL; + GstBuffer *next_buffer = NULL; GstVideoFrame gl_frame; GstVideoInfo gl_info; @@ -977,25 +925,16 @@ gst_glimage_sink_prepare (GstBaseSink * bsink, GstBuffer * buf) return GST_FLOW_NOT_NEGOTIATED; if (gst_gl_upload_perform_with_buffer (glimage_sink->upload, buf, - &uploaded_buffer) != GST_GL_UPLOAD_DONE) + &next_buffer) != GST_GL_UPLOAD_DONE) goto upload_failed; - if (!(next_buffer = - gst_gl_color_convert_perform (glimage_sink->convert, - uploaded_buffer))) { - gst_buffer_unref (uploaded_buffer); - goto upload_failed; - } - gst_video_info_from_caps (&gl_info, glimage_sink->gl_caps); if (!gst_video_frame_map (&gl_frame, &gl_info, next_buffer, GST_MAP_READ | GST_MAP_GL)) { - gst_buffer_unref (uploaded_buffer); gst_buffer_unref (next_buffer); goto upload_failed; } - gst_buffer_unref (uploaded_buffer); glimage_sink->next_tex = *(guint *) gl_frame.data[0]; @@ -1487,9 +1426,8 @@ gst_glimage_sink_redisplay (GstGLImageSink * gl_sink) return FALSE; if (gst_gl_window_is_running (window)) { - gulong handler_id = - g_signal_handler_find (GST_ELEMENT_PARENT (gl_sink), G_SIGNAL_MATCH_ID, - gst_gl_image_sink_bin_signals[SIGNAL_BIN_CLIENT_DRAW], 0, + gulong handler_id = g_signal_handler_find (gl_sink, G_SIGNAL_MATCH_ID, + gst_glimage_sink_signals[CLIENT_DRAW_SIGNAL], 0, NULL, NULL, NULL); if (G_UNLIKELY (!gl_sink->redisplay_shader) && (!handler_id diff --git a/ext/gl/gstglimagesink.h b/ext/gl/gstglimagesink.h index 01851a4d49..8ada89a377 100644 --- a/ext/gl/gstglimagesink.h +++ b/ext/gl/gstglimagesink.h @@ -70,7 +70,6 @@ struct _GstGLImageSink gboolean ignore_alpha; GstGLUpload *upload; - GstGLColorConvert *convert; guint next_tex; GstBuffer *next_buffer; diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 8e35af10b0..907a160dc4 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -42,6 +42,9 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_pad_finalize (GObject * object); +static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer); enum { @@ -71,10 +74,13 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; + gobject_class->finalize = gst_gl_mixer_pad_finalize; vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; + + klass->upload_buffer = _default_pad_upload_buffer; } static void @@ -198,12 +204,22 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } -/* copies the given caps */ -static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +GstCaps * +gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - return gst_gl_caps_replace_all_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + GstCaps *result, *tmp; + + result = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")); + + tmp = gst_gl_upload_transform_caps (context, GST_PAD_SRC, result, NULL); + gst_caps_unref (result); + result = tmp; + GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, result); + + return result; } static GstCaps * @@ -284,6 +300,17 @@ gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad) { } +static void +gst_gl_mixer_pad_finalize (GObject * object) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); + + gst_buffer_replace (&pad->uploaded_buffer, NULL); + gst_object_replace ((GstObject **) & pad->upload, NULL); + + G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); +} + /* GLMixer signals and args */ enum { @@ -301,15 +328,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) - ); - -static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", - GST_PAD_SINK, - GST_PAD_REQUEST, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) + "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA")) ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); @@ -340,7 +359,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) GstVideoAggregatorClass *videoaggregator_class = (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); + GstCaps *upload_caps, *elem_caps, *templ; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); @@ -353,8 +373,15 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); + + elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA"); + upload_caps = gst_gl_upload_get_input_template_caps (); + templ = gst_caps_intersect (elem_caps, upload_caps); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_factory)); + gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, templ)); + gst_caps_unref (upload_caps); + gst_caps_unref (elem_caps); + gst_caps_unref (templ); agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; @@ -365,7 +392,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; - videoaggregator_class->update_caps = _update_caps; videoaggregator_class->find_best_format = NULL; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; @@ -545,8 +571,11 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) update_pool = FALSE; } - if (!pool) + if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) { + if (pool) + gst_object_unref (pool); pool = gst_gl_buffer_pool_new (context); + } config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, min, max); @@ -570,6 +599,63 @@ context_error: } } +static GstBuffer * +_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, + GstBuffer * buffer) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); + GstGLMixerPad *pad = frame->pad; + GstVideoInfo gl_info; + GstGLSyncMeta *sync_meta; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + + if (!pad->upload) { + GstCaps *in_caps, *upload_caps; + + in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + upload_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (upload_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + pad->upload = gst_gl_upload_new (GST_GL_BASE_MIXER (mix)->context); + + if (!gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps)) { + GST_WARNING_OBJECT (pad, "Failed to configure uploader"); + gst_caps_unref (in_caps); + gst_caps_unref (upload_caps); + gst_object_unref (pad->upload); + pad->upload = NULL; + return NULL; + } + + gst_caps_unref (in_caps); + gst_caps_unref (upload_caps); + } + + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); + + if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (pad->upload, + vaggpad->buffer, &pad->uploaded_buffer) || !pad->uploaded_buffer) { + GST_WARNING_OBJECT (pad, "Failed to upload input buffer %p", + vaggpad->buffer); + return NULL; + } + + if (gst_video_frame_map (&frame->v_frame, &gl_info, pad->uploaded_buffer, + GST_MAP_READ | GST_MAP_GL)) { + frame->texture = *(guint *) frame->v_frame.data[0]; + frame->mapped = TRUE; + } + + return pad->uploaded_buffer; +} + gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { @@ -577,7 +663,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GList *walk; guint out_tex; gboolean res = TRUE; - guint array_index = 0; + gint array_index = 0; GstVideoFrame out_frame; GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); @@ -602,34 +688,28 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad); GstVideoAggregatorPad *vaggpad = walk->data; GstGLMixerFrameData *frame; frame = g_ptr_array_index (mix->frames, array_index); frame->pad = pad; frame->texture = 0; + frame->mapped = FALSE; walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; + g_assert (pad_class->upload_buffer); - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + if (pad->uploaded_buffer) + gst_buffer_unref (pad->uploaded_buffer); + pad->uploaded_buffer = + pad_class->upload_buffer (mix, frame, vaggpad->buffer); - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, - GST_MAP_READ | GST_MAP_GL)) { - frame->texture = *(guint *) gl_frame.data[0]; - gst_video_frame_unmap (&gl_frame); - } + GST_DEBUG_OBJECT (pad, + "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT, + pad->uploaded_buffer, vaggpad->buffer); } ++array_index; @@ -652,6 +732,19 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); out: + for (--array_index; array_index >= 0; array_index--) { + GstGLMixerFrameData *frame; + + frame = g_ptr_array_index (mix->frames, array_index); + + if (frame->mapped) + gst_video_frame_unmap (&frame->v_frame); + frame->mapped = FALSE; + frame->texture = 0; + + gst_buffer_replace (&frame->pad->uploaded_buffer, NULL); + } + GST_OBJECT_UNLOCK (mix); gst_video_frame_unmap (&out_frame); @@ -761,6 +854,18 @@ gst_gl_mixer_start (GstAggregator * agg) return GST_AGGREGATOR_CLASS (parent_class)->start (agg); } +static gboolean +_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, + gpointer user_data) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); + + gst_buffer_replace (&pad->uploaded_buffer, NULL); + gst_object_replace ((GstObject **) & pad->upload, NULL); + + return TRUE; +} + static gboolean gst_gl_mixer_stop (GstAggregator * agg) { @@ -768,6 +873,9 @@ gst_gl_mixer_stop (GstAggregator * agg) GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + gst_aggregator_iterate_sinkpads (agg, + (GstAggregatorPadForeachFunc) _clean_upload, NULL); + GST_OBJECT_LOCK (agg); g_ptr_array_free (mix->frames, TRUE); mix->frames = NULL; diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 01eed34f20..38b51d369d 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -52,11 +52,16 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass; struct _GstGLMixerPad { GstGLBaseMixerPad parent; + + GstGLUpload *upload; + GstBuffer *uploaded_buffer; }; struct _GstGLMixerPadClass { GstGLBaseMixerPadClass parent_class; + + GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); }; GType gst_gl_mixer_pad_get_type (void); @@ -109,12 +114,15 @@ struct _GstGLMixerClass struct _GstGLMixerFrameData { GstGLMixerPad *pad; + gboolean mapped; + GstVideoFrame v_frame; guint texture; }; GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); +GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 726674b4df..0dc13b7c67 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -190,6 +190,9 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); + if (mosaic->shader) + gst_object_unref (mosaic->shader); + //blocking call, wait the opengl thread has compiled the shader return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, mosaic_v_src, mosaic_f_src, &mosaic->shader); diff --git a/ext/gl/gstgltestsrc.c b/ext/gl/gstgltestsrc.c index 100ecaae23..e91aff333b 100644 --- a/ext/gl/gstgltestsrc.c +++ b/ext/gl/gstgltestsrc.c @@ -68,7 +68,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) + "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA")) ); #define gst_gl_test_src_parent_class parent_class @@ -455,8 +455,7 @@ static GstCaps * gst_gl_test_src_getcaps (GstBaseSrc * bsrc, GstCaps * filter) { GstCaps *tmp = NULL; - GstCaps *result = - gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); + GstCaps *result = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (bsrc)); if (filter) { tmp = gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST); diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c index 243fc7189c..4ff7c1889d 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.c +++ b/gst-libs/gst/gl/gstglcolorconvert.c @@ -676,23 +676,21 @@ gst_gl_color_convert_caps_remove_format_info (GstCaps * caps) } GstCaps * -gst_gl_color_convert_transform_caps (GstGLContext * convert, +gst_gl_color_convert_transform_caps (GstGLContext * context, GstPadDirection direction, GstCaps * caps, GstCaps * filter) { - GstCaps *templ, *result; + GstCaps *templ, *tmp, *result; templ = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, GST_GL_COLOR_CONVERT_FORMATS)); - caps = gst_gl_color_convert_caps_remove_format_info (caps); - result = gst_caps_intersect (caps, templ); - gst_caps_unref (caps); + tmp = gst_gl_color_convert_caps_remove_format_info (caps); + result = gst_caps_intersect (tmp, templ); + gst_caps_unref (tmp); gst_caps_unref (templ); if (filter) { - GstCaps *tmp; - tmp = gst_caps_intersect_full (filter, result, GST_CAPS_INTERSECT_FIRST); gst_caps_unref (result); result = tmp; diff --git a/gst-libs/gst/gl/gstglfilter.c b/gst-libs/gst/gl/gstglfilter.c index 169ded9f9f..f06fb5dc4f 100644 --- a/gst-libs/gst/gl/gstglfilter.c +++ b/gst-libs/gst/gl/gstglfilter.c @@ -37,21 +37,12 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); static GstStaticPadTemplate gst_gl_filter_src_pad_template = -GST_STATIC_PAD_TEMPLATE ("src", + GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) - ); - -static GstStaticPadTemplate gst_gl_filter_sink_pad_template = -GST_STATIC_PAD_TEMPLATE ("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) + "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA")) ); /* Properties */ @@ -63,7 +54,8 @@ enum #define gst_gl_filter_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLFilter, gst_gl_filter, GST_TYPE_GL_BASE_FILTER, GST_DEBUG_CATEGORY_INIT (gst_gl_filter_debug, "glfilter", 0, - "glfilter element");); + "glfilter element"); + ); static void gst_gl_filter_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -94,6 +86,7 @@ gst_gl_filter_class_init (GstGLFilterClass * klass) { GObjectClass *gobject_class; GstElementClass *element_class; + GstCaps *upload_caps, *elem_caps, *templ; gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -118,8 +111,15 @@ gst_gl_filter_class_init (GstGLFilterClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&gst_gl_filter_src_pad_template)); + + elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA"); + upload_caps = gst_gl_upload_get_input_template_caps (); + templ = gst_caps_intersect (elem_caps, upload_caps); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&gst_gl_filter_sink_pad_template)); + gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, templ)); + gst_caps_unref (upload_caps); + gst_caps_unref (elem_caps); + gst_caps_unref (templ); } static void @@ -156,6 +156,11 @@ gst_gl_filter_reset (GstGLFilter * filter) { gst_caps_replace (&filter->out_caps, NULL); + if (filter->upload) { + gst_object_unref (filter->upload); + filter->upload = NULL; + } + if (filter->pool) { gst_object_unref (filter->pool); filter->pool = NULL; @@ -605,8 +610,7 @@ gst_gl_filter_caps_remove_size (GstCaps * caps) } static GstCaps * -gst_gl_filter_set_caps_features (const GstCaps * caps, - const gchar * feature_name) +gst_gl_filter_set_caps_features (GstCaps * caps, const gchar * feature_name) { GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); @@ -617,23 +621,48 @@ static GstCaps * gst_gl_filter_transform_caps (GstBaseTransform * bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter_caps) { + GstGLContext *context = GST_GL_BASE_FILTER (bt)->context; GstCaps *tmp = NULL; GstCaps *result = NULL; - result = gst_gl_filter_caps_remove_size (caps); - tmp = result; - GST_DEBUG_OBJECT (bt, "size removal returned caps %" GST_PTR_FORMAT, tmp); + if (direction == GST_PAD_SINK) { + result = gst_gl_upload_transform_caps (context, direction, caps, NULL); + } else { + tmp = + gst_gl_caps_replace_all_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + result = gst_caps_merge (gst_caps_ref (caps), tmp); + } - result = - gst_gl_filter_set_caps_features (tmp, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - gst_caps_unref (tmp); - tmp = result; + GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, result); + + tmp = gst_gl_filter_caps_remove_size (result); + gst_caps_unref (result); + result = tmp; + GST_DEBUG_OBJECT (bt, "size removal returned caps %" GST_PTR_FORMAT, result); + + tmp = + gst_gl_filter_set_caps_features (result, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + gst_caps_unref (result); + result = tmp; + + if (direction == GST_PAD_SRC) { + tmp = gst_gl_upload_transform_caps (context, direction, result, NULL); + gst_caps_unref (result); + result = tmp; + } else { + tmp = + gst_gl_caps_replace_all_caps_features (result, + GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); + result = gst_caps_merge (result, tmp); + } + GST_DEBUG_OBJECT (bt, "transfer returned %" GST_PTR_FORMAT, result); if (filter_caps) { - result = - gst_caps_intersect_full (filter_caps, tmp, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (tmp); - } else { + tmp = + gst_caps_intersect_full (filter_caps, result, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (result); result = tmp; } @@ -777,10 +806,11 @@ config_failed: static gboolean gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) { + GstGLFilter *filter = GST_GL_FILTER (trans); GstGLContext *context; GstBufferPool *pool = NULL; GstStructure *config; - GstCaps *caps; + GstCaps *caps, *upload_caps; guint min, max, size; gboolean update_pool; @@ -832,6 +862,18 @@ gst_gl_filter_decide_allocation (GstBaseTransform * trans, GstQuery * query) gst_object_unref (pool); + if (!filter->upload) + filter->upload = gst_gl_upload_new (context); + + upload_caps = gst_caps_copy (caps); + gst_caps_set_features (upload_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + if (!gst_gl_upload_set_caps (filter->upload, caps, upload_caps)) { + gst_caps_unref (upload_caps); + return FALSE; + } + gst_caps_unref (upload_caps); + return TRUE; } @@ -853,11 +895,18 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf, GstGLFilterClass *filter_class; guint in_tex, out_tex; GstVideoFrame gl_frame, out_frame; + GstBuffer *gl_buffer; gboolean ret; filter_class = GST_GL_FILTER_GET_CLASS (filter); - if (!gst_video_frame_map (&gl_frame, &filter->in_info, inbuf, + if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (filter->upload, + inbuf, &gl_buffer) || !gl_buffer) { + ret = FALSE; + goto gl_buf_error; + } + + if (!gst_video_frame_map (&gl_frame, &filter->in_info, gl_buffer, GST_MAP_READ | GST_MAP_GL)) { ret = FALSE; goto inbuf_error; @@ -883,6 +932,8 @@ gst_gl_filter_filter_texture (GstGLFilter * filter, GstBuffer * inbuf, unmap_out_error: gst_video_frame_unmap (&gl_frame); inbuf_error: + gst_buffer_unref (gl_buffer); +gl_buf_error: return ret; } diff --git a/gst-libs/gst/gl/gstglfilter.h b/gst-libs/gst/gl/gstglfilter.h index a5caf3211a..2a69f26c53 100644 --- a/gst-libs/gst/gl/gstglfilter.h +++ b/gst-libs/gst/gl/gstglfilter.h @@ -66,6 +66,8 @@ struct _GstGLFilter GstCaps *out_caps; /* */ + GstGLUpload *upload; + GLuint fbo; GLuint depthbuffer; diff --git a/gst-libs/gst/gl/gstglutils.c b/gst-libs/gst/gl/gstglutils.c index e4d9cad67d..48e1653fe8 100644 --- a/gst-libs/gst/gl/gstglutils.c +++ b/gst-libs/gst/gl/gstglutils.c @@ -881,26 +881,16 @@ gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align, } GstCaps * -gst_gl_caps_replace_all_caps_features (const GstCaps * caps, +gst_gl_caps_replace_all_caps_features (GstCaps * caps, const gchar * feature_name) { GstCaps *tmp = gst_caps_copy (caps); guint n = gst_caps_get_size (tmp); guint i = 0; - for (i = 0; i < n; i++) { - GstCapsFeatures *features = gst_caps_get_features (tmp, i); - if (features) { - guint n_f = gst_caps_features_get_size (features); - guint j = 0; - for (j = 0; j < n_f; j++) { - gst_caps_features_remove_id (features, - gst_caps_features_get_nth_id (features, j)); - } - } - - gst_caps_features_add (features, feature_name); - } + for (i = 0; i < n; i++) + gst_caps_set_features (tmp, i, + gst_caps_features_from_string (feature_name)); return tmp; } diff --git a/gst-libs/gst/gl/gstglutils.h b/gst-libs/gst/gl/gstglutils.h index c915bfb6fe..5b1933d06b 100644 --- a/gst-libs/gst/gl/gstglutils.h +++ b/gst-libs/gst/gl/gstglutils.h @@ -103,7 +103,7 @@ gboolean gst_gl_run_query (GstElement * element, GstQuery * query, GstPadDirection direction); gsize gst_gl_get_plane_data_size (GstVideoInfo * info, GstVideoAlignment * align, guint plane); -GstCaps * gst_gl_caps_replace_all_caps_features (const GstCaps * caps, +GstCaps * gst_gl_caps_replace_all_caps_features (GstCaps * caps, const gchar * feature_name); G_END_DECLS