From 8c93c842ef8a19e2b3c296d9e33c31c07afd6ab0 Mon Sep 17 00:00:00 2001 From: Gwenole Beauchesne Date: Mon, 9 Feb 2015 21:09:07 +0100 Subject: [PATCH] plugins: add support for BGRA textures. Some frameworks (EFL) expect BGRA textures for storage. However, adding support for that broadly into GStreamer framework implies two kinds of hacks: (i) libgstgl helpers currently do not support BGRA textures correctly, (ii) we need to better parse downstream suggested caps and intersect them with what the VA plugin elements can offer to them for GL texturing. --- gst/vaapi/gstvaapidecode.c | 15 ++++--- gst/vaapi/gstvaapipluginbase.c | 2 +- gst/vaapi/gstvaapipluginutil.c | 57 ++++++++++++++++++++++----- gst/vaapi/gstvaapipluginutil.h | 3 +- gst/vaapi/gstvaapipostproc.c | 11 +----- gst/vaapi/gstvaapivideometa_texture.c | 36 ++++++++++++----- 6 files changed, 85 insertions(+), 39 deletions(-) diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c index f42e6f8367..d2213c1663 100644 --- a/gst/vaapi/gstvaapidecode.c +++ b/gst/vaapi/gstvaapidecode.c @@ -79,7 +79,7 @@ static const char gst_vaapidecode_src_caps_str[] = GST_VIDEO_CAPS_MAKE_WITH_FEATURES( GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE, "{ ENCODED, I420, YV12, NV12 }") ";" GST_VIDEO_CAPS_MAKE_WITH_FEATURES( - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") ";" + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }") ";" GST_VIDEO_CAPS_MAKE("{ I420, YV12, NV12 }"); #else GST_VAAPI_SURFACE_CAPS; @@ -148,7 +148,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode, feature = gst_vaapi_find_preferred_caps_feature( GST_VIDEO_DECODER_SRC_PAD(vdec), - GST_VIDEO_INFO_FORMAT(&ref_state->info)); + GST_VIDEO_INFO_FORMAT(&ref_state->info), &out_format); #endif format = GST_VIDEO_INFO_FORMAT(&ref_state->info); @@ -160,9 +160,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode, return FALSE; vi = &state->info; - out_format = format; - if (format == GST_VIDEO_FORMAT_ENCODED) { - out_format = GST_VIDEO_FORMAT_I420; + if (format != out_format) { gst_video_info_init(&vis); gst_video_info_set_format(&vis, out_format, GST_VIDEO_INFO_WIDTH(vi), GST_VIDEO_INFO_HEIGHT(vi)); @@ -174,7 +172,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode, vis = *vi; switch (feature) { case GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META: - gst_video_info_change_format(&vis, GST_VIDEO_FORMAT_RGBA, + gst_video_info_change_format(&vis, out_format, GST_VIDEO_INFO_WIDTH(vi), GST_VIDEO_INFO_HEIGHT(vi)); features = gst_caps_features_new( GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL); @@ -524,6 +522,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query) gboolean need_pool; GstVideoCodecState *state; GstVaapiCapsFeature feature; + GstVideoFormat out_format; gst_query_parse_allocation(query, &caps, &need_pool); @@ -531,7 +530,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query) #if GST_CHECK_VERSION(1,1,0) && (USE_GLX || USE_EGL) decode->has_texture_upload_meta = gst_vaapi_find_preferred_caps_feature(GST_VIDEO_DECODER_SRC_PAD(vdec), - GST_VIDEO_FORMAT_ENCODED) == + GST_VIDEO_FORMAT_ENCODED, &out_format) == GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META; #endif @@ -543,7 +542,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query) state = gst_video_decoder_get_output_state(vdec); if (!gst_caps_is_always_compatible(caps, state->caps)) { if (decode->has_texture_upload_meta) - gst_video_info_change_format(&state->info, GST_VIDEO_FORMAT_RGBA, + gst_video_info_change_format(&state->info, out_format, GST_VIDEO_INFO_WIDTH(&state->info), GST_VIDEO_INFO_HEIGHT(&state->info)); gst_vaapidecode_update_src_caps(decode, state); diff --git a/gst/vaapi/gstvaapipluginbase.c b/gst/vaapi/gstvaapipluginbase.c index 130ed50941..718e5f3ae6 100644 --- a/gst/vaapi/gstvaapipluginbase.c +++ b/gst/vaapi/gstvaapipluginbase.c @@ -674,7 +674,7 @@ gst_vaapi_plugin_base_decide_allocation (GstVaapiPluginBase * plugin, if (!feature) feature = gst_vaapi_find_preferred_caps_feature (plugin->srcpad, - GST_VIDEO_FORMAT_ENCODED); + GST_VIDEO_FORMAT_ENCODED, NULL); has_video_meta = gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); diff --git a/gst/vaapi/gstvaapipluginutil.c b/gst/vaapi/gstvaapipluginutil.c index b6abb590fa..1bb7c97eb7 100644 --- a/gst/vaapi/gstvaapipluginutil.c +++ b/gst/vaapi/gstvaapipluginutil.c @@ -634,8 +634,20 @@ gst_vaapi_video_format_new_template_caps_with_features (GstVideoFormat format, return caps; } +static GstCaps * +new_gl_texture_upload_meta_caps (void) +{ +#if GST_CHECK_VERSION(1,1,0) + return gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES ( + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }")); +#else + return gst_caps_new_empty (); +#endif +} + GstVaapiCapsFeature -gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format) +gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format, + GstVideoFormat * out_format_ptr) { GstVaapiCapsFeature feature = GST_VAAPI_CAPS_FEATURE_SYSTEM_MEMORY; #if GST_CHECK_VERSION(1,1,0) @@ -645,29 +657,27 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format) GstCaps *sysmem_caps = NULL; GstCaps *vaapi_caps = NULL; GstCaps *out_caps; + GstVideoFormat out_format; out_caps = gst_pad_peer_query_caps (pad, NULL); if (!out_caps) goto cleanup; - gl_texture_upload_caps = - gst_vaapi_video_format_new_template_caps_with_features - (GST_VIDEO_FORMAT_RGBA, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); + out_format = format == GST_VIDEO_FORMAT_ENCODED ? + GST_VIDEO_FORMAT_I420 : format; + + gl_texture_upload_caps = new_gl_texture_upload_meta_caps (); if (!gl_texture_upload_caps) goto cleanup; - if (format == GST_VIDEO_FORMAT_ENCODED) - format = GST_VIDEO_FORMAT_I420; - vaapi_caps = - gst_vaapi_video_format_new_template_caps_with_features (format, + gst_vaapi_video_format_new_template_caps_with_features (out_format, GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE); if (!vaapi_caps) goto cleanup; sysmem_caps = - gst_vaapi_video_format_new_template_caps_with_features (format, + gst_vaapi_video_format_new_template_caps_with_features (out_format, GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY); if (!sysmem_caps) goto cleanup; @@ -707,6 +717,33 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format) #endif } + if (out_format_ptr) { +#if GST_CHECK_VERSION(1,1,0) + if (feature == GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META) { + GstStructure *structure; + gchar *format_str; + out_format = GST_VIDEO_FORMAT_UNKNOWN; + do { + caps = gst_caps_intersect_full (out_caps, gl_texture_upload_caps, + GST_CAPS_INTERSECT_FIRST); + if (!caps) + break; + structure = gst_caps_get_structure (caps, 0); + if (!structure) + break; + if (!gst_structure_get (structure, "format", G_TYPE_STRING, + &format_str, NULL)) + break; + out_format = gst_video_format_from_string (format_str); + g_free (format_str); + } while (0); + if (!out_format) + goto cleanup; +#endif + } + *out_format_ptr = out_format; + } + cleanup: gst_caps_replace (&gl_texture_upload_caps, NULL); gst_caps_replace (&sysmem_caps, NULL); diff --git a/gst/vaapi/gstvaapipluginutil.h b/gst/vaapi/gstvaapipluginutil.h index 0f6490ef2f..b72b01cad1 100644 --- a/gst/vaapi/gstvaapipluginutil.h +++ b/gst/vaapi/gstvaapipluginutil.h @@ -90,7 +90,8 @@ gst_vaapi_video_format_new_template_caps_with_features (GstVideoFormat format, G_GNUC_INTERNAL GstVaapiCapsFeature -gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format); +gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format, + GstVideoFormat * out_format_ptr); G_GNUC_INTERNAL const gchar * diff --git a/gst/vaapi/gstvaapipostproc.c b/gst/vaapi/gstvaapipostproc.c index e701b8cdff..e5d563084a 100644 --- a/gst/vaapi/gstvaapipostproc.c +++ b/gst/vaapi/gstvaapipostproc.c @@ -77,7 +77,7 @@ static const char gst_vaapipostproc_src_caps_str[] = GST_CAPS_INTERLACED_FALSE "; " #if GST_CHECK_VERSION(1,1,0) GST_VIDEO_CAPS_MAKE_WITH_FEATURES ( - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") ", " + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }") ", " GST_CAPS_INTERLACED_FALSE "; " #endif #if GST_CHECK_VERSION(1,0,0) @@ -1152,14 +1152,7 @@ gst_vaapipostproc_transform_caps_impl (GstBaseTransform * trans, feature = gst_vaapi_find_preferred_caps_feature (GST_BASE_TRANSFORM_SRC_PAD (trans), - out_format); - switch (feature) { - case GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META: - out_format = GST_VIDEO_FORMAT_RGBA; - break; - default: - break; - } + out_format, &out_format); #else out_format = GST_VIDEO_FORMAT_ENCODED; #endif diff --git a/gst/vaapi/gstvaapivideometa_texture.c b/gst/vaapi/gstvaapivideometa_texture.c index da5119e61d..5052145fe9 100644 --- a/gst/vaapi/gstvaapivideometa_texture.c +++ b/gst/vaapi/gstvaapivideometa_texture.c @@ -55,6 +55,11 @@ meta_texture_ensure_format (GstVaapiVideoMetaTexture * meta, meta->gl_format = GL_RGBA; meta->texture_type = GST_VIDEO_GL_TEXTURE_TYPE_RGBA; break; + case GST_VIDEO_FORMAT_BGRA: + meta->gl_format = GL_BGRA_EXT; + /* FIXME: add GST_VIDEO_GL_TEXTURE_TYPE_BGRA extension */ + meta->texture_type = GST_VIDEO_GL_TEXTURE_TYPE_RGBA; + break; default: goto error_unsupported_format; } @@ -67,19 +72,26 @@ error_unsupported_format: return FALSE; } -static void -meta_texture_ensure_size_from_buffer (GstVaapiVideoMetaTexture * meta, +static gboolean +meta_texture_ensure_info_from_buffer (GstVaapiVideoMetaTexture * meta, GstBuffer * buffer) { GstVideoMeta *vmeta; + GstVideoFormat format; if (!buffer || !(vmeta = gst_buffer_get_video_meta (buffer))) { + format = DEFAULT_FORMAT; meta->width = 0; meta->height = 0; } else { + const GstVideoFormatInfo *const fmt_info = + gst_video_format_get_info (vmeta->format); + format = (fmt_info && GST_VIDEO_FORMAT_INFO_IS_RGB (fmt_info)) ? + vmeta->format : DEFAULT_FORMAT; meta->width = vmeta->width; meta->height = vmeta->height; } + return meta_texture_ensure_format (meta, format); } static void @@ -102,9 +114,13 @@ meta_texture_new (void) return NULL; meta->texture = NULL; - meta_texture_ensure_format (meta, DEFAULT_FORMAT); - meta_texture_ensure_size_from_buffer (meta, NULL); + if (!meta_texture_ensure_info_from_buffer (meta, NULL)) + goto error; return meta; + +error: + meta_texture_free (meta); + return NULL; } static GstVaapiVideoMetaTexture * @@ -172,7 +188,9 @@ gst_buffer_add_texture_upload_meta (GstBuffer * buffer) if (!meta_texture) return FALSE; - meta_texture_ensure_size_from_buffer (meta_texture, buffer); + if (!meta_texture_ensure_info_from_buffer (meta_texture, buffer)) + goto error; + meta = gst_buffer_add_video_gl_texture_upload_meta (buffer, GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, &meta_texture->texture_type, gst_vaapi_texture_upload, @@ -193,10 +211,8 @@ gst_buffer_ensure_texture_upload_meta (GstBuffer * buffer) GstVideoGLTextureUploadMeta *const meta = gst_buffer_get_video_gl_texture_upload_meta (buffer); - if (meta) { - meta_texture_ensure_size_from_buffer (meta->user_data, buffer); - return TRUE; - } - return gst_buffer_add_texture_upload_meta (buffer); + return meta ? + meta_texture_ensure_info_from_buffer (meta->user_data, buffer) : + gst_buffer_add_texture_upload_meta (buffer); } #endif