plugins: add support for BGRA textures.

Some frameworks (EFL) expect BGRA textures for storage. However,
adding support for that broadly into GStreamer framework  implies
two kinds of hacks: (i) libgstgl helpers currently do not support
BGRA textures correctly, (ii) we need to better parse downstream
suggested caps and intersect them with what the VA plugin elements
can offer to them for GL texturing.

gst/vaapi/gstvaapidecode.c

@@ -79,7 +79,7 @@ static const char gst_vaapidecode_src_caps_str[] =
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES(
         GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE, "{ ENCODED, I420, YV12, NV12 }") ";"
     GST_VIDEO_CAPS_MAKE_WITH_FEATURES(
-        GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") ";"
+        GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }") ";"
     GST_VIDEO_CAPS_MAKE("{ I420, YV12, NV12 }");
 #else
     GST_VAAPI_SURFACE_CAPS;
@@ -148,7 +148,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode,
 
     feature = gst_vaapi_find_preferred_caps_feature(
         GST_VIDEO_DECODER_SRC_PAD(vdec),
-        GST_VIDEO_INFO_FORMAT(&ref_state->info));
+        GST_VIDEO_INFO_FORMAT(&ref_state->info), &out_format);
 #endif
 
     format = GST_VIDEO_INFO_FORMAT(&ref_state->info);
@@ -160,9 +160,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode,
         return FALSE;
 
     vi = &state->info;
-    out_format = format;
+    if (format != out_format) {
-    if (format == GST_VIDEO_FORMAT_ENCODED) {
-        out_format = GST_VIDEO_FORMAT_I420;
         gst_video_info_init(&vis);
         gst_video_info_set_format(&vis, out_format,
             GST_VIDEO_INFO_WIDTH(vi), GST_VIDEO_INFO_HEIGHT(vi));
@@ -174,7 +172,7 @@ gst_vaapidecode_update_src_caps(GstVaapiDecode *decode,
     vis = *vi;
     switch (feature) {
     case GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META:
-        gst_video_info_change_format(&vis, GST_VIDEO_FORMAT_RGBA,
+        gst_video_info_change_format(&vis, out_format,
             GST_VIDEO_INFO_WIDTH(vi), GST_VIDEO_INFO_HEIGHT(vi));
         features = gst_caps_features_new(
             GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL);
@@ -524,6 +522,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query)
     gboolean need_pool;
     GstVideoCodecState *state;
     GstVaapiCapsFeature feature;
+    GstVideoFormat out_format;
 
     gst_query_parse_allocation(query, &caps, &need_pool);
 
@@ -531,7 +530,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query)
 #if GST_CHECK_VERSION(1,1,0) && (USE_GLX || USE_EGL)
     decode->has_texture_upload_meta =
         gst_vaapi_find_preferred_caps_feature(GST_VIDEO_DECODER_SRC_PAD(vdec),
-            GST_VIDEO_FORMAT_ENCODED) ==
+            GST_VIDEO_FORMAT_ENCODED, &out_format) ==
         GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META;
 #endif
 
@@ -543,7 +542,7 @@ gst_vaapidecode_decide_allocation(GstVideoDecoder *vdec, GstQuery *query)
     state = gst_video_decoder_get_output_state(vdec);
     if (!gst_caps_is_always_compatible(caps, state->caps)) {
         if (decode->has_texture_upload_meta)
-            gst_video_info_change_format(&state->info, GST_VIDEO_FORMAT_RGBA,
+            gst_video_info_change_format(&state->info, out_format,
                 GST_VIDEO_INFO_WIDTH(&state->info),
                 GST_VIDEO_INFO_HEIGHT(&state->info));
         gst_vaapidecode_update_src_caps(decode, state);

gst/vaapi/gstvaapipluginbase.c

@@ -674,7 +674,7 @@ gst_vaapi_plugin_base_decide_allocation (GstVaapiPluginBase * plugin,
   if (!feature)
     feature =
         gst_vaapi_find_preferred_caps_feature (plugin->srcpad,
-        GST_VIDEO_FORMAT_ENCODED);
+        GST_VIDEO_FORMAT_ENCODED, NULL);
 
   has_video_meta = gst_query_find_allocation_meta (query,
       GST_VIDEO_META_API_TYPE, NULL);

gst/vaapi/gstvaapipluginutil.c

@@ -634,8 +634,20 @@ gst_vaapi_video_format_new_template_caps_with_features (GstVideoFormat format,
   return caps;
 }
 
+static GstCaps *
+new_gl_texture_upload_meta_caps (void)
+{
+#if GST_CHECK_VERSION(1,1,0)
+  return gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (
+      GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }"));
+#else
+  return gst_caps_new_empty ();
+#endif
+}
+
 GstVaapiCapsFeature
-gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format)
+gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format,
+    GstVideoFormat * out_format_ptr)
 {
   GstVaapiCapsFeature feature = GST_VAAPI_CAPS_FEATURE_SYSTEM_MEMORY;
 #if GST_CHECK_VERSION(1,1,0)
@@ -645,29 +657,27 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format)
   GstCaps *sysmem_caps = NULL;
   GstCaps *vaapi_caps = NULL;
   GstCaps *out_caps;
+  GstVideoFormat out_format;
 
   out_caps = gst_pad_peer_query_caps (pad, NULL);
   if (!out_caps)
     goto cleanup;
 
-  gl_texture_upload_caps =
+  out_format = format == GST_VIDEO_FORMAT_ENCODED ?
-      gst_vaapi_video_format_new_template_caps_with_features
+    GST_VIDEO_FORMAT_I420 : format;
-      (GST_VIDEO_FORMAT_RGBA,
+
-      GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META);
+  gl_texture_upload_caps = new_gl_texture_upload_meta_caps ();
   if (!gl_texture_upload_caps)
     goto cleanup;
 
-  if (format == GST_VIDEO_FORMAT_ENCODED)
-    format = GST_VIDEO_FORMAT_I420;
-
   vaapi_caps =
-      gst_vaapi_video_format_new_template_caps_with_features (format,
+      gst_vaapi_video_format_new_template_caps_with_features (out_format,
       GST_CAPS_FEATURE_MEMORY_VAAPI_SURFACE);
   if (!vaapi_caps)
     goto cleanup;
 
   sysmem_caps =
-      gst_vaapi_video_format_new_template_caps_with_features (format,
+      gst_vaapi_video_format_new_template_caps_with_features (out_format,
       GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY);
   if (!sysmem_caps)
     goto cleanup;
@@ -707,6 +717,33 @@ gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format)
 #endif
   }
 
+  if (out_format_ptr) {
+#if GST_CHECK_VERSION(1,1,0)
+    if (feature == GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META) {
+      GstStructure *structure;
+      gchar *format_str;
+      out_format = GST_VIDEO_FORMAT_UNKNOWN;
+      do {
+        caps = gst_caps_intersect_full (out_caps, gl_texture_upload_caps,
+            GST_CAPS_INTERSECT_FIRST);
+        if (!caps)
+          break;
+        structure = gst_caps_get_structure (caps, 0);
+        if (!structure)
+          break;
+        if (!gst_structure_get (structure, "format", G_TYPE_STRING,
+                &format_str, NULL))
+          break;
+        out_format = gst_video_format_from_string (format_str);
+        g_free (format_str);
+      } while (0);
+      if (!out_format)
+        goto cleanup;
+#endif
+    }
+    *out_format_ptr = out_format;
+  }
+
 cleanup:
   gst_caps_replace (&gl_texture_upload_caps, NULL);
   gst_caps_replace (&sysmem_caps, NULL);

gst/vaapi/gstvaapipluginutil.h

@@ -90,7 +90,8 @@ gst_vaapi_video_format_new_template_caps_with_features (GstVideoFormat format,
 
 G_GNUC_INTERNAL
 GstVaapiCapsFeature
-gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format);
+gst_vaapi_find_preferred_caps_feature (GstPad * pad, GstVideoFormat format,
+    GstVideoFormat * out_format_ptr);
 
 G_GNUC_INTERNAL
 const gchar *

gst/vaapi/gstvaapipostproc.c

@@ -77,7 +77,7 @@ static const char gst_vaapipostproc_src_caps_str[] =
   GST_CAPS_INTERLACED_FALSE "; "
 #if GST_CHECK_VERSION(1,1,0)
   GST_VIDEO_CAPS_MAKE_WITH_FEATURES (
-      GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") ", "
+      GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "{ RGBA, BGRA }") ", "
   GST_CAPS_INTERLACED_FALSE "; "
 #endif
 #if GST_CHECK_VERSION(1,0,0)
@@ -1152,14 +1152,7 @@ gst_vaapipostproc_transform_caps_impl (GstBaseTransform * trans,
 
   feature =
       gst_vaapi_find_preferred_caps_feature (GST_BASE_TRANSFORM_SRC_PAD (trans),
-      out_format);
+      out_format, &out_format);
-  switch (feature) {
-    case GST_VAAPI_CAPS_FEATURE_GL_TEXTURE_UPLOAD_META:
-      out_format = GST_VIDEO_FORMAT_RGBA;
-      break;
-    default:
-      break;
-  }
 #else
   out_format = GST_VIDEO_FORMAT_ENCODED;
 #endif

gst/vaapi/gstvaapivideometa_texture.c

@@ -55,6 +55,11 @@ meta_texture_ensure_format (GstVaapiVideoMetaTexture * meta,
       meta->gl_format = GL_RGBA;
       meta->texture_type = GST_VIDEO_GL_TEXTURE_TYPE_RGBA;
       break;
+    case GST_VIDEO_FORMAT_BGRA:
+      meta->gl_format = GL_BGRA_EXT;
+      /* FIXME: add GST_VIDEO_GL_TEXTURE_TYPE_BGRA extension */
+      meta->texture_type = GST_VIDEO_GL_TEXTURE_TYPE_RGBA;
+      break;
     default:
       goto error_unsupported_format;
   }
@@ -67,19 +72,26 @@ error_unsupported_format:
   return FALSE;
 }
 
-static void
+static gboolean
-meta_texture_ensure_size_from_buffer (GstVaapiVideoMetaTexture * meta,
+meta_texture_ensure_info_from_buffer (GstVaapiVideoMetaTexture * meta,
     GstBuffer * buffer)
 {
   GstVideoMeta *vmeta;
+  GstVideoFormat format;
 
   if (!buffer || !(vmeta = gst_buffer_get_video_meta (buffer))) {
+    format = DEFAULT_FORMAT;
     meta->width = 0;
     meta->height = 0;
   } else {
+    const GstVideoFormatInfo *const fmt_info =
+        gst_video_format_get_info (vmeta->format);
+    format = (fmt_info && GST_VIDEO_FORMAT_INFO_IS_RGB (fmt_info)) ?
+        vmeta->format : DEFAULT_FORMAT;
     meta->width = vmeta->width;
     meta->height = vmeta->height;
   }
+  return meta_texture_ensure_format (meta, format);
 }
 
 static void
@@ -102,9 +114,13 @@ meta_texture_new (void)
     return NULL;
 
   meta->texture = NULL;
-  meta_texture_ensure_format (meta, DEFAULT_FORMAT);
+  if (!meta_texture_ensure_info_from_buffer (meta, NULL))
-  meta_texture_ensure_size_from_buffer (meta, NULL);
+    goto error;
   return meta;
+
+error:
+  meta_texture_free (meta);
+  return NULL;
 }
 
 static GstVaapiVideoMetaTexture *
@@ -172,7 +188,9 @@ gst_buffer_add_texture_upload_meta (GstBuffer * buffer)
   if (!meta_texture)
     return FALSE;
 
-  meta_texture_ensure_size_from_buffer (meta_texture, buffer);
+  if (!meta_texture_ensure_info_from_buffer (meta_texture, buffer))
+    goto error;
+
   meta = gst_buffer_add_video_gl_texture_upload_meta (buffer,
       GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL,
       1, &meta_texture->texture_type, gst_vaapi_texture_upload,
@@ -193,10 +211,8 @@ gst_buffer_ensure_texture_upload_meta (GstBuffer * buffer)
   GstVideoGLTextureUploadMeta *const meta =
       gst_buffer_get_video_gl_texture_upload_meta (buffer);
 
-  if (meta) {
+  return meta ?
-    meta_texture_ensure_size_from_buffer (meta->user_data, buffer);
+      meta_texture_ensure_info_from_buffer (meta->user_data, buffer) :
-    return TRUE;
+      gst_buffer_add_texture_upload_meta (buffer);
-  }
-  return gst_buffer_add_texture_upload_meta (buffer);
 }
 #endif