plugins: util: Add a helper function to detect supported caps.

This helper function iterate all profiles and entrypoints belong
to the specified codec, query the VAConfigAttribRTFormat and list
all possible video formats.
This function is used by each codec to get the template sink caps
(for encode) or src caps(for decode) at register time, when just
all possible formats are listed and no need to be very accurate.
So there is no context created for the performance reason. Most
codecs just use YUV kinds of formats as the input/output, so we do
not include RGB kinds of formats. User can specified more formats
in extra_fmts(For example, jpeg may need BGRA) if needed.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/315>

gst/vaapi/gstvaapipluginutil.c

@@ -25,6 +25,7 @@
 #include "gstcompat.h"
 #include "gstvaapivideocontext.h"
 #include <gst/vaapi/gstvaapiprofilecaps.h>
+#include <gst/vaapi/gstvaapiutils.h>
 #if USE_DRM
 # include <gst/vaapi/gstvaapidisplay_drm.h>
 #endif
@@ -1138,3 +1139,107 @@ gst_vaapi_build_caps_from_formats (GArray * formats, gint min_width,
 
   return out_caps;
 }
+
+/**
+ * gst_vaapi_build_template_caps_by_codec:
+ * @display: a #GstVaapiDisplay
+ * @usage: used for encode, decode or postproc
+ * @codec: a #GstVaapiCodec specify the codec to detect
+ * @extra_fmts: a #GArray of extra #GstVideoFormat
+ *
+ * Called by vaapi elements to detect the all possible video formats belong to
+ * the specified codec and build the caps. Only YUV kinds of formats are detected
+ * because so far almost all codecs use YUV kinds of formats as input/output.
+ * extra_fmts can specified more formats to be included.
+ *
+ * Returns: a built #GstCaps if succeeds, or %NULL if error.
+ **/
+GstCaps *
+gst_vaapi_build_template_caps_by_codec (GstVaapiDisplay * display,
+    GstVaapiContextUsage usage, GstVaapiCodec codec, GArray * extra_fmts)
+{
+  GArray *profiles = NULL;
+  GArray *supported_fmts = NULL;
+  GstCaps *out_caps = NULL;
+  guint i, e;
+  GstVaapiProfile profile;
+  guint value;
+  guint chroma;
+  GstVaapiChromaType gst_chroma;
+  GstVaapiEntrypoint entrypoint_start, entrypoint_end;
+
+  if (usage == GST_VAAPI_CONTEXT_USAGE_ENCODE) {
+    profiles = gst_vaapi_display_get_encode_profiles (display);
+    entrypoint_start = GST_VAAPI_ENTRYPOINT_SLICE_ENCODE;
+    entrypoint_end = GST_VAAPI_ENTRYPOINT_SLICE_ENCODE_LP;
+  } else if (usage == GST_VAAPI_CONTEXT_USAGE_DECODE) {
+    profiles = gst_vaapi_display_get_decode_profiles (display);
+    entrypoint_start = GST_VAAPI_ENTRYPOINT_VLD;
+    entrypoint_end = GST_VAAPI_ENTRYPOINT_MOCO;
+  }
+  /* TODO: VPP */
+
+  if (!profiles)
+    goto out;
+
+  chroma = 0;
+  for (i = 0; i < profiles->len; i++) {
+    profile = g_array_index (profiles, GstVaapiProfile, i);
+    if (gst_vaapi_profile_get_codec (profile) != codec)
+      continue;
+
+    for (e = entrypoint_start; e <= entrypoint_end; e++) {
+      if (!gst_vaapi_get_config_attribute (display,
+              gst_vaapi_profile_get_va_profile (profile),
+              gst_vaapi_entrypoint_get_va_entrypoint (e),
+              VAConfigAttribRTFormat, &value))
+        continue;
+
+      chroma |= value;
+    }
+  }
+
+  if (!chroma)
+    goto out;
+
+  for (gst_chroma = GST_VAAPI_CHROMA_TYPE_YUV420;
+      gst_chroma <= GST_VAAPI_CHROMA_TYPE_YUV444_12BPP; gst_chroma++) {
+    GArray *fmts;
+    if (!(chroma & from_GstVaapiChromaType (gst_chroma)))
+      continue;
+
+    fmts = gst_vaapi_video_format_get_formats_by_chroma (gst_chroma);
+    if (!fmts)
+      continue;
+
+    /* One format can not belong to different chroma, no need to merge */
+    if (supported_fmts == NULL) {
+      supported_fmts = fmts;
+    } else {
+      for (i = 0; i < fmts->len; i++)
+        g_array_append_val (supported_fmts,
+            g_array_index (fmts, GstVideoFormat, i));
+      g_array_unref (fmts);
+    }
+  }
+
+  if (!supported_fmts)
+    goto out;
+
+  if (extra_fmts) {
+    for (i = 0; i < extra_fmts->len; i++)
+      g_array_append_val (supported_fmts,
+          g_array_index (extra_fmts, GstVideoFormat, i));
+  }
+
+  out_caps = gst_vaapi_build_caps_from_formats (supported_fmts, 1, 1,
+      G_MAXINT, G_MAXINT, TRUE);
+
+out:
+  if (profiles)
+    g_array_unref (profiles);
+  if (supported_fmts)
+    g_array_unref (supported_fmts);
+
+  return out_caps;
+}

gst/vaapi/gstvaapipluginutil.h

@@ -27,6 +27,7 @@
 
 #include <gst/vaapi/gstvaapidisplay.h>
 #include <gst/vaapi/gstvaapisurface.h>
+#include <gst/vaapi/gstvaapicontext.h>
 #include "gstvaapivideomemory.h"
 
 typedef GstVaapiProfile (*GstVaapiStrToProfileFunc) (const gchar * str);
@@ -165,4 +166,9 @@ GstCaps *
 gst_vaapi_build_caps_from_formats (GArray * formats, gint min_width,
     gint min_height, gint max_width, gint max_height, guint mem_type);
 
+G_GNUC_INTERNAL
+GstCaps *
+gst_vaapi_build_template_caps_by_codec (GstVaapiDisplay * display,
+    GstVaapiContextUsage usage, GstVaapiCodec codec, GArray * extra_fmts);
+
 #endif /* GST_VAAPI_PLUGIN_UTIL_H */