plugin: encode: List all possible profiles to detect input formats.

The current get_profile just return one possible profile for the encode,
which is not enough.  For example, if we want to support HEVC 4:4:4
profile, the input of encode should be VYUA rather than NV12 in HEVC
main profile. So the command line:

gst-launch-1.0 videotestsrc num-buffers=200 ! capsfilter \
caps=video/x-raw,format=VUYA,width=800,height=600 ! vaapih265enc \
tune=low-power init-qp=30 ! fakesink

can not work because vaapih265enc just report NV12 in sink caps, we need
to specify the profile obviously like:

gst-launch-1.0 videotestsrc num-buffers=200 ! capsfilter \
caps=video/x-raw,format=VUYA,width=800,height=600 ! vaapih265enc \
tune=low-power init-qp=30 ! capsfilter caps=video/x-h265, \
profile=main-444 ! fakesink

The encode should have the ability to choose the profile based on input
format automatically. If the input video format is VUYA, the main-444
profile should be auto choosed.

We modify to let get_allowed_profiles of each encode sub class to return
an array of all supported profiles based on downstream's allowed caps, or
return NULL if no valid profiles specified by downstream.
If no allowed profiles found, all profiles which belong to the current
encoder's codec will be the candidates.
The function gst_vaapi_encoder_get_surface_attributes collects the surface's
attributes for that profile list we just get.

So for this case, both NV12 and VUYA should be returned.

TODO: some codec like VP9, need to implement the get_profile() function.

gst-libs/gst/vaapi/gstvaapiencoder.c

@@ -1554,61 +1554,44 @@ merge_profile_surface_attributes (GstVaapiEncoder * encoder,
 /**
  * gst_vaapi_encoder_get_surface_attributres:
  * @encoder: a #GstVaapiEncoder instances
- * @profile: a #GstVaapiProfile to test
+ * @profiles: a #GArray of #GstVaapiProfile to be test
  * @min_width (out): the minimal surface width
  * @min_height (out): the minimal surface height
  * @max_width (out): the maximal surface width
  * @max_height (out): the maximal surface height
  *
- * Fetches the valid surface's attributes for @profile if it is valid,
+ * Fetches the valid surface's attributes for the specified @profiles
- * Otherwise, it collects surface's attributes for all profiles which
- * belong to the current encoder's codec.
  *
  * Returns: a #GArray of valid formats we get or %NULL if failed.
  **/
 GArray *
 gst_vaapi_encoder_get_surface_attributes (GstVaapiEncoder * encoder,
-    GstVaapiProfile profile, gint * min_width, gint * min_height,
+    GArray * profiles, gint * min_width, gint * min_height,
     gint * max_width, gint * max_height)
 {
-  const GstVaapiEncoderClassData *const cdata =
-      GST_VAAPI_ENCODER_GET_CLASS (encoder)->class_data;
   GstVaapiConfigSurfaceAttributes attribs = {
     G_MAXINT, G_MAXINT, 1, 1, 0, NULL
   };
-  GArray *profiles;
+  GstVaapiProfile profile;
   guint i;
 
-  if (profile != GST_VAAPI_PROFILE_UNKNOWN) {
-    if (get_profile_surface_attributes (encoder, profile, &attribs))
-      goto success;
-    return NULL;
-  }
-
-  /* no specific context neither specific profile, let's iterate among
-   * the codec's profiles */
-  profiles = gst_vaapi_display_get_encode_profiles (encoder->display);
-  if (!profiles)
-    return NULL;
-
   attribs.formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
   for (i = 0; i < profiles->len; i++) {
     profile = g_array_index (profiles, GstVaapiProfile, i);
-    if (gst_vaapi_profile_get_codec (profile) == cdata->codec) {
+    g_assert (profile != GST_VAAPI_PROFILE_UNKNOWN);
+    GST_LOG ("Detect input formats of profile %s",
+        gst_vaapi_profile_get_va_name (profile));
+
     if (!merge_profile_surface_attributes (encoder, profile, &attribs)) {
       GST_INFO ("Can not get surface formats for profile %s",
           gst_vaapi_profile_get_va_name (profile));
       continue;
     }
   }
-  }
-
-  g_array_unref (profiles);
 
   if (!attribs.formats)
     return NULL;
 
-success:
   if (min_width)
     *min_width = attribs.min_width;
   if (min_height)

gst-libs/gst/vaapi/gstvaapiencoder.h

@@ -182,7 +182,7 @@ gst_vaapi_encoder_flush (GstVaapiEncoder * encoder);
 
 GArray *
 gst_vaapi_encoder_get_surface_attributes (GstVaapiEncoder * encoder,
-    GstVaapiProfile profile, gint * min_width, gint * min_height,
+    GArray * profiles, gint * min_width, gint * min_height,
     gint * max_width, gint * max_height);
 
 GstVaapiProfile

gst/vaapi/gstvaapiencode.c

@@ -340,30 +340,29 @@ gst_vaapiencode_buffer_loop (GstVaapiEncode * encode)
   gst_pad_pause_task (GST_VAAPI_PLUGIN_BASE_SRC_PAD (encode));
 }
 
-static GstVaapiProfile
+static GArray *
-get_profile (GstVaapiEncode * encode)
+get_profiles (GstVaapiEncode * encode)
 {
   GstVaapiEncodeClass *klass = GST_VAAPIENCODE_GET_CLASS (encode);
+  GArray *profiles = NULL;
 
-  if (klass->get_profile) {
+  if (klass->get_allowed_profiles) {
-    GstVaapiProfile profile = GST_VAAPI_PROFILE_UNKNOWN;
     GstCaps *allowed =
         gst_pad_get_allowed_caps (GST_VAAPI_PLUGIN_BASE_SRC_PAD (encode));
+    GST_LOG_OBJECT (encode,
+        "Get allowed sink caps from downstream %" GST_PTR_FORMAT, allowed);
+    if (allowed && !gst_caps_is_empty (allowed) && !gst_caps_is_any (allowed))
+      profiles = klass->get_allowed_profiles (encode, allowed);
 
-    if (allowed) {
+    if (allowed)
-      if (!gst_caps_is_empty (allowed) && !gst_caps_is_any (allowed))
-        profile = klass->get_profile (allowed);
       gst_caps_unref (allowed);
+
+    if (profiles)
+      return profiles;
   }
 
-    if (profile != GST_VAAPI_PROFILE_UNKNOWN)
+  profiles = gst_vaapi_encoder_get_available_profiles (encode->encoder);
-      return profile;
+  return profiles;
-  }
-
-  if (encode->encoder)
-    return gst_vaapi_encoder_get_profile (encode->encoder);
-
-  return GST_VAAPI_PROFILE_UNKNOWN;
 }
 
 static gboolean
@@ -374,7 +373,7 @@ ensure_allowed_sinkpad_caps (GstVaapiEncode * encode)
   GstCaps *va_caps, *dma_caps;
   GArray *formats = NULL;
   gboolean ret = FALSE;
-  GstVaapiProfile profile;
+  GArray *profiles = NULL;
   guint i, size;
   GstStructure *structure;
   gint min_width, min_height, max_width, max_height;
@@ -384,11 +383,14 @@ ensure_allowed_sinkpad_caps (GstVaapiEncode * encode)
   if (!encode->encoder)
     return TRUE;
 
-  profile = get_profile (encode);
+  /* First, get all possible profiles. */
+  profiles = get_profiles (encode);
+  if (profiles == NULL)
+    goto failed_get_profiles;
 
-  /* First get all supported formats, all these formats should be recognized
+  /* Then get all supported formats, all these formats should be recognized
      in video-format map. */
-  formats = gst_vaapi_encoder_get_surface_attributes (encode->encoder, profile,
+  formats = gst_vaapi_encoder_get_surface_attributes (encode->encoder, profiles,
       &min_width, &min_height, &max_width, &max_height);
   if (!formats)
     goto failed_get_attributes;
@@ -431,6 +433,8 @@ bail:
   if (!encode->allowed_sinkpad_caps)
     encode->allowed_sinkpad_caps = gst_caps_new_empty ();
 
+  if (profiles)
+    g_array_unref (profiles);
   if (out_caps)
     gst_caps_unref (out_caps);
   if (raw_caps)
@@ -449,6 +453,11 @@ failed_create_raw_caps:
     GST_WARNING_OBJECT (encode, "failed to create raw sink caps");
     goto bail;
   }
+failed_get_profiles:
+  {
+    GST_WARNING_OBJECT (encode, "failed to get supported profiles");
+    goto bail;
+  }
 }
 
 static GstCaps *

gst/vaapi/gstvaapiencode.h

@@ -81,7 +81,9 @@ struct _GstVaapiEncodeClass
   GstFlowReturn       (*alloc_buffer)   (GstVaapiEncode * encode,
                                          GstVaapiCodedBuffer * coded_buf,
                                          GstBuffer ** outbuf_ptr);
-  GstVaapiProfile     (*get_profile)    (GstCaps * caps);
+  /* Get all possible profiles based on allowed caps */
+  GArray *            (*get_allowed_profiles)  (GstVaapiEncode * encode,
+                                                GstCaps * allowed);
 
 #if USE_H264_FEI_ENCODER
 

gst/vaapi/gstvaapiencode_h264.c

@@ -128,23 +128,12 @@ gst_vaapiencode_h264_finalize (GObject * object)
   G_OBJECT_CLASS (gst_vaapiencode_h264_parent_class)->finalize (object);
 }
 
-static GstVaapiProfile
+static GArray *
-gst_vaapiencode_h264_get_profile (GstCaps * caps)
+gst_vaapiencode_h264_get_allowed_profiles (GstVaapiEncode * encode,
+    GstCaps * allowed)
 {
-  guint i;
+  return gst_vaapi_h26x_encoder_get_profiles_from_caps (allowed,
-
+      gst_vaapi_utils_h264_get_profile_from_string);
-  for (i = 0; i < gst_caps_get_size (caps); i++) {
-    GstStructure *const structure = gst_caps_get_structure (caps, i);
-    const GValue *const value = gst_structure_get_value (structure, "profile");
-
-    if (value && G_VALUE_HOLDS_STRING (value)) {
-      const gchar *str = g_value_get_string (value);
-      if (str)
-        return gst_vaapi_utils_h264_get_profile_from_string (str);
-    }
-  }
-
-  return GST_VAAPI_PROFILE_UNKNOWN;
 }
 
 typedef struct
@@ -556,7 +545,8 @@ gst_vaapiencode_h264_class_init (GstVaapiEncodeH264Class * klass)
   object_class->set_property = gst_vaapiencode_set_property_subclass;
   object_class->get_property = gst_vaapiencode_get_property_subclass;
 
-  encode_class->get_profile = gst_vaapiencode_h264_get_profile;
+  encode_class->get_allowed_profiles =
+      gst_vaapiencode_h264_get_allowed_profiles;
   encode_class->set_config = gst_vaapiencode_h264_set_config;
   encode_class->get_caps = gst_vaapiencode_h264_get_caps;
   encode_class->alloc_encoder = gst_vaapiencode_h264_alloc_encoder;

gst/vaapi/gstvaapiencode_h265.c

@@ -99,23 +99,12 @@ gst_vaapiencode_h265_finalize (GObject * object)
   G_OBJECT_CLASS (gst_vaapiencode_h265_parent_class)->finalize (object);
 }
 
-static GstVaapiProfile
+static GArray *
-gst_vaapiencode_h265_get_profile (GstCaps * caps)
+gst_vaapiencode_h265_get_allowed_profiles (GstVaapiEncode * encode,
+    GstCaps * allowed)
 {
-  guint i;
+  return gst_vaapi_h26x_encoder_get_profiles_from_caps (allowed,
-
+      gst_vaapi_utils_h265_get_profile_from_string);
-  for (i = 0; i < gst_caps_get_size (caps); i++) {
-    GstStructure *const structure = gst_caps_get_structure (caps, i);
-    const GValue *const value = gst_structure_get_value (structure, "profile");
-
-    if (value && G_VALUE_HOLDS_STRING (value)) {
-      const gchar *str = g_value_get_string (value);
-      if (str)
-        return gst_vaapi_utils_h265_get_profile_from_string (str);
-    }
-  }
-
-  return GST_VAAPI_PROFILE_UNKNOWN;
 }
 
 typedef struct
@@ -386,7 +375,8 @@ gst_vaapiencode_h265_class_init (GstVaapiEncodeH265Class * klass)
   object_class->set_property = gst_vaapiencode_set_property_subclass;
   object_class->get_property = gst_vaapiencode_get_property_subclass;
 
-  encode_class->get_profile = gst_vaapiencode_h265_get_profile;
+  encode_class->get_allowed_profiles =
+      gst_vaapiencode_h265_get_allowed_profiles;
   encode_class->set_config = gst_vaapiencode_h265_set_config;
   encode_class->get_caps = gst_vaapiencode_h265_get_caps;
   encode_class->alloc_encoder = gst_vaapiencode_h265_alloc_encoder;