encoder: h264: allow target decoder constraints.

Allow user to precise the largest profile to use for encoding due
to target decoder constraints. For instance, if CABAC entropy coding
mode is requested by "constrained-baseline" profile only is desired,
then an error is returned during codec configuration.

Also make sure that the suitable profile we derived actually matches
what the HW can cope with.

https://bugzilla.gnome.org/show_bug.cgi?id=719694

gst-libs/gst/vaapi/gstvaapiencoder_h264.c

@@ -154,6 +154,37 @@ _check_sps_pps_status (GstVaapiEncoderH264 * encoder,
   }
 }
 
+/* Determines the largest supported profile by the underlying hardware */
+static gboolean
+ensure_hw_profile_limits (GstVaapiEncoderH264 * encoder)
+{
+  GstVaapiDisplay *const display = GST_VAAPI_ENCODER_DISPLAY (encoder);
+  GArray *profiles;
+  guint i, profile_idc, max_profile_idc;
+
+  if (encoder->hw_max_profile_idc)
+    return TRUE;
+
+  profiles = gst_vaapi_display_get_encode_profiles (display);
+  if (!profiles)
+    return FALSE;
+
+  max_profile_idc = 0;
+  for (i = 0; i < profiles->len; i++) {
+    const GstVaapiProfile profile =
+        g_array_index (profiles, GstVaapiProfile, i);
+    profile_idc = gst_vaapi_utils_h264_get_profile_idc (profile);
+    if (!profile_idc)
+      continue;
+    if (max_profile_idc < profile_idc)
+      max_profile_idc = profile_idc;
+  }
+  g_array_unref (profiles);
+
+  encoder->hw_max_profile_idc = max_profile_idc;
+  return TRUE;
+}
+
 /* Derives the profile supported by the underlying hardware */
 static gboolean
 ensure_hw_profile (GstVaapiEncoderH264 * encoder)
@@ -197,6 +228,36 @@ error_unsupported_profile:
   }
 }
 
+/* Check target decoder constraints */
+static gboolean
+ensure_profile_limits (GstVaapiEncoderH264 * encoder)
+{
+  GstVaapiProfile profile;
+
+  if (!encoder->max_profile_idc
+      || encoder->profile_idc <= encoder->max_profile_idc)
+    return TRUE;
+
+  GST_WARNING ("lowering coding tools to meet target decoder constraints");
+
+  /* Try Main profile coding tools */
+  if (encoder->max_profile_idc < 100) {
+    encoder->use_dct8x8 = FALSE;
+    profile = GST_VAAPI_PROFILE_H264_MAIN;
+  }
+
+  /* Try Constrained Baseline profile coding tools */
+  if (encoder->max_profile_idc < 77) {
+    encoder->num_bframes = 0;
+    encoder->use_cabac = FALSE;
+    profile = GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE;
+  }
+
+  encoder->profile = profile;
+  encoder->profile_idc = encoder->max_profile_idc;
+  return TRUE;
+}
+
 /* Derives the minimum profile from the active coding tools */
 static gboolean
 ensure_profile (GstVaapiEncoderH264 * encoder)
@@ -1230,14 +1291,21 @@ ensure_misc (GstVaapiEncoderH264 * encoder, GstVaapiEncPicture * picture)
   return TRUE;
 }
 
-static gboolean
+static GstVaapiEncoderStatus
 ensure_profile_and_level (GstVaapiEncoderH264 * encoder)
 {
-  if (!ensure_profile (encoder))
-    return FALSE;
+  if (!ensure_profile (encoder) || !ensure_profile_limits (encoder))
+    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
+
   if (!ensure_level (encoder))
-    return FALSE;
-  return TRUE;
+    return GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED;
+
+  /* Check HW constraints */
+  if (!ensure_hw_profile_limits (encoder))
+    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
+  if (encoder->profile_idc > encoder->hw_max_profile_idc)
+    return GST_VAAPI_ENCODER_STATUS_ERROR_UNSUPPORTED_PROFILE;
+  return GST_VAAPI_ENCODER_STATUS_SUCCESS;
 }
 
 static gboolean
@@ -1601,12 +1669,15 @@ gst_vaapi_encoder_h264_reconfigure (GstVaapiEncoder * base_encoder)
 {
   GstVaapiEncoderH264 *const encoder =
       GST_VAAPI_ENCODER_H264_CAST (base_encoder);
+  GstVaapiEncoderStatus status;
 
   encoder->mb_width = (GST_VAAPI_ENCODER_WIDTH (encoder) + 15) / 16;
   encoder->mb_height = (GST_VAAPI_ENCODER_HEIGHT (encoder) + 15) / 16;
 
-  if (!ensure_profile_and_level (encoder))
-    goto error;
+  status = ensure_profile_and_level (encoder);
+  if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
+    return status;
+
   if (!ensure_bitrate (encoder))
     goto error;
 
@@ -1819,6 +1890,40 @@ gst_vaapi_encoder_h264_get_default_properties (void)
   return props;
 }
 
+/**
+ * gst_vaapi_encoder_h264_set_max_profile:
+ * @encoder: a #GstVaapiEncoderH264
+ * @profile: an H.264 #GstVaapiProfile
+ *
+ * Notifies the @encoder to use coding tools from the supplied
+ * @profile at most.
+ *
+ * This means that if the minimal profile derived to
+ * support the specified coding tools is greater than this @profile,
+ * then an error is returned when the @encoder is configured.
+ *
+ * Return value: %TRUE on success
+ */
+gboolean
+gst_vaapi_encoder_h264_set_max_profile (GstVaapiEncoderH264 * encoder,
+    GstVaapiProfile profile)
+{
+  guint8 profile_idc;
+
+  g_return_val_if_fail (encoder != NULL, FALSE);
+  g_return_val_if_fail (profile != GST_VAAPI_PROFILE_UNKNOWN, FALSE);
+
+  if (gst_vaapi_profile_get_codec (profile) != GST_VAAPI_CODEC_H264)
+    return FALSE;
+
+  profile_idc = gst_vaapi_utils_h264_get_profile_idc (profile);
+  if (!profile_idc)
+    return FALSE;
+
+  encoder->max_profile_idc = profile_idc;
+  return TRUE;
+}
+
 /**
  * gst_vaapi_encoder_h264_get_profile_and_level:
  * @encoder: a #GstVaapiEncoderH264

gst-libs/gst/vaapi/gstvaapiencoder_h264.h

@@ -60,6 +60,10 @@ gst_vaapi_encoder_h264_new (GstVaapiDisplay * display);
 GPtrArray *
 gst_vaapi_encoder_h264_get_default_properties (void);
 
+gboolean
+gst_vaapi_encoder_h264_set_max_profile (GstVaapiEncoderH264 * encoder,
+    GstVaapiProfile profile);
+
 gboolean
 gst_vaapi_encoder_h264_get_profile_and_level (GstVaapiEncoderH264 * encoder,
     GstVaapiProfile * out_profile_ptr, GstVaapiLevelH264 * out_level_ptr);

gst-libs/gst/vaapi/gstvaapiencoder_h264_priv.h

@@ -39,6 +39,8 @@ struct _GstVaapiEncoderH264
   GstVaapiProfile profile;
   GstVaapiLevelH264 level;
   guint8 profile_idc;
+  guint8 max_profile_idc;
+  guint8 hw_max_profile_idc;
   guint8 level_idc;
   guint32 idr_period;
   guint32 init_qp;

gst-libs/gst/vaapi/gstvaapiutils_h264.c

@@ -127,6 +127,16 @@ map_lookup_name (const struct map *m, const gchar * name)
   return NULL;
 }
 
+/** Returns a relative score for the supplied GstVaapiProfile */
+guint
+gst_vaapi_utils_h264_get_profile_score (GstVaapiProfile profile)
+{
+  const struct map *const m =
+      map_lookup_value (gst_vaapi_h264_profile_map, profile);
+
+  return m ? 1 + (m - gst_vaapi_h264_profile_map) : 0;
+}
+
 /** Returns GstVaapiProfile from H.264 profile_idc value */
 GstVaapiProfile
 gst_vaapi_utils_h264_get_profile (guint8 profile_idc)

gst-libs/gst/vaapi/gstvaapiutils_h264.h

@@ -71,6 +71,10 @@ typedef enum
   GST_VAAPI_LEVEL_H264_L5_2,
 } GstVaapiLevelH264;
 
+/* Returns a relative score for the supplied GstVaapiProfile */
+guint
+gst_vaapi_utils_h264_get_profile_score (GstVaapiProfile profile);
+
 /* Returns GstVaapiProfile from a string representation */
 GstVaapiProfile
 gst_vaapi_utils_h264_get_profile_from_string (const gchar * str);

gst/vaapi/gstvaapiencode_h264.c

@@ -22,6 +22,7 @@
 #include "gst/vaapi/sysdeps.h"
 #include <gst/vaapi/gstvaapidisplay.h>
 #include <gst/vaapi/gstvaapiencoder_h264.h>
+#include <gst/vaapi/gstvaapiutils_h264.h>
 #include "gstvaapiencode_h264.h"
 #include "gstvaapipluginutil.h"
 #if GST_CHECK_VERSION(1,0,0)
@@ -59,7 +60,8 @@ static const char gst_vaapiencode_h264_sink_caps_str[] =
 
 /* *INDENT-OFF* */
 static const char gst_vaapiencode_h264_src_caps_str[] =
-  GST_CODEC_CAPS;
+  GST_CODEC_CAPS ", "
+  "profile = (string) { constrained-baseline, baseline, main, high }";
 /* *INDENT-ON* */
 
 /* *INDENT-OFF* */
@@ -123,6 +125,87 @@ gst_vaapiencode_h264_get_property (GObject * object,
   }
 }
 
+typedef struct
+{
+  GstVaapiProfile best_profile;
+  guint best_score;
+} FindBestProfileData;
+
+static void
+find_best_profile_value (FindBestProfileData * data, const GValue * value)
+{
+  const gchar *str;
+  GstVaapiProfile profile;
+  guint score;
+
+  if (!value || !G_VALUE_HOLDS_STRING (value))
+    return;
+
+  str = g_value_get_string (value);
+  if (!str)
+    return;
+  profile = gst_vaapi_utils_h264_get_profile_from_string (str);
+  if (!profile)
+    return;
+  score = gst_vaapi_utils_h264_get_profile_score (profile);
+  if (score < data->best_score)
+    return;
+  data->best_profile = profile;
+  data->best_score = score;
+}
+
+static GstVaapiProfile
+find_best_profile (GstCaps * caps)
+{
+  FindBestProfileData data;
+  guint i, j, num_structures, num_values;
+
+  data.best_profile = GST_VAAPI_PROFILE_UNKNOWN;
+  data.best_score = 0;
+
+  num_structures = gst_caps_get_size (caps);
+  for (i = 0; i < num_structures; i++) {
+    GstStructure *const structure = gst_caps_get_structure (caps, i);
+    const GValue *const value = gst_structure_get_value (structure, "profile");
+
+    if (!value)
+      continue;
+    if (G_VALUE_HOLDS_STRING (value))
+      find_best_profile_value (&data, value);
+    else if (GST_VALUE_HOLDS_LIST (value)) {
+      num_values = gst_value_list_get_size (value);
+      for (j = 0; j < num_values; j++)
+        find_best_profile_value (&data, gst_value_list_get_value (value, j));
+    }
+  }
+  return data.best_profile;
+}
+
+static gboolean
+gst_vaapiencode_h264_set_config (GstVaapiEncode * base_encode)
+{
+  GstVaapiEncoderH264 *const encoder =
+      GST_VAAPI_ENCODER_H264 (base_encode->encoder);
+  GstCaps *allowed_caps;
+  GstVaapiProfile profile;
+
+  /* Check for the largest profile that is supported */
+  allowed_caps =
+      gst_pad_get_allowed_caps (GST_VAAPI_PLUGIN_BASE_SRC_PAD (base_encode));
+  if (!allowed_caps)
+    return TRUE;
+
+  profile = find_best_profile (allowed_caps);
+  gst_caps_unref (allowed_caps);
+  if (profile) {
+    GST_INFO ("using %s profile as target decoder constraints",
+        gst_vaapi_utils_h264_get_profile_string (profile));
+    if (!gst_vaapi_encoder_h264_set_max_profile (encoder, profile))
+      return FALSE;
+  }
+  return TRUE;
+}
+
 static GstCaps *
 gst_vaapiencode_h264_get_caps (GstVaapiEncode * base_encode)
 {
@@ -305,6 +388,7 @@ gst_vaapiencode_h264_class_init (GstVaapiEncodeH264Class * klass)
   object_class->get_property = gst_vaapiencode_h264_get_property;
 
   encode_class->get_properties = gst_vaapi_encoder_h264_get_default_properties;
+  encode_class->set_config = gst_vaapiencode_h264_set_config;
   encode_class->get_caps = gst_vaapiencode_h264_get_caps;
   encode_class->alloc_encoder = gst_vaapiencode_h264_alloc_encoder;
   encode_class->alloc_buffer = gst_vaapiencode_h264_alloc_buffer;