libs: encoder: check for maximum number of slices

Right now, H264 and HEVC can set as a property the number of slices to
process. But each driver can set a maximum number of slices, depending
on the supported profile & entry point.

This patch verifies the current num_slices to process against the maximum
permitted by the driver and the media size.

https://bugzilla.gnome.org/show_bug.cgi?id=780955

gst-libs/gst/vaapi/gstvaapiencoder.c

@@ -1386,6 +1386,55 @@ gst_vaapi_encoder_get_surface_formats (GstVaapiEncoder * encoder,
   return formats;
 }
 
+/**
+ * gst_vaapi_encoder_ensure_num_slices:
+ * @encoder: a #GstVaapiEncoder
+ * @profile: a #GstVaapiProfile
+ * @entrypoint: a #GstVaapiEntrypoint
+ * @media_max_slices: the number of the slices permitted by the stream
+ * @num_slices: (out): the possible number of slices to process
+ *
+ * This function will clamp the @num_slices provided by the user,
+ * according the limit of the number of slices permitted by the stream
+ * and by the hardware.
+ *
+ * We need to pass the @profile and the @entrypoint, because at the
+ * moment the encoder base class, still doesn't have them assigned,
+ * and this function is meant to be called by the derived classes
+ * while they are configured.
+ *
+ * Returns: %TRUE if the number of slices is different than zero.
+ **/
+gboolean
+gst_vaapi_encoder_ensure_num_slices (GstVaapiEncoder * encoder,
+    GstVaapiProfile profile, GstVaapiEntrypoint entrypoint,
+    guint media_max_slices, guint * num_slices)
+{
+  VAProfile va_profile;
+  VAEntrypoint va_entrypoint;
+  guint max_slices, num;
+
+  va_profile = gst_vaapi_profile_get_va_profile (profile);
+  va_entrypoint = gst_vaapi_entrypoint_get_va_entrypoint (entrypoint);
+
+  if (!gst_vaapi_get_config_attribute (encoder->display, va_profile,
+          va_entrypoint, VAConfigAttribEncMaxSlices, &max_slices)) {
+    *num_slices = 1;
+    return TRUE;
+  }
+
+  num = *num_slices;
+  if (num > max_slices)
+    num = max_slices;
+  if (num > media_max_slices)
+    num = media_max_slices;
+
+  if (num == 0)
+    return FALSE;
+  *num_slices = num;
+  return TRUE;
+}
+
 /**
  * gst_vaapi_encoder_add_roi:
  * @encoder: a #GstVaapiEncoder

gst-libs/gst/vaapi/gstvaapiencoder_h264.c

@@ -2436,9 +2436,8 @@ reset_properties (GstVaapiEncoderH264 * encoder)
     encoder->min_qp = encoder->init_qp;
 
   mb_size = encoder->mb_width * encoder->mb_height;
-  if (encoder->num_slices > (mb_size + 1) / 2)
-    encoder->num_slices = (mb_size + 1) / 2;
-  g_assert (encoder->num_slices);
+  g_assert (gst_vaapi_encoder_ensure_num_slices (base_encoder, encoder->profile,
+          encoder->entrypoint, (mb_size + 1) / 2, &encoder->num_slices));
 
   if (encoder->num_bframes > (base_encoder->keyframe_period + 1) / 2)
     encoder->num_bframes = (base_encoder->keyframe_period + 1) / 2;

gst-libs/gst/vaapi/gstvaapiencoder_h265.c

@@ -1960,9 +1960,9 @@ reset_properties (GstVaapiEncoderH265 * encoder)
     encoder->min_qp = encoder->init_qp;
 
   ctu_size = encoder->ctu_width * encoder->ctu_height;
-  if (encoder->num_slices > (ctu_size + 1) / 2)
-    encoder->num_slices = (ctu_size + 1) / 2;
-  g_assert (encoder->num_slices);
+  g_assert (gst_vaapi_encoder_ensure_num_slices (base_encoder, encoder->profile,
+          GST_VAAPI_ENTRYPOINT_SLICE_ENCODE, (ctu_size + 1) / 2,
+          &encoder->num_slices));
 
   if (encoder->num_bframes > (base_encoder->keyframe_period + 1) / 2)
     encoder->num_bframes = (base_encoder->keyframe_period + 1) / 2;

gst-libs/gst/vaapi/gstvaapiencoder_priv.h

@@ -360,6 +360,12 @@ gboolean
 gst_vaapi_encoder_ensure_param_quality_level (GstVaapiEncoder * encoder,
     GstVaapiEncPicture * picture);
 
+G_GNUC_INTERNAL
+gboolean
+gst_vaapi_encoder_ensure_num_slices (GstVaapiEncoder * encoder,
+    GstVaapiProfile profile, GstVaapiEntrypoint entrypoint,
+    guint media_max_slices, guint * num_slices);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_ENCODER_PRIV_H */