encoder: h264: expose more coding tools.

Add new H.264 coding tools to improve compression:
- "cabac": enable CABAC entropy coding (default: FALSE);
- "dct8x8": enable spatial transform 8x8 (default: FALSE).

https://bugzilla.gnome.org/show_bug.cgi?id=719693

Signed-off-by: Gwenole Beauchesne <gwenole.beauchesne@intel.com>

gst-libs/gst/vaapi/gstvaapiencoder_h264.c

@@ -53,12 +53,6 @@
 #define GST_VAAPI_ENCODER_H264_NAL_REF_IDC_MEDIUM      2
 #define GST_VAAPI_ENCODER_H264_NAL_REF_IDC_HIGH        3
 
-typedef enum
-{
-  GST_VAAPI_ENCODER_H264_ENTROPY_MODE_CAVLC = 0,
-  GST_VAAPI_ENCODER_H264_ENTROPY_MODE_CABAC = 1
-} GstVaapiEncoderH264EntropyMode;
-
 typedef enum
 {
   GST_VAAPI_ENCODER_H264_NAL_UNKNOWN = 0,
@@ -214,9 +208,13 @@ ensure_profile (GstVaapiEncoderH264 * encoder)
   profile = GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE;
 
   /* Main profile coding tools */
-  if (encoder->num_bframes > 0)
+  if (encoder->num_bframes > 0 || encoder->use_cabac)
     profile = GST_VAAPI_PROFILE_H264_MAIN;
 
+  /* High profile coding tools */
+  if (encoder->use_dct8x8)
+    profile = GST_VAAPI_PROFILE_H264_HIGH;
+
   encoder->profile = profile;
   encoder->profile_idc = gst_vaapi_utils_h264_get_profile_idc (profile);
   return TRUE;
@@ -966,12 +964,11 @@ fill_va_picture_param (GstVaapiEncoderH264 * encoder,
       GST_VAAPI_ENC_PICTURE_IS_IDR (picture);
   pic_param->pic_fields.bits.reference_pic_flag =
       (picture->type != GST_VAAPI_PICTURE_TYPE_B);
-  pic_param->pic_fields.bits.entropy_coding_mode_flag =
+  pic_param->pic_fields.bits.entropy_coding_mode_flag = encoder->use_cabac;
-      GST_VAAPI_ENCODER_H264_ENTROPY_MODE_CABAC;
   pic_param->pic_fields.bits.weighted_pred_flag = FALSE;
   pic_param->pic_fields.bits.weighted_bipred_idc = 0;
   pic_param->pic_fields.bits.constrained_intra_pred_flag = 0;
-  pic_param->pic_fields.bits.transform_8x8_mode_flag = (encoder->profile_idc >= 100);       /* enable 8x8 */
+  pic_param->pic_fields.bits.transform_8x8_mode_flag = encoder->use_dct8x8;
   /* enable debloking */
   pic_param->pic_fields.bits.deblocking_filter_control_present_flag = TRUE;
   pic_param->pic_fields.bits.redundant_pic_cnt_present_flag = FALSE;
@@ -1685,6 +1682,12 @@ gst_vaapi_encoder_h264_set_property (GstVaapiEncoder * base_encoder,
     case GST_VAAPI_ENCODER_H264_PROP_NUM_SLICES:
       encoder->num_slices = g_value_get_uint (value);
       break;
+    case GST_VAAPI_ENCODER_H264_PROP_CABAC:
+      encoder->use_cabac = g_value_get_boolean (value);
+      break;
+    case GST_VAAPI_ENCODER_H264_PROP_DCT8X8:
+      encoder->use_dct8x8 = g_value_get_boolean (value);
+      break;
     default:
       return GST_VAAPI_ENCODER_STATUS_ERROR_INVALID_PARAMETER;
   }
@@ -1785,6 +1788,34 @@ gst_vaapi_encoder_h264_get_default_properties (void)
           "Number of slices per frame",
           1, 200, 1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  /**
+   * GstVaapiEncoderH264:cabac:
+   *
+   * Enable CABAC entropy coding mode for improved compression ratio,
+   * at the expense that the minimum target profile is Main. Default
+   * is CAVLC entropy coding mode.
+   */
+  GST_VAAPI_ENCODER_PROPERTIES_APPEND (props,
+      GST_VAAPI_ENCODER_H264_PROP_CABAC,
+      g_param_spec_boolean ("cabac",
+          "Enable CABAC",
+          "Enable CABAC entropy coding mode",
+          FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  /**
+   * GstVaapiEncoderH264:dct8x8:
+   *
+   * Enable adaptive use of 8x8 transforms in I-frames. This improves
+   * the compression ratio by the minimum target profile is High.
+   * Default is to use 4x4 DCT only.
+   */
+  GST_VAAPI_ENCODER_PROPERTIES_APPEND (props,
+      GST_VAAPI_ENCODER_H264_PROP_DCT8X8,
+      g_param_spec_boolean ("dct8x8",
+          "Enable 8x8 DCT",
+          "Enable adaptive use of 8x8 transforms in I-frames",
+          FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
   return props;
 }
 

gst-libs/gst/vaapi/gstvaapiencoder_h264.h

@@ -39,6 +39,9 @@ typedef struct _GstVaapiEncoderH264 GstVaapiEncoderH264;
  * @GST_VAAPI_ENCODER_H264_PROP_INIT_QP: Initial quantizer value (uint).
  * @GST_VAAPI_ENCODER_H264_PROP_MIN_QP: Minimal quantizer value (uint).
  * @GST_VAAPI_ENCODER_H264_PROP_NUM_SLICES: Number of slices per frame (uint).
+ * @GST_VAAPI_ENCODER_H264_PROP_CABAC: Enable CABAC entropy coding mode (bool).
+ * @GST_VAAPI_ENCODER_H264_PROP_DCT8X8: Enable adaptive use of 8x8
+ *   transforms in I-frames (bool).
  *
  * The set of H.264 encoder specific configurable properties.
  */
@@ -47,6 +50,8 @@ typedef enum {
   GST_VAAPI_ENCODER_H264_PROP_INIT_QP = -2,
   GST_VAAPI_ENCODER_H264_PROP_MIN_QP = -3,
   GST_VAAPI_ENCODER_H264_PROP_NUM_SLICES = -4,
+  GST_VAAPI_ENCODER_H264_PROP_CABAC = -5,
+  GST_VAAPI_ENCODER_H264_PROP_DCT8X8 = -6,
 } GstVaapiEncoderH264Prop;
 
 GstVaapiEncoder *

gst-libs/gst/vaapi/gstvaapiencoder_h264_priv.h

@@ -47,6 +47,8 @@ struct _GstVaapiEncoderH264
   guint32 num_bframes;
   guint32 mb_width;
   guint32 mb_height;
+  gboolean use_cabac;
+  gboolean use_dct8x8;
 
   /* re-ordering */
   GQueue reorder_frame_list;