libs: encoder: Enable trellis quantization method.

The advanced trellis algorithm is supported in VA driver. We add
its support as a property named "trellis" of encoder.
It only works for H264 now, should be more in future.

gst-libs/gst/vaapi/gstvaapiencoder.c

@@ -206,6 +206,21 @@ gst_vaapi_encoder_properties_get_default (const GstVaapiEncoderClass * klass)
           "higher value means lower-quality)",
           -10, 10, -10, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+
+  /**
+   * GstVaapiEncoder: trellis:
+   *
+   * The trellis quantization method the encoder can use.
+   * Trellis is an improved quantization algorithm.
+   *
+   */
+  GST_VAAPI_ENCODER_PROPERTIES_APPEND (props,
+      GST_VAAPI_ENCODER_PROP_TRELLIS,
+      g_param_spec_boolean ("trellis",
+          "Trellis Quantization",
+          "The Trellis Quantization Method of Encoder",
+          FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
   return props;
 }
 
@@ -273,6 +288,35 @@ gst_vaapi_encoder_ensure_param_control_rate (GstVaapiEncoder * encoder,
   return TRUE;
 }
 
+gboolean
+gst_vaapi_encoder_ensure_param_trellis (GstVaapiEncoder * encoder,
+    GstVaapiEncPicture * picture)
+{
+#if VA_CHECK_VERSION(1,0,0)
+  GstVaapiEncMiscParam *misc;
+  VAEncMiscParameterQuantization *param;
+
+  if (!encoder->trellis)
+    return TRUE;
+
+  misc = GST_VAAPI_ENC_QUANTIZATION_MISC_PARAM_NEW (encoder);
+  if (!misc)
+    return FALSE;
+  if (!misc->data)
+    return FALSE;
+
+  param = (VAEncMiscParameterQuantization *) misc->data;
+  param->quantization_flags.bits.disable_trellis = 0;
+  param->quantization_flags.bits.enable_trellis_I = 1;
+  param->quantization_flags.bits.enable_trellis_B = 1;
+  param->quantization_flags.bits.enable_trellis_P = 1;
+
+  gst_vaapi_enc_picture_add_misc_param (picture, misc);
+  gst_vaapi_codec_object_replace (&misc, NULL);
+#endif
+  return TRUE;
+}
+
 gboolean
 gst_vaapi_encoder_ensure_param_roi_regions (GstVaapiEncoder * encoder,
     GstVaapiEncPicture * picture)
@@ -1028,6 +1072,24 @@ gst_vaapi_encoder_reconfigure_internal (GstVaapiEncoder * encoder)
       GST_VAAPI_ENCODER_QUALITY_LEVEL (encoder));
 #endif
 
+  if (encoder->trellis) {
+#if VA_CHECK_VERSION(1,0,0)
+    guint quantization_method = 0;
+    if (get_config_attribute (encoder, VAConfigAttribEncQuantization,
+            &quantization_method) == FALSE
+        || !(quantization_method & VA_ENC_QUANTIZATION_TRELLIS_SUPPORTED)) {
+
+      GST_INFO ("Trellis Quantization is not supported,"
+          " trellis will be disabled");
+      encoder->trellis = FALSE;
+    }
+#else
+    GST_INFO ("The encode trellis quantization option is not supported"
+        " in this VAAPI version.");
+    encoder->trellis = FALSE;
+#endif
+  }
+
   codedbuf_size = encoder->codedbuf_pool ?
       gst_vaapi_coded_buffer_pool_get_buffer_size (GST_VAAPI_CODED_BUFFER_POOL
       (encoder)) : 0;
@@ -1152,6 +1214,10 @@ set_property (GstVaapiEncoder * encoder, gint prop_id, const GValue * value)
       encoder->default_roi_value = g_value_get_int (value);
       status = GST_VAAPI_ENCODER_STATUS_SUCCESS;
       break;
+    case GST_VAAPI_ENCODER_PROP_TRELLIS:
+      status =
+          gst_vaapi_encoder_set_trellis (encoder, g_value_get_boolean (value));
+      break;
   }
   return status;
 
@@ -1409,6 +1475,40 @@ error_operation_failed:
   }
 }
 
+/**
+ * gst_vaapi_encoder_set_trellis:
+ * @encoder: a #GstVaapiEncoder
+ * @trellis: whether to use trellis quantization
+ *
+ * Notifies the @encoder to use the supplied @trellis option.
+ *
+ * Note: currently, the tuning option can only be specified before the
+ * last call to gst_vaapi_encoder_set_codec_state(), which shall occur
+ * before the first frame is encoded. Afterwards, any change to this
+ * parameter causes gst_vaapi_encoder_set_tuning() to return
+ * @GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED.
+ *
+ * Return value: a #GstVaapiEncoderStatus
+ */
+GstVaapiEncoderStatus
+gst_vaapi_encoder_set_trellis (GstVaapiEncoder * encoder, gboolean trellis)
+{
+  g_return_val_if_fail (encoder != NULL, 0);
+
+  if (encoder->trellis != trellis && encoder->num_codedbuf_queued > 0)
+    goto error_operation_failed;
+
+  encoder->trellis = trellis;
+  return GST_VAAPI_ENCODER_STATUS_SUCCESS;
+
+  /* ERRORS */
+error_operation_failed:
+  {
+    GST_ERROR ("could not change trellis options after encoding started");
+    return GST_VAAPI_ENCODER_STATUS_ERROR_OPERATION_FAILED;
+  }
+}
+
 /* Initialize default values for configurable properties */
 static gboolean
 gst_vaapi_encoder_init_properties (GstVaapiEncoder * encoder)

gst-libs/gst/vaapi/gstvaapiencoder.h

@@ -117,6 +117,7 @@ typedef enum {
  * @GST_VAAPI_ENCODER_PROP_KEYFRAME_PERIOD: The maximal distance
  *   between two keyframes (uint).
  * @GST_VAAPI_ENCODER_PROP_TUNE: The tuning options (#GstVaapiEncoderTune).
+ * @GST_VAAPI_ENCODER_PROP_TRELLIS: Use trellis quantization method (gboolean).
  *
  * The set of configurable properties for the encoder.
  */
@@ -126,7 +127,8 @@ typedef enum {
   GST_VAAPI_ENCODER_PROP_KEYFRAME_PERIOD,
   GST_VAAPI_ENCODER_PROP_TUNE,
   GST_VAAPI_ENCODER_PROP_QUALITY_LEVEL,
-  GST_VAAPI_ENCODER_PROP_DEFAULT_ROI_VALUE
+  GST_VAAPI_ENCODER_PROP_DEFAULT_ROI_VALUE,
+  GST_VAAPI_ENCODER_PROP_TRELLIS
 } GstVaapiEncoderProp;
 
 /**
@@ -192,6 +194,9 @@ GstVaapiEncoderStatus
 gst_vaapi_encoder_set_quality_level (GstVaapiEncoder * encoder,
     guint quality_level);
 
+GstVaapiEncoderStatus
+gst_vaapi_encoder_set_trellis (GstVaapiEncoder * encoder, gboolean trellis);
+
 GstVaapiEncoderStatus
 gst_vaapi_encoder_get_buffer_with_timeout (GstVaapiEncoder * encoder,
     GstVaapiCodedBufferProxy ** out_codedbuf_proxy_ptr, guint64 timeout);

gst-libs/gst/vaapi/gstvaapiencoder_h264.c

@@ -2566,6 +2566,9 @@ ensure_misc_params (GstVaapiEncoderH264 * encoder, GstVaapiEncPicture * picture)
     }
   }
 
+  if (!gst_vaapi_encoder_ensure_param_trellis (base_encoder, picture))
+    return FALSE;
+
   if (!gst_vaapi_encoder_ensure_param_roi_regions (base_encoder, picture))
     return FALSE;
 

gst-libs/gst/vaapi/gstvaapiencoder_objects.h

@@ -342,6 +342,12 @@ gst_vaapi_enc_picture_encode (GstVaapiEncPicture * picture);
       VAEncMiscParameterTypeQualityLevel,                               \
       sizeof (VAEncMiscParameterBufferQualityLevel))
 
+/* GstVaapiEncQuantizationMiscParam */
+#define GST_VAAPI_ENC_QUANTIZATION_MISC_PARAM_NEW(encoder)              \
+  gst_vaapi_enc_misc_param_new (GST_VAAPI_ENCODER_CAST (encoder),       \
+      VAEncMiscParameterTypeQuantization,                               \
+      sizeof (VAEncMiscParameterQuantization))
+
 /* GstVaapiEncPicture  */
 #define GST_VAAPI_ENC_PICTURE_NEW(codec, encoder, frame)                \
   gst_vaapi_enc_picture_new (GST_VAAPI_ENCODER_CAST (encoder),          \

gst-libs/gst/vaapi/gstvaapiencoder_priv.h

@@ -289,6 +289,9 @@ struct _GstVaapiEncoder
   VAEncMiscParameterHRD va_hrd;
 
   gint8 default_roi_value;
+
+  /* trellis quantization */
+  gboolean trellis;
 };
 
 struct _GstVaapiEncoderClassData
@@ -426,6 +429,11 @@ gboolean
 gst_vaapi_encoder_ensure_param_roi_regions (GstVaapiEncoder * encoder,
     GstVaapiEncPicture * picture);
 
+G_GNUC_INTERNAL
+gboolean
+gst_vaapi_encoder_ensure_param_trellis (GstVaapiEncoder * encoder,
+    GstVaapiEncPicture * picture);
+
 G_GNUC_INTERNAL
 gboolean
 gst_vaapi_encoder_ensure_num_slices (GstVaapiEncoder * encoder,