va: vp9enc: enable ICQ and QVBR modes

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6433>

subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c

@@ -1818,7 +1818,7 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
    * speed and quality, while the others control encoding bit rate and
    * quality. The lower value has better quality(maybe bigger MV search
    * range) but slower speed, the higher value has faster speed but lower
-   * quality.
+   * quality. It is valid for all modes.
    *
    * The possible composition to control the bit rate and quality:
    *
@@ -1855,6 +1855,18 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
    *    target bit rate, and encoder will try its best to make the QP
    *    with in the ["max-qp", "min-qp"] range. Other paramters are
    *    ignored.
+   *
+   * 5. ICQ mode: "rate-control=ICQ", which is similar to CQP mode
+   *    except that its QP(qindex in VP9) may be increased or decreaed
+   *    to avoid huge bit rate fluctuation. The "qp" specifies a quality
+   *    factor as the base quality value. Other properties are ignored.
+   *
+   * 6. QVBR mode: "rate-control=QVBR", which is similar to VBR mode
+   *    with the same usage of "bitrate", "target-percentage" and
+   *    "cpb-size" properties. Besides that, the "qp"(the qindex in VP9)
+   *    specifies a quality factor as the base quality value which the
+   *    driver should try its best to meet. Other properties are ignored.
+   *
    */
 
   GstVaBaseEnc *base = GST_VA_BASE_ENC (self);
@@ -1894,6 +1906,17 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
     self->rc.rc_ctrl_mode = VA_RC_NONE;
   }
 
+  /* ICQ mode and QVBR mode do not need max/min qp. */
+  if (self->rc.rc_ctrl_mode == VA_RC_ICQ || self->rc.rc_ctrl_mode == VA_RC_QVBR) {
+    self->rc.min_qindex = 0;
+    self->rc.max_qindex = 255;
+
+    update_property_uint (base, &self->prop.min_qp, self->rc.min_qindex,
+        PROP_MIN_QP);
+    update_property_uint (base, &self->prop.max_qp, self->rc.max_qindex,
+        PROP_MAX_QP);
+  }
+
   if (self->rc.min_qindex > self->rc.max_qindex) {
     GST_INFO_OBJECT (self, "The min_qindex %d is bigger than the max_qindex"
         " %d, set it to the max_qindex", self->rc.min_qindex,
@@ -1933,7 +1956,8 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
 
   /* Calculate a bitrate if it is not set. */
   if ((self->rc.rc_ctrl_mode == VA_RC_CBR || self->rc.rc_ctrl_mode == VA_RC_VBR
-          || self->rc.rc_ctrl_mode == VA_RC_VCM) && bitrate == 0) {
+          || self->rc.rc_ctrl_mode == VA_RC_VCM
+          || self->rc.rc_ctrl_mode == VA_RC_QVBR) && bitrate == 0) {
     /* FIXME: Provide better estimation. */
     /* Choose the max value of all levels' MainCR which is 8, and x2 for
        conservative calculation. So just using a 1/16 compression ratio,
@@ -1969,6 +1993,8 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
 
   /* Adjust the setting based on RC mode. */
   switch (self->rc.rc_ctrl_mode) {
+    case VA_RC_NONE:
+    case VA_RC_ICQ:
     case VA_RC_CQP:
       self->rc.max_bitrate = 0;
       self->rc.target_bitrate = 0;
@@ -1985,11 +2011,13 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
       self->rc.sharpness_level = 0;
       break;
     case VA_RC_VBR:
+      self->rc.base_qindex = DEFAULT_BASE_QINDEX;
+      /* Fall through. */
+    case VA_RC_QVBR:
       g_assert (self->rc.target_percentage >= 10);
       self->rc.max_bitrate = (guint) gst_util_uint64_scale_int (bitrate,
           100, self->rc.target_percentage);
       self->rc.target_bitrate = bitrate;
-      self->rc.base_qindex = DEFAULT_BASE_QINDEX;
       self->rc.filter_level = DEFAULT_LOOP_FILTER_LEVEL;
       self->rc.sharpness_level = 0;
       break;
@@ -2017,7 +2045,9 @@ _vp9_ensure_rate_control (GstVaVp9Enc * self)
       "Target bitrate: %u bits/sec", self->rc.max_bitrate,
       self->rc.target_bitrate);
 
-  if (self->rc.rc_ctrl_mode != VA_RC_NONE && self->rc.rc_ctrl_mode != VA_RC_CQP)
+  if (self->rc.rc_ctrl_mode == VA_RC_CBR || self->rc.rc_ctrl_mode == VA_RC_VBR
+      || self->rc.rc_ctrl_mode == VA_RC_VCM
+      || self->rc.rc_ctrl_mode == VA_RC_QVBR)
     _vp9_calculate_bitrate_hrd (self);
 
   /* notifications */
@@ -3001,7 +3031,9 @@ gst_va_vp9_enc_class_init (gpointer g_klass, gpointer class_data)
    * The basic quantizer value for all frames.
    */
   properties[PROP_QP] = g_param_spec_uint ("qp", "The frame QP",
-      "The basic quantizer value for all frames.", 0, 255, DEFAULT_BASE_QINDEX,
+      "In CQP mode, it specifies the basic quantizer value for all frames. "
+      "In ICQ and QVBR modes, it specifies a quality factor. In other "
+      "modes, it is ignored", 0, 255, DEFAULT_BASE_QINDEX,
       G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT);
 
   /**