nvenc: Add properties to support bframe encoding if device supports it

Note that bframe encoding capability varies with GPU architecture

sys/nvcodec/gstnvbaseenc.c

@@ -834,6 +834,32 @@ _find_frame_with_output_buffer (GstNvBaseEnc * nvenc, NV_ENC_OUTPUT_PTR out_buf)
   return ret;
 }
 
+static const gchar *
+picture_type_to_string (NV_ENC_PIC_TYPE type)
+{
+  switch (type) {
+    case NV_ENC_PIC_TYPE_P:
+      return "P";
+    case NV_ENC_PIC_TYPE_B:
+      return "B";
+    case NV_ENC_PIC_TYPE_I:
+      return "I";
+    case NV_ENC_PIC_TYPE_IDR:
+      return "IDR";
+    case NV_ENC_PIC_TYPE_BI:
+      return "BI";
+    case NV_ENC_PIC_TYPE_SKIPPED:
+      return "SKIPPED";
+    case NV_ENC_PIC_TYPE_INTRA_REFRESH:
+      return "INTRA-REFRESH";
+    case NV_ENC_PIC_TYPE_UNKNOWN:
+    default:
+      break;
+  }
+
+  return "UNKNOWN";
+}
+
 static gpointer
 gst_nv_base_enc_bitstream_thread (gpointer user_data)
 {
@@ -910,9 +936,6 @@ gst_nv_base_enc_bitstream_thread (gpointer user_data)
       GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame);
     }
 
-    /* TODO: use lock_bs.outputTimeStamp and lock_bs.outputDuration */
-    /* TODO: check pts/dts is handled properly if there are B-frames */
-
     nv_ret = NvEncUnlockBitstream (nvenc->encoder, state->out_buf);
 
     if (nv_ret != NV_ENC_SUCCESS) {
@@ -927,6 +950,16 @@ gst_nv_base_enc_bitstream_thread (gpointer user_data)
       goto error_shutdown;
     }
 
+    frame->dts = frame->pts;
+    frame->pts = lock_bs.outputTimeStamp;
+    frame->duration = lock_bs.outputDuration;
+
+    GST_LOG_OBJECT (nvenc, "frame index %" G_GUINT32_FORMAT
+        ", frame type %s, dts %" GST_TIME_FORMAT
+        ", pts %" GST_TIME_FORMAT,
+        lock_bs.frameIdx, picture_type_to_string (lock_bs.pictureType),
+        GST_TIME_ARGS (frame->dts), GST_TIME_ARGS (frame->pts));
+
     frame->output_buffer = buffer;
 
     nv_ret =
@@ -1218,6 +1251,7 @@ gst_nv_base_enc_setup_rate_control (GstNvBaseEnc * nvenc,
     rc_params->enableLookahead = 1;
     rc_params->lookaheadDepth = nvenc->rc_lookahead;
     rc_params->disableIadapt = !nvenc->i_adapt;
+    rc_params->disableBadapt = !nvenc->b_adapt;
   }
 
   rc_params->strictGOPTarget = nvenc->strict_gop;
@@ -1413,6 +1447,16 @@ gst_nv_base_enc_set_format (GstVideoEncoder * enc, GstVideoCodecState * state)
     params->encodeConfig->frameIntervalP = 1;
   } else if (nvenc->gop_size > 0) {
     params->encodeConfig->gopLength = nvenc->gop_size;
+    /* frameIntervalP
+     * 0: All Intra frames
+     * 1: I/P only
+     * n ( > 1): n - 1 bframes
+     */
+    params->encodeConfig->frameIntervalP = nvenc->bframes + 1;
+  } else {
+    /* gop size == 0 means all intra frames */
+    params->encodeConfig->gopLength = 1;
+    params->encodeConfig->frameIntervalP = 0;
   }
 
   g_assert (nvenc_class->set_encoder_config);

sys/nvcodec/gstnvbaseenc.h

@@ -67,6 +67,7 @@ typedef struct {
   gboolean custom_vbv_bufsize;
   gboolean lookahead;
   gboolean temporal_aq;
+  gint bframes;
 } GstNvEncDeviceCaps;
 
 typedef struct {
@@ -139,6 +140,8 @@ typedef struct {
   guint           vbv_buffersize;
   guint           rc_lookahead;
   gboolean        temporal_aq;
+  guint           bframes;
+  gboolean        b_adapt;
 } GstNvBaseEnc;
 
 typedef struct {

sys/nvcodec/gstnvenc.c

@@ -507,14 +507,15 @@ gst_nvenc_get_supported_codec_profiles (gpointer enc, GUID codec_id)
   gint support_10bit = 0;
   GstNvEncCodecProfile profiles[] = {
     /* avc profiles */
-    {"baseline", NV_ENC_H264_PROFILE_BASELINE_GUID, NV_ENC_CODEC_H264_GUID,
-        FALSE, FALSE, FALSE},
     {"main", NV_ENC_H264_PROFILE_MAIN_GUID, NV_ENC_CODEC_H264_GUID, FALSE,
         FALSE, FALSE},
     {"high", NV_ENC_H264_PROFILE_HIGH_GUID, NV_ENC_CODEC_H264_GUID, FALSE,
         FALSE, FALSE},
     {"high-4:4:4", NV_ENC_H264_PROFILE_HIGH_444_GUID, NV_ENC_CODEC_H264_GUID,
         TRUE, FALSE, FALSE},
+    /* put baseline to last since it does not support bframe */
+    {"baseline", NV_ENC_H264_PROFILE_BASELINE_GUID, NV_ENC_CODEC_H264_GUID,
+        FALSE, FALSE, FALSE},
     /* hevc profiles */
     {"main", NV_ENC_HEVC_PROFILE_MAIN_GUID, NV_ENC_CODEC_HEVC_GUID, FALSE,
         FALSE, FALSE},
@@ -733,6 +734,12 @@ gst_nv_enc_register (GstPlugin * plugin, GUID codec_id, const gchar * codec,
       device_caps.temporal_aq = FALSE;
     }
 
+    caps_param.capsToQuery = NV_ENC_CAPS_NUM_MAX_BFRAMES;
+    if (NvEncGetEncodeCaps (enc, codec_id, &caps_param,
+            &device_caps.bframes) != NV_ENC_SUCCESS) {
+      device_caps.bframes = 0;
+    }
+
     DEBUG_DEVICE_CAPS (i,
         codec, "weighted prediction", device_caps.weighted_prediction);
 
@@ -744,6 +751,8 @@ gst_nv_enc_register (GstPlugin * plugin, GUID codec_id, const gchar * codec,
     DEBUG_DEVICE_CAPS (i, codec, "temporal adaptive quantization",
         device_caps.temporal_aq);
 
+    GST_DEBUG ("[device-%d %s] max bframes: %d", i, codec, device_caps.bframes);
+
     interlace_modes = gst_nvenc_get_interlace_modes (enc, codec_id);
 
     sink_templ = gst_caps_new_empty_simple ("video/x-raw");

sys/nvcodec/gstnvh264enc.c

@@ -47,6 +47,8 @@ enum
   PROP_VBV_BUFFER_SIZE,
   PROP_RC_LOOKAHEAD,
   PROP_TEMPORAL_AQ,
+  PROP_BFRAMES,
+  PROP_B_ADAPT,
 };
 
 #define DEFAULT_AUD TRUE
@@ -54,6 +56,8 @@ enum
 #define DEFAULT_VBV_BUFFER_SIZE 0
 #define DEFAULT_RC_LOOKAHEAD 0
 #define DEFAULT_TEMPORAL_AQ FALSE
+#define DEFAULT_BFRAMES 0
+#define DEFAULT_B_ADAPT FALSE
 
 static gboolean gst_nv_h264_enc_open (GstVideoEncoder * enc);
 static gboolean gst_nv_h264_enc_close (GstVideoEncoder * enc);
@@ -139,6 +143,24 @@ gst_nv_h264_enc_class_init (GstNvH264EncClass * klass, gpointer data)
             G_PARAM_STATIC_STRINGS));
   }
 
+  if (device_caps->bframes > 0) {
+    g_object_class_install_property (gobject_class, PROP_BFRAMES,
+        g_param_spec_uint ("bframes", "B-Frames",
+            "Number of B-frames between I and P "
+            "(Exposed only if supported by device)", 0, device_caps->bframes,
+            DEFAULT_BFRAMES,
+            G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
+            G_PARAM_STATIC_STRINGS));
+
+    g_object_class_install_property (gobject_class, PROP_B_ADAPT,
+        g_param_spec_boolean ("b-adapt", "B Adapt",
+            "Enable adaptive B-frame insert when lookahead is enabled "
+            "(Exposed only if supported by device)",
+            DEFAULT_B_ADAPT,
+            G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
+            G_PARAM_STATIC_STRINGS));
+  }
+
   if (cdata->is_default)
     long_name = g_strdup ("NVENC H.264 Video Encoder");
   else
@@ -180,6 +202,8 @@ gst_nv_h264_enc_init (GstNvH264Enc * nvenc)
   baseenc->vbv_buffersize = DEFAULT_VBV_BUFFER_SIZE;
   baseenc->rc_lookahead = DEFAULT_RC_LOOKAHEAD;
   baseenc->temporal_aq = DEFAULT_TEMPORAL_AQ;
+  baseenc->bframes = DEFAULT_BFRAMES;
+  baseenc->b_adapt = DEFAULT_B_ADAPT;
 }
 
 static void
@@ -505,6 +529,21 @@ gst_nv_h264_enc_set_property (GObject * object, guint prop_id,
         reconfig = TRUE;
       }
       break;
+    case PROP_BFRAMES:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        nvenc->bframes = g_value_get_uint (value);
+        reconfig = TRUE;
+      }
+      break;
+    case PROP_B_ADAPT:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        nvenc->b_adapt = g_value_get_boolean (value);
+      }
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -555,6 +594,20 @@ gst_nv_h264_enc_get_property (GObject * object, guint prop_id, GValue * value,
         g_value_set_boolean (value, nvenc->temporal_aq);
       }
       break;
+    case PROP_BFRAMES:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        g_value_set_uint (value, nvenc->bframes);
+      }
+      break;
+    case PROP_B_ADAPT:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        g_value_set_boolean (value, nvenc->b_adapt);
+      }
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;

sys/nvcodec/gstnvh265enc.c

@@ -49,6 +49,8 @@ enum
   PROP_VBV_BUFFER_SIZE,
   PROP_RC_LOOKAHEAD,
   PROP_TEMPORAL_AQ,
+  PROP_BFRAMES,
+  PROP_B_ADAPT,
 };
 
 #define DEFAULT_AUD TRUE
@@ -56,6 +58,8 @@ enum
 #define DEFAULT_VBV_BUFFER_SIZE 0
 #define DEFAULT_RC_LOOKAHEAD 0
 #define DEFAULT_TEMPORAL_AQ FALSE
+#define DEFAULT_BFRAMES 0
+#define DEFAULT_B_ADAPT FALSE
 
 static gboolean gst_nv_h265_enc_open (GstVideoEncoder * enc);
 static gboolean gst_nv_h265_enc_close (GstVideoEncoder * enc);
@@ -143,6 +147,24 @@ gst_nv_h265_enc_class_init (GstNvH265EncClass * klass, gpointer data)
             G_PARAM_STATIC_STRINGS));
   }
 
+  if (device_caps->bframes > 0) {
+    g_object_class_install_property (gobject_class, PROP_BFRAMES,
+        g_param_spec_uint ("bframes", "B-Frames",
+            "Number of B-frames between I and P "
+            "(Exposed only if supported by device)", 0, device_caps->bframes,
+            DEFAULT_BFRAMES,
+            G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
+            G_PARAM_STATIC_STRINGS));
+
+    g_object_class_install_property (gobject_class, PROP_B_ADAPT,
+        g_param_spec_boolean ("b-adapt", "B Adapt",
+            "Enable adaptive B-frame insert when lookahead is enabled "
+            "(Exposed only if supported by device)",
+            DEFAULT_B_ADAPT,
+            G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY |
+            G_PARAM_STATIC_STRINGS));
+  }
+
   if (cdata->is_default)
     long_name = g_strdup ("NVENC HEVC Video Encoder");
   else
@@ -184,6 +206,8 @@ gst_nv_h265_enc_init (GstNvH265Enc * nvenc)
   baseenc->vbv_buffersize = DEFAULT_VBV_BUFFER_SIZE;
   baseenc->rc_lookahead = DEFAULT_RC_LOOKAHEAD;
   baseenc->temporal_aq = DEFAULT_TEMPORAL_AQ;
+  baseenc->bframes = DEFAULT_BFRAMES;
+  baseenc->b_adapt = DEFAULT_B_ADAPT;
 }
 
 static void
@@ -660,6 +684,21 @@ gst_nv_h265_enc_set_property (GObject * object, guint prop_id,
         reconfig = TRUE;
       }
       break;
+    case PROP_BFRAMES:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        nvenc->bframes = g_value_get_uint (value);
+        reconfig = TRUE;
+      }
+      break;
+    case PROP_B_ADAPT:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        nvenc->b_adapt = g_value_get_boolean (value);
+      }
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -710,6 +749,20 @@ gst_nv_h265_enc_get_property (GObject * object, guint prop_id, GValue * value,
         g_value_set_boolean (value, nvenc->temporal_aq);
       }
       break;
+    case PROP_BFRAMES:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        g_value_set_uint (value, nvenc->bframes);
+      }
+      break;
+    case PROP_B_ADAPT:
+      if (!device_caps->bframes) {
+        G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      } else {
+        g_value_set_boolean (value, nvenc->b_adapt);
+      }
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;