diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp
index a5d4fa468c..f42fb80cd9 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.cpp
@@ -578,6 +578,11 @@ gst_nv_encoder_propose_allocation (GstVideoEncoder * encoder, GstQuery * query)
 
   features = gst_caps_get_features (caps, 0);
   min_buffers = gst_nv_encoder_get_task_size (self);
+  if (min_buffers == 0) {
+    GstNvEncoderClass *klass = GST_NV_ENCODER_GET_CLASS (self);
+
+    min_buffers = klass->calculate_min_buffers (self);
+  }
 
   switch (priv->subclass_device_mode) {
     case GST_NV_ENCODER_DEVICE_AUTO_SELECT:
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h
index 7a146bc5f0..126a608d7b 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvencoder.h
@@ -222,6 +222,8 @@ struct _GstNvEncoderClass
                                        const GstVideoInfo * info,
                                        GstBuffer * buffer,
                                        GstNvEncoderDeviceData * data);
+
+  guint       (*calculate_min_buffers) (GstNvEncoder * encoder);
 };
 
 GType gst_nv_encoder_get_type (void);
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp
index 92371ffa2b..2a03e613e4 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh264encoder.cpp
@@ -227,6 +227,7 @@ gst_nv_h264_encoder_check_reconfigure (GstNvEncoder * encoder,
 static gboolean gst_nv_h264_encoder_select_device (GstNvEncoder * encoder,
     const GstVideoInfo * info, GstBuffer * buffer,
     GstNvEncoderDeviceData * data);
+static guint gst_nv_h264_encoder_calculate_min_buffers (GstNvEncoder * encoder);
 
 static void
 gst_nv_h264_encoder_class_init (GstNvH264EncoderClass * klass, gpointer data)
@@ -467,6 +468,8 @@ gst_nv_h264_encoder_class_init (GstNvH264EncoderClass * klass, gpointer data)
       GST_DEBUG_FUNCPTR (gst_nv_h264_encoder_check_reconfigure);
   nvenc_class->select_device =
       GST_DEBUG_FUNCPTR (gst_nv_h264_encoder_select_device);
+  nvenc_class->calculate_min_buffers =
+      GST_DEBUG_FUNCPTR (gst_nv_h264_encoder_calculate_min_buffers);
 
   klass->device_caps = cdata->device_caps;
   klass->cuda_device_id = cdata->cuda_device_id;
@@ -1738,6 +1741,24 @@ gst_nv_h264_encoder_select_device (GstNvEncoder * encoder,
   return TRUE;
 }
 
+static guint
+gst_nv_h264_encoder_calculate_min_buffers (GstNvEncoder * encoder)
+{
+  GstNvH264Encoder *self = GST_NV_H264_ENCODER (encoder);
+  guint num_buffers;
+
+  /* At least 4 surfaces are required as documented by Nvidia Encoder guide */
+  num_buffers = 4;
+
+  /* lookahead depth */
+  num_buffers += self->rc_lookahead;
+
+  /* B frames + 1 */
+  num_buffers += self->bframes + 1;
+
+  return num_buffers;
+}
+
 static GstNvEncoderClassData *
 gst_nv_h264_encoder_create_class_data (GstObject * device, gpointer session,
     GstNvEncoderDeviceMode device_mode)
diff --git a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp
index 9be487917e..076c407aa8 100644
--- a/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp
+++ b/subprojects/gst-plugins-bad/sys/nvcodec/gstnvh265encoder.cpp
@@ -232,6 +232,7 @@ gst_nv_h265_encoder_check_reconfigure (GstNvEncoder * encoder,
 static gboolean gst_nv_h265_encoder_select_device (GstNvEncoder * encoder,
     const GstVideoInfo * info, GstBuffer * buffer,
     GstNvEncoderDeviceData * data);
+static guint gst_nv_h265_encoder_calculate_min_buffers (GstNvEncoder * encoder);
 
 static void
 gst_nv_h265_encoder_class_init (GstNvH265EncoderClass * klass, gpointer data)
@@ -468,6 +469,8 @@ gst_nv_h265_encoder_class_init (GstNvH265EncoderClass * klass, gpointer data)
       GST_DEBUG_FUNCPTR (gst_nv_h265_encoder_check_reconfigure);
   nvenc_class->select_device =
       GST_DEBUG_FUNCPTR (gst_nv_h265_encoder_select_device);
+  nvenc_class->calculate_min_buffers =
+      GST_DEBUG_FUNCPTR (gst_nv_h265_encoder_calculate_min_buffers);
 
   klass->device_caps = cdata->device_caps;
   klass->cuda_device_id = cdata->cuda_device_id;
@@ -1750,6 +1753,24 @@ gst_nv_h265_encoder_select_device (GstNvEncoder * encoder,
   return TRUE;
 }
 
+static guint
+gst_nv_h265_encoder_calculate_min_buffers (GstNvEncoder * encoder)
+{
+  GstNvH265Encoder *self = GST_NV_H265_ENCODER (encoder);
+  guint num_buffers;
+
+  /* At least 4 surfaces are required as documented by Nvidia Encoder guide */
+  num_buffers = 4;
+
+  /* lookahead depth */
+  num_buffers += self->rc_lookahead;
+
+  /* B frames + 1 */
+  num_buffers += self->bframes + 1;
+
+  return num_buffers;
+}
+
 static GstNvEncoderClassData *
 gst_nv_h265_encoder_create_class_data (GstObject * device, gpointer session,
     GstNvEncoderDeviceMode device_mode)