diff --git a/subprojects/gst-plugins-bad/sys/va/gstvaav1enc.c b/subprojects/gst-plugins-bad/sys/va/gstvaav1enc.c
index b3fd01f6ac..683eacad4a 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvaav1enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvaav1enc.c
@@ -2864,8 +2864,10 @@ gst_va_av1_enc_reconfig (GstVaBaseEnc * base)
       GST_VIDEO_INFO_FPS_N (&base->input_state->info));
   gst_video_encoder_set_latency (venc, latency, latency);
 
-  max_ref_frames = GST_AV1_NUM_REF_FRAMES + 3 /* scratch frames */ ;
+  max_ref_frames = GST_AV1_NUM_REF_FRAMES;
   max_ref_frames += base->preferred_output_delay;
+  base->min_buffers = max_ref_frames;
+  max_ref_frames += 3 /* scratch frames */ ;
 
   /* second check after calculations */
   do_reopen |=
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
index 7301295fec..d8e2667ca1 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.c
@@ -76,6 +76,7 @@ gst_va_base_enc_reset_state_default (GstVaBaseEnc * base)
   base->rt_format = 0;
   base->codedbuf_size = 0;
   base->preferred_output_delay = 0;
+  base->min_buffers = 1;
   g_atomic_int_set (&base->reconf, FALSE);
 }
 
@@ -417,7 +418,7 @@ gst_va_base_enc_propose_allocation (GstVideoEncoder * venc, GstQuery * query)
   if (!(allocator = _allocator_from_caps (base, caps)))
     return FALSE;
 
-  pool = gst_va_pool_new_with_config (caps, 1, 0, usage_hint,
+  pool = gst_va_pool_new_with_config (caps, base->min_buffers, 0, usage_hint,
       GST_VA_FEATURE_AUTO, allocator, &params);
   if (!pool) {
     gst_object_unref (allocator);
@@ -428,7 +429,7 @@ gst_va_base_enc_propose_allocation (GstVideoEncoder * venc, GstQuery * query)
     goto config_failed;
 
   gst_query_add_allocation_param (query, allocator, &params);
-  gst_query_add_allocation_pool (query, pool, size, 1, 0);
+  gst_query_add_allocation_pool (query, pool, size, base->min_buffers, 0);
 
   GST_DEBUG_OBJECT (base,
       "proposing %" GST_PTR_FORMAT " with allocator %" GST_PTR_FORMAT,
@@ -958,6 +959,7 @@ gst_va_base_enc_init (GstVaBaseEnc * self)
   g_queue_init (&self->ref_list);
   g_queue_init (&self->output_list);
   gst_video_info_init (&self->in_info);
+  self->min_buffers = 1;
 
   self->dts_queue = gst_vec_deque_new_for_struct (sizeof (GstClockTime), 8);
 
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
index c16c561661..f4c7182e56 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
+++ b/subprojects/gst-plugins-bad/sys/va/gstvabaseenc.h
@@ -61,6 +61,8 @@ struct _GstVaBaseEnc
   gint height;
   guint rt_format;
   guint codedbuf_size;
+  /* The min buffer number required for reorder and output delay. */
+  guint min_buffers;
 
   GstClockTime start_pts;
   GstClockTime frame_duration;
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
index fca867263f..d4acaaf074 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
@@ -1658,8 +1658,10 @@ gst_va_h264_enc_reconfig (GstVaBaseEnc * base)
       GST_VIDEO_INFO_FPS_N (&base->input_state->info));
   gst_video_encoder_set_latency (venc, latency, latency);
 
-  max_ref_frames = self->gop.num_ref_frames + 3 /* scratch frames */ ;
+  max_ref_frames = self->gop.num_ref_frames;
   max_ref_frames += base->preferred_output_delay;
+  base->min_buffers = max_ref_frames;
+  max_ref_frames += 3 /* scratch frames */ ;
 
   /* second check after calculations */
   do_reopen |=
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah265enc.c b/subprojects/gst-plugins-bad/sys/va/gstvah265enc.c
index 18a00a5039..b1d75b2759 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvah265enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah265enc.c
@@ -4649,8 +4649,9 @@ gst_va_h265_enc_reconfig (GstVaBaseEnc * base)
 
   max_ref_frames = self->gop.b_pyramid ?
       self->gop.highest_pyramid_level + 2 : self->gop.num_ref_frames;
-  max_ref_frames += 3 /* scratch frames */ ;
   max_ref_frames += base->preferred_output_delay;
+  base->min_buffers = max_ref_frames;
+  max_ref_frames += 3 /* scratch frames */ ;
 
   /* second check after calculations */
   do_reopen |=
diff --git a/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c b/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
index 0a397a7717..317b66638b 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
@@ -2189,8 +2189,10 @@ gst_va_vp9_enc_reconfig (GstVaBaseEnc * base)
       GST_VIDEO_INFO_FPS_N (&base->input_state->info));
   gst_video_encoder_set_latency (venc, latency, latency);
 
-  max_ref_frames = GST_VP9_REF_FRAMES + 3 /* scratch frames */ ;
+  max_ref_frames = GST_VP9_REF_FRAMES;
   max_ref_frames += base->preferred_output_delay;
+  base->min_buffers = max_ref_frames;
+  max_ref_frames += 3 /* scratch frames */ ;
 
   /* second check after calculations */
   do_reopen |=