diff --git a/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c b/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
index 2a0bdd3f52..0a397a7717 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvavp9enc.c
@@ -2096,13 +2096,15 @@ gst_va_vp9_enc_reconfig (GstVaBaseEnc * base)
   VAProfile profile;
   gboolean do_renegotiation = TRUE, do_reopen, need_negotiation;
   guint max_ref_frames, max_surfaces = 0,
-      rt_format, depth = 0, chrome = 0, codedbuf_size;
+      rt_format, depth = 0, chrome = 0, codedbuf_size, latency_num;
   gint width, height;
+  GstClockTime latency;
 
   width = GST_VIDEO_INFO_WIDTH (&base->in_info);
   height = GST_VIDEO_INFO_HEIGHT (&base->in_info);
   format = GST_VIDEO_INFO_FORMAT (&base->in_info);
   codedbuf_size = base->codedbuf_size;
+  latency_num = base->preferred_output_delay + self->gop.gf_group_size - 1;
 
   need_negotiation =
       !gst_va_encoder_get_reconstruct_pool_config (base->encoder, &reconf_caps,
@@ -2135,6 +2137,13 @@ gst_va_vp9_enc_reconfig (GstVaBaseEnc * base)
 
   gst_va_base_enc_reset_state (base);
 
+  if (base->is_live) {
+    base->preferred_output_delay = 0;
+  } else {
+    /* FIXME: An experience value for most of the platforms. */
+    base->preferred_output_delay = 4;
+  }
+
   base->profile = profile;
   base->rt_format = rt_format;
   self->depth = depth;
@@ -2168,7 +2177,20 @@ gst_va_vp9_enc_reconfig (GstVaBaseEnc * base)
   if (!_vp9_init_packed_headers (self))
     return FALSE;
 
+  /* Let the downstream know the new latency. */
+  if (latency_num != base->preferred_output_delay + self->gop.gf_group_size - 1) {
+    need_negotiation = TRUE;
+    latency_num = base->preferred_output_delay + self->gop.gf_group_size - 1;
+  }
+
+  /* Set the latency */
+  latency = gst_util_uint64_scale (latency_num,
+      GST_VIDEO_INFO_FPS_D (&base->input_state->info) * GST_SECOND,
+      GST_VIDEO_INFO_FPS_N (&base->input_state->info));
+  gst_video_encoder_set_latency (venc, latency, latency);
+
   max_ref_frames = GST_VP9_REF_FRAMES + 3 /* scratch frames */ ;
+  max_ref_frames += base->preferred_output_delay;
 
   /* second check after calculations */
   do_reopen |=