diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
index c0b61c8c26..082cdad3c3 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
@@ -1123,6 +1123,9 @@ _generate_gop_structure (GstVaH264Enc * self)
       self->gop.ref_num_list0--;
       self->gop.ref_num_list1++;
     }
+
+    if (self->gop.ref_num_list0 > list0)
+      self->gop.ref_num_list0 = list0;
   }
 
   /* It's OK, keep slots for GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME frame. */