diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index fd57a3f0b4..8c3b9b09bd 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -1623,7 +1623,7 @@ ensure_context (GstVaapiDecoderH264 * decoder, GstH264SPS * sps)
   info.chroma_type = priv->chroma_type;
   info.width = sps->width;
   info.height = sps->height;
-  info.ref_frames = 16;
+  info.ref_frames = dpb_size;
 
   if (!gst_vaapi_decoder_ensure_context (GST_VAAPI_DECODER (decoder), &info))
     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
index dc81c6accc..6ec0f5ca0a 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h265.c
@@ -1202,7 +1202,7 @@ ensure_context (GstVaapiDecoderH265 * decoder, GstH265SPS * sps)
   info.chroma_type = priv->chroma_type;
   info.width = sps->width;
   info.height = sps->height;
-  info.ref_frames = 16;
+  info.ref_frames = dpb_size;
 
   if (!gst_vaapi_decoder_ensure_context (GST_VAAPI_DECODER (decoder), &info))
     return GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;