diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
index 6eae117d62..0b10f9a07f 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.c
@@ -377,7 +377,6 @@ gst_vulkan_encoder_new_video_session_parameters (GstVulkanEncoder * self,
  * @in_buffer: (transfer none): the input #GstBuffer.
  * @width: the picture width
  * @height: the picture height
- * @nb_refs: the picture number of references
  *
  * Create a new vulkan encode picture from the input buffer.
  *
@@ -386,7 +385,7 @@ gst_vulkan_encoder_new_video_session_parameters (GstVulkanEncoder * self,
  */
 GstVulkanEncoderPicture *
 gst_vulkan_encoder_picture_new (GstVulkanEncoder * self, GstBuffer * in_buffer,
-    int width, int height, gsize size, gint nb_refs)
+    int width, int height, gsize size)
 {
   GstVulkanEncoderPicture *pic;
   GstVulkanEncoderPrivate *priv;
@@ -422,7 +421,6 @@ gst_vulkan_encoder_picture_new (GstVulkanEncoder * self, GstBuffer * in_buffer,
       VK_BUFFER_USAGE_VIDEO_ENCODE_DST_BIT_KHR, size_aligned);
   pic->width = width;
   pic->height = height;
-  pic->nb_refs = nb_refs;
   pic->packed_headers =
       g_ptr_array_new_with_free_func ((GDestroyNotify) gst_buffer_unref);
   pic->slotIndex = -1;
@@ -1063,6 +1061,7 @@ bail:
  * gst_vulkan_encoder_encode:
  * @self: a #GstVulkanEncoder
  * @pic: a #GstVulkanEncoderPicture
+ * @nb_refs: number of @ref_pics
  * @ref_pics: an array of #GstVulkanEncoderPicture
  *
  * Encode a picture according to its reference pictures.
@@ -1072,7 +1071,8 @@ bail:
  */
 gboolean
 gst_vulkan_encoder_encode (GstVulkanEncoder * self,
-    GstVulkanEncoderPicture * pic, GstVulkanEncoderPicture ** ref_pics)
+    GstVulkanEncoderPicture * pic, guint nb_refs,
+    GstVulkanEncoderPicture ** ref_pics)
 {
   GstVulkanEncoderPrivate *priv;
   GError *err = NULL;
@@ -1179,7 +1179,7 @@ gst_vulkan_encoder_encode (GstVulkanEncoder * self,
   };
   /* *INDENT-ON* */
 
-  for (i = 0; i < pic->nb_refs; i++) {
+  for (i = 0; i < nb_refs; i++) {
     /* *INDENT-OFF* */
     ref_slots[i] = (VkVideoReferenceSlotInfoKHR) {
       .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
@@ -1295,8 +1295,8 @@ gst_vulkan_encoder_encode (GstVulkanEncoder * self,
         .imageViewBinding = pic->img_view->view,
     },
     .pSetupReferenceSlot = &ref_slots[ref_slot_num - 1],
-    .referenceSlotCount = pic->nb_refs,
-    .pReferenceSlots = pic->nb_refs ? ref_slots  : NULL,
+    .referenceSlotCount = nb_refs,
+    .pReferenceSlots = nb_refs ? ref_slots  : NULL,
     .precedingExternallyEncodedBytes = 0,
   };
   /* *INDENT-ON* */
diff --git a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.h b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.h
index f8826fbc26..63d38a333a 100644
--- a/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.h
+++ b/subprojects/gst-plugins-bad/gst-libs/gst/vulkan/gstvkencoder-private.h
@@ -40,7 +40,6 @@ typedef struct _GstVulkanEncoderPicture GstVulkanEncoderPicture;
 
 /**
  * GstVulkanEncoderPicture:
- * @nb_refs: number of references
  * @slotIndex: slot index
  * @packed_headers: packed headers
  * @width: picture width
@@ -56,7 +55,6 @@ typedef struct _GstVulkanEncoderPicture GstVulkanEncoderPicture;
  */
 struct _GstVulkanEncoderPicture
 {
-  gint nb_refs;
   gint slotIndex;
 
   /* picture parameters */
@@ -168,6 +166,7 @@ gboolean                gst_vulkan_encoder_create_dpb_pool      (GstVulkanEncode
 GST_VULKAN_API
 gboolean                gst_vulkan_encoder_encode               (GstVulkanEncoder * self,
                                                                  GstVulkanEncoderPicture * pic,
+                                                                 guint nb_refs,
                                                                  GstVulkanEncoderPicture ** ref_pics);
 GST_VULKAN_API
 gboolean                gst_vulkan_encoder_caps                 (GstVulkanEncoder * self,
@@ -179,7 +178,6 @@ GstVulkanEncoderPicture* gst_vulkan_encoder_picture_new         (GstVulkanEncode
                                                                  GstBuffer * in_buffer,
                                                                  gint width,
                                                                  gint height,
-                                                                 gsize size,
-                                                                 gint nb_refs);
+                                                                 gsize size);
 GST_VULKAN_API
 void                     gst_vulkan_encoder_picture_free        (GstVulkanEncoderPicture * pic);
diff --git a/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh264.c b/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh264.c
index 0bcddf05c2..87c03dd307 100644
--- a/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh264.c
+++ b/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh264.c
@@ -352,7 +352,7 @@ error:
 
 static GstVulkanH264EncodeFrame *
 allocate_frame (GstVulkanEncoder * enc, int width,
-    int height, gboolean is_ref, gint nb_refs)
+    int height, gboolean is_ref)
 {
   GstVulkanH264EncodeFrame *frame;
   GstBuffer *in_buffer, *img_buffer;
@@ -362,7 +362,7 @@ allocate_frame (GstVulkanEncoder * enc, int width,
   upload_buffer_to_image(img_pool, in_buffer, &img_buffer);
 
   frame = _h264_encode_frame_new (gst_vulkan_encoder_picture_new (enc,
-      img_buffer, width, height, width * height * 3, nb_refs), is_ref);
+      img_buffer, width, height, width * height * 3), is_ref);
   fail_unless (frame);
   fail_unless (frame->picture);
   gst_buffer_unref (in_buffer);
@@ -426,7 +426,9 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
     /* *INDENT-ON* */
   };
 
-  if (picture->nb_refs) {
+  ref_pics_num = list0_num + list1_num;
+
+  if (ref_pics_num > 0) {
     /* *INDENT-OFF* */
     frame->ref_list_info = (StdVideoEncodeH264ReferenceListsInfo) {
       .flags = {
@@ -445,8 +447,8 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
       .pRefList1ModOperations = NULL,
       .pRefPicMarkingOperations = NULL,
     };
-    frame->pic_info.pRefLists = &frame->ref_list_info;
     /* *INDENT-ON* */
+    frame->pic_info.pRefLists = &frame->ref_list_info;
   }
 
   memset (frame->ref_list_info.RefPicList0, STD_VIDEO_H264_NO_REFERENCE_PICTURE,
@@ -544,17 +546,14 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH264EncodeFrame * frame,
   for (i = 0; i < list0_num; i++) {
     ref_pics[i] = list0[i]->picture;
     frame->ref_list_info.RefPicList0[0] = list0[i]->picture->slotIndex;
-    ref_pics_num++;
   }
   for (i = 0; i < list1_num; i++) {
     ref_pics[i + list0_num] = list1[i]->picture;
     frame->ref_list_info.RefPicList1[i] = list1[i]->picture->slotIndex;
-    ref_pics_num++;
   }
 
-  picture->nb_refs = ref_pics_num;
-
-  fail_unless (gst_vulkan_encoder_encode (enc, picture, ref_pics));
+  fail_unless (gst_vulkan_encoder_encode (enc, picture, ref_pics_num,
+          ref_pics));
 }
 
 static void
@@ -798,7 +797,7 @@ GST_START_TEST (test_encoder_h264_i)
 
   /* Encode N_BUFFERS of I-Frames */
   for (i = 0; i < N_BUFFERS; i++) {
-    frame = allocate_frame (enc, width, height, TRUE, 0);
+    frame = allocate_frame (enc, width, height, TRUE);
     encode_frame (enc, frame, STD_VIDEO_H264_SLICE_TYPE_I,
         frame_num, NULL, 0, NULL, 0, sps_id, pps_id);
     check_encoded_frame (frame, GST_H264_NAL_SLICE_IDR);
@@ -840,7 +839,7 @@ GST_START_TEST (test_encoder_h264_i_p)
   img_pool = allocate_image_buffer_pool (enc, width, height);
 
   /* Encode first picture as an IDR-Frame */
-  frame = allocate_frame (enc, width, height, TRUE, 0);
+  frame = allocate_frame (enc, width, height, TRUE);
   encode_frame (enc, frame, STD_VIDEO_H264_SLICE_TYPE_I,
       frame_num, NULL, 0, NULL, 0, sps_id, pps_id);
   check_encoded_frame (frame, GST_H264_NAL_SLICE_IDR);
@@ -849,7 +848,7 @@ GST_START_TEST (test_encoder_h264_i_p)
 
   /* Encode following pictures as P-Frames */
   for (i = 1; i < N_BUFFERS; i++) {
-    frame = allocate_frame (enc, width, height, TRUE, list0_num);
+    frame = allocate_frame (enc, width, height, TRUE);
     frame->pic_num = frame_num;
     frame->pic_order_cnt = frame_num;
 
@@ -905,7 +904,7 @@ GST_START_TEST (test_encoder_h264_i_p_b)
   img_pool = allocate_image_buffer_pool (enc, width, height);
 
   /* Encode 1st picture as an IDR-Frame */
-  frame = allocate_frame (enc, width, height, TRUE, 0);
+  frame = allocate_frame (enc, width, height, TRUE);
   fail_unless (frame->picture != NULL);
   encode_frame (enc, frame, STD_VIDEO_H264_SLICE_TYPE_I,
       frame_num, NULL, 0, NULL, 0, sps_id, pps_id);
@@ -915,7 +914,7 @@ GST_START_TEST (test_encoder_h264_i_p_b)
   frame_num++;
 
   /* Encode 4th picture as a P-Frame */
-  frame = allocate_frame (enc, width, height, TRUE, list0_num);
+  frame = allocate_frame (enc, width, height, TRUE);
   frame->pic_num = 3;
   frame->pic_order_cnt = frame->pic_num * 2;
   encode_frame (enc, frame, STD_VIDEO_H264_SLICE_TYPE_P,
@@ -926,7 +925,7 @@ GST_START_TEST (test_encoder_h264_i_p_b)
   frame_num++;
 
   /* Encode second picture as a B-Frame */
-  frame = allocate_frame (enc, width, height, FALSE, list0_num + list1_num);
+  frame = allocate_frame (enc, width, height, FALSE);
   frame->pic_num = 1;
   frame->pic_order_cnt = frame->pic_num * 2;
   encode_frame (enc, frame, STD_VIDEO_H264_SLICE_TYPE_B,
@@ -936,7 +935,7 @@ GST_START_TEST (test_encoder_h264_i_p_b)
   _h264_encode_frame_free (frame);
 
   /* Encode third picture as a B-Frame */
-  frame = allocate_frame (enc, width, height, FALSE, list0_num + list1_num);
+  frame = allocate_frame (enc, width, height, FALSE);
   frame->pic_num = 2;
   frame->pic_order_cnt = frame->pic_num * 2;
 
diff --git a/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh265.c b/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh265.c
index bc30c58b34..5574199fe5 100644
--- a/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh265.c
+++ b/subprojects/gst-plugins-bad/tests/check/libs/vkvideoencodeh265.c
@@ -357,7 +357,7 @@ error:
 /* allocate a frame to be encoded from given buffer pools */
 static GstVulkanH265EncodeFrame *
 allocate_frame (GstVulkanEncoder * enc, int width,
-    int height, gboolean is_ref, gint nb_refs)
+    int height, gboolean is_ref)
 {
   GstVulkanH265EncodeFrame *frame;
   GstBuffer *in_buffer, *img_buffer;
@@ -369,7 +369,7 @@ allocate_frame (GstVulkanEncoder * enc, int width,
   upload_buffer_to_image(img_pool, in_buffer, &img_buffer);
 
   frame = _h265_encode_frame_new (gst_vulkan_encoder_picture_new (enc,
-      img_buffer, width, height, width * height * 3, nb_refs), is_ref);
+      img_buffer, width, height, width * height * 3), is_ref);
   fail_unless (frame);
   fail_unless (frame->picture);
   gst_buffer_unref (in_buffer);
@@ -403,6 +403,8 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
 
   GST_DEBUG ("Encoding frame num: %d", frame_num);
 
+  ref_pics_num = list0_num + list1_num;
+
   frame->slice_wt = (StdVideoEncodeH265WeightTable) {
     /* *INDENT-OFF* */
     .flags = (StdVideoEncodeH265WeightTableFlags) {
@@ -503,7 +505,7 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
     /* *INDENT-ON* */
   };
 
-  if (picture->nb_refs) {
+  if (ref_pics_num > 0) {
     frame->ref_list_info = (StdVideoEncodeH265ReferenceListsInfo) {
       /* *INDENT-OFF* */
       .flags = (StdVideoEncodeH265ReferenceListsInfoFlags) {
@@ -610,17 +612,14 @@ encode_frame (GstVulkanEncoder * enc, GstVulkanH265EncodeFrame * frame,
   for (i = 0; i < list0_num; i++) {
     ref_pics[i] = list0[i]->picture;
     frame->ref_list_info.RefPicList0[0] = list0[i]->picture->slotIndex;
-    ref_pics_num++;
   }
   for (i = 0; i < list1_num; i++) {
     ref_pics[i + list0_num] = list1[i]->picture;
-    ref_pics_num++;
     frame->ref_list_info.RefPicList1[i] = list1[i]->picture->slotIndex;
   }
 
-  picture->nb_refs = ref_pics_num;
-
-  fail_unless (gst_vulkan_encoder_encode (enc, picture, ref_pics));
+  fail_unless (gst_vulkan_encoder_encode (enc, picture, ref_pics_num,
+          ref_pics));
 }
 
 static void
@@ -970,7 +969,7 @@ GST_START_TEST (test_encoder_h265_i)
 
   /* Encode N_BUFFERS I-Frames */
   for (i = 0; i < N_BUFFERS; i++) {
-    frame = allocate_frame (enc, width, height, TRUE, 0);
+    frame = allocate_frame (enc, width, height, TRUE);
     encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_I,
         frame_num, NULL, 0, NULL, 0, vps_id, sps_id, pps_id);
     check_encoded_frame (frame, GST_H265_NAL_SLICE_IDR_W_RADL);
@@ -1012,7 +1011,7 @@ GST_START_TEST (test_encoder_h265_i_p)
   buffer_pool = allocate_buffer_pool (enc, width, height);
   img_pool = allocate_image_buffer_pool (enc, width, height);
 
-  frame = allocate_frame (enc, width, height, TRUE, 0);
+  frame = allocate_frame (enc, width, height, TRUE);
   /* Encode first picture as an IDR-Frame */
   encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_I,
       frame_num, NULL, 0, NULL, 0, vps_id, sps_id, pps_id);
@@ -1022,7 +1021,7 @@ GST_START_TEST (test_encoder_h265_i_p)
 
   /* Encode following pictures as a P-Frames */
   for (i = 1; i < N_BUFFERS; i++) {
-    frame = allocate_frame (enc, width, height, TRUE, list0_num);
+    frame = allocate_frame (enc, width, height, TRUE);
     frame->pic_num = frame_num;
     encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_P,
         frame_num, list0, list0_num, NULL, 0, vps_id, sps_id, pps_id);
@@ -1075,7 +1074,7 @@ GST_START_TEST (test_encoder_h265_i_p_b)
   img_pool = allocate_image_buffer_pool (enc, width, height);
 
   /* Encode first picture as an IDR-Frame */
-  frame = allocate_frame (enc, width, height, TRUE, 0);
+  frame = allocate_frame (enc, width, height, TRUE);
   frame->pic_num = frame_num;
   encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_I,
       frame_num, NULL, 0, NULL, 0, vps_id, sps_id, pps_id);
@@ -1084,7 +1083,7 @@ GST_START_TEST (test_encoder_h265_i_p_b)
   frame_num++;
 
   /* Encode 4th picture as a P-Frame */
-  frame = allocate_frame (enc, width, height, TRUE, list0_num);
+  frame = allocate_frame (enc, width, height, TRUE);
   frame->pic_num = frame_num + 2;
   encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_P,
       frame_num, list0, list0_num, NULL, 0, vps_id, sps_id, pps_id);
@@ -1093,7 +1092,7 @@ GST_START_TEST (test_encoder_h265_i_p_b)
   frame_num++;
 
   /* Encode 2nd picture as a B-Frame */
-  frame = allocate_frame (enc, width, height, FALSE, list0_num + list1_num);
+  frame = allocate_frame (enc, width, height, FALSE);
   frame->pic_num = frame_num - 1;
   encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_B,
       frame_num, list0, list0_num, list1, list1_num, vps_id, sps_id, pps_id);
@@ -1102,7 +1101,7 @@ GST_START_TEST (test_encoder_h265_i_p_b)
   _h265_encode_frame_free (frame);
 
   /* Encode 3rd picture as a B-Frame */
-  frame = allocate_frame (enc, width, height, FALSE, list0_num + list1_num);
+  frame = allocate_frame (enc, width, height, FALSE);
   frame->pic_num = frame_num - 1;
   encode_frame (enc, frame, STD_VIDEO_H265_SLICE_TYPE_B,
       frame_num, list0, list0_num, list1, list1_num, vps_id, sps_id, pps_id);