diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c
index e23d8b5aff..8c9371bf56 100644
--- a/gst-libs/gst/codecs/gsth264decoder.c
+++ b/gst-libs/gst/codecs/gsth264decoder.c
@@ -1256,7 +1256,6 @@ gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self,
         to_mark = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_x);
         if (to_mark) {
           to_mark->ref = FALSE;
-          gst_h264_picture_unref (to_mark);
         } else {
           GST_WARNING_OBJECT (self, "Invalid short term ref pic num to unmark");
           return FALSE;
@@ -1286,7 +1285,6 @@ gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self,
         if (to_mark) {
           to_mark->long_term = TRUE;
           to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx;
-          gst_h264_picture_unref (to_mark);
         } else {
           GST_WARNING_OBJECT (self,
               "Invalid short term ref pic num to mark as long ref");
diff --git a/gst-libs/gst/codecs/gsth264picture.c b/gst-libs/gst/codecs/gsth264picture.c
index 2d1d6c0d1b..ceb8694e36 100644
--- a/gst-libs/gst/codecs/gsth264picture.c
+++ b/gst-libs/gst/codecs/gsth264picture.c
@@ -316,7 +316,7 @@ gst_h264_dpb_mark_all_non_ref (GstH264Dpb * dpb)
  *
  * Find a short term reference picture which has matching picture number
  *
- * Returns: (nullable) (transfer full): a #GstH264Picture
+ * Returns: (nullable) (transfer none): a #GstH264Picture
  */
 GstH264Picture *
 gst_h264_dpb_get_short_ref_by_pic_num (GstH264Dpb * dpb, gint pic_num)
@@ -330,7 +330,7 @@ gst_h264_dpb_get_short_ref_by_pic_num (GstH264Dpb * dpb, gint pic_num)
         g_array_index (dpb->pic_list, GstH264Picture *, i);
 
     if (picture->ref && !picture->long_term && picture->pic_num == pic_num)
-      return gst_h264_picture_ref (picture);
+      return picture;
   }
 
   GST_WARNING ("No short term reference picture for %d", pic_num);