diff --git a/gst/rtp/gstrtpmp4vdepay.c b/gst/rtp/gstrtpmp4vdepay.c
index 04a1442549..c1c911afe8 100644
--- a/gst/rtp/gstrtpmp4vdepay.c
+++ b/gst/rtp/gstrtpmp4vdepay.c
@@ -152,11 +152,10 @@ gst_rtp_mp4v_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps)
       GstBuffer *buffer;
 
       buffer = gst_value_get_buffer (&v);
-      gst_buffer_ref (buffer);
-      g_value_unset (&v);
-
       gst_caps_set_simple (srccaps,
           "codec_data", GST_TYPE_BUFFER, buffer, NULL);
+      /* caps takes ref */
+      g_value_unset (&v);
     } else {
       g_warning ("cannot convert config to buffer");
     }