diff --git a/gst/rtp/gstrtph264depay.c b/gst/rtp/gstrtph264depay.c
index cd05e62051..482fb62f2b 100644
--- a/gst/rtp/gstrtph264depay.c
+++ b/gst/rtp/gstrtph264depay.c
@@ -955,7 +955,6 @@ static GstBuffer *
 gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
 {
   GstRtpH264Depay *rtph264depay;
-  GstBuffer *buf;
   GstBuffer *outbuf = NULL;
   guint8 nal_unit_type;
 
@@ -982,7 +981,6 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
 
     payload_len = gst_rtp_buffer_get_payload_len (rtp);
     payload = gst_rtp_buffer_get_payload (rtp);
-    buf = gst_rtp_buffer_get_payload_buffer (rtp);
     marker = gst_rtp_buffer_get_marker (rtp);
 
     GST_DEBUG_OBJECT (rtph264depay, "receiving %d bytes", payload_len);
@@ -1067,8 +1065,8 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
           memcpy (map.data + sizeof (sync_bytes), payload, nalu_size);
           gst_buffer_unmap (outbuf, &map);
 
-          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf, buf,
-              g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
+          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf,
+              rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
 
           outbuf =
               gst_rtp_h264_depay_handle_nal (rtph264depay, outbuf, timestamp,
@@ -1153,8 +1151,8 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
           map.data[sizeof (sync_bytes)] = nal_header;
           gst_buffer_unmap (outbuf, &map);
 
-          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf, buf,
-              g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
+          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf,
+              rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
 
           GST_DEBUG_OBJECT (rtph264depay, "queueing %d bytes", outsize);
 
@@ -1169,8 +1167,8 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
           outbuf = gst_buffer_new_and_alloc (outsize);
           gst_buffer_fill (outbuf, 0, payload, outsize);
 
-          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf, buf,
-              g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
+          gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf,
+              rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
 
           GST_DEBUG_OBJECT (rtph264depay, "queueing %d bytes", outsize);
 
@@ -1207,8 +1205,8 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
         memcpy (map.data + sizeof (sync_bytes), payload, nalu_size);
         gst_buffer_unmap (outbuf, &map);
 
-        gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf, buf,
-            g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
+        gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph264depay), outbuf,
+            rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR));
 
         outbuf = gst_rtp_h264_depay_handle_nal (rtph264depay, outbuf, timestamp,
             marker);
@@ -1217,35 +1215,29 @@ gst_rtp_h264_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp)
     }
   }
 
-  gst_buffer_unref (buf);
-
   return outbuf;
 
   /* ERRORS */
 empty_packet:
   {
     GST_DEBUG_OBJECT (rtph264depay, "empty packet");
-    gst_buffer_unref (buf);
     return NULL;
   }
 undefined_type:
   {
     GST_ELEMENT_WARNING (rtph264depay, STREAM, DECODE,
         (NULL), ("Undefined packet type"));
-    gst_buffer_unref (buf);
     return NULL;
   }
 waiting_start:
   {
     GST_DEBUG_OBJECT (rtph264depay, "waiting for start");
-    gst_buffer_unref (buf);
     return NULL;
   }
 not_implemented:
   {
     GST_ELEMENT_ERROR (rtph264depay, STREAM, FORMAT,
         (NULL), ("NAL unit type %d not supported yet", nal_unit_type));
-    gst_buffer_unref (buf);
     return NULL;
   }
 }