diff --git a/gst/mpeg4videoparse/mpeg4videoparse.c b/gst/mpeg4videoparse/mpeg4videoparse.c
index d23f2cc1fd..254db9fb7d 100644
--- a/gst/mpeg4videoparse/mpeg4videoparse.c
+++ b/gst/mpeg4videoparse/mpeg4videoparse.c
@@ -502,10 +502,11 @@ gst_mpeg4vparse_push (GstMpeg4VParse * parse, gsize size)
     GstBuffer *out_buf;
 
     out_buf = gst_adapter_take_buffer (parse->adapter, parse->offset);
-    out_buf = gst_buffer_make_metadata_writable (out_buf);
-    GST_BUFFER_TIMESTAMP (out_buf) = parse->timestamp;
 
     if (G_LIKELY (out_buf)) {
+      out_buf = gst_buffer_make_metadata_writable (out_buf);
+      GST_BUFFER_TIMESTAMP (out_buf) = parse->timestamp;
+
       /* Set GST_BUFFER_FLAG_DELTA_UNIT if it's not an intra frame */
       if (!parse->intra_frame) {
         GST_BUFFER_FLAG_SET (out_buf, GST_BUFFER_FLAG_DELTA_UNIT);
@@ -540,10 +541,10 @@ gst_mpeg4vparse_push (GstMpeg4VParse * parse, gsize size)
 
           /* insert header */
           superbuf = gst_buffer_merge (parse->config, out_buf);
-
-          GST_BUFFER_TIMESTAMP (superbuf) = timestamp;
           gst_buffer_unref (out_buf);
-          out_buf = superbuf;
+
+          out_buf = gst_buffer_make_metadata_writable (superbuf);
+          GST_BUFFER_TIMESTAMP (out_buf) = timestamp;
 
           if (G_UNLIKELY (timestamp != -1)) {
             parse->last_report = timestamp;