diff --git a/gst/flv/gstflvmux.c b/gst/flv/gstflvmux.c
index 584558c37c..6d0a140a19 100644
--- a/gst/flv/gstflvmux.c
+++ b/gst/flv/gstflvmux.c
@@ -418,7 +418,7 @@ gst_flv_mux_video_pad_setcaps (GstFlvMuxPad * pad, GstCaps * caps)
     const GValue *val = gst_structure_get_value (s, "codec_data");
 
     if (val)
-      pad->codec_data = gst_buffer_ref (gst_value_get_buffer (val));
+      gst_buffer_replace (&pad->codec_data, gst_value_get_buffer (val));
   }
 
   gst_object_unref (mux);
@@ -559,7 +559,7 @@ gst_flv_mux_audio_pad_setcaps (GstFlvMuxPad * pad, GstCaps * caps)
     const GValue *val = gst_structure_get_value (s, "codec_data");
 
     if (val)
-      pad->codec_data = gst_buffer_ref (gst_value_get_buffer (val));
+      gst_buffer_replace (&pad->codec_data, gst_value_get_buffer (val));
   }
 
   gst_object_unref (mux);