diff --git a/gst/h264parse/gsth264parse.c b/gst/h264parse/gsth264parse.c
index 4a2bd414b6..7351d907e9 100644
--- a/gst/h264parse/gsth264parse.c
+++ b/gst/h264parse/gsth264parse.c
@@ -1297,9 +1297,9 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
         GST_DEBUG_OBJECT (h264parse, "setting new codec_data");
         gst_caps_set_simple (src_caps, "codec_data", GST_TYPE_BUFFER, buf,
             NULL);
-        gst_buffer_unref (buf);
         modified = TRUE;
       }
+      gst_buffer_unref (buf);
     } else {
       GST_DEBUG_OBJECT (h264parse, "no codec_data yet");
     }
@@ -1314,10 +1314,9 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
   /* save as new caps, caps will be set when pushing data */
   /* avoid replacing caps by a mere identical copy, thereby triggering
    * negotiating (which e.g. some container might not appreciate) */
-  if (modified) {
+  if (modified)
     gst_caps_replace (&h264parse->src_caps, src_caps);
-    gst_caps_unref (src_caps);
-  }
+  gst_caps_unref (src_caps);
 
   return TRUE;
 }