diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c
index 9b90ebe176..1d50fc1730 100644
--- a/gst/videoparsers/gsth264parse.c
+++ b/gst/videoparsers/gsth264parse.c
@@ -1863,14 +1863,20 @@ gst_h264_parse_update_src_caps (GstH264Parse * h264parse, GstCaps * caps)
 
     src_caps = gst_pad_get_current_caps (GST_BASE_PARSE_SRC_PAD (h264parse));
 
-    if (src_caps
-        && gst_structure_has_field (gst_caps_get_structure (src_caps, 0),
-            "codec_data")) {
+    if (src_caps) {
       /* use codec data from old caps for comparison; we don't want to resend caps
          if everything is same except codec data; */
-      gst_caps_set_value (caps, "codec_data",
-          gst_structure_get_value (gst_caps_get_structure (src_caps, 0),
-              "codec_data"));
+      if (gst_structure_has_field (gst_caps_get_structure (src_caps, 0),
+              "codec_data")) {
+        gst_caps_set_value (caps, "codec_data",
+            gst_structure_get_value (gst_caps_get_structure (src_caps, 0),
+                "codec_data"));
+      } else if (!buf) {
+        GstStructure *s;
+        /* remove any left-over codec-data hanging around */
+        s = gst_caps_get_structure (caps, 0);
+        gst_structure_remove_field (s, "codec_data");
+      }
     }
 
     if (!(src_caps && gst_caps_is_strictly_equal (src_caps, caps))) {