diff --git a/gst/rtp/gstrtph264depay.c b/gst/rtp/gstrtph264depay.c
index 6a9a97be2a..6ce38205ca 100644
--- a/gst/rtp/gstrtph264depay.c
+++ b/gst/rtp/gstrtph264depay.c
@@ -311,7 +311,7 @@ gst_rtp_h264_depay_set_output_caps (GstRtpH264Depay * rtph264depay,
 static gboolean
 gst_rtp_h264_set_src_caps (GstRtpH264Depay * rtph264depay)
 {
-  gboolean res, update_caps;
+  gboolean res = TRUE;
   GstCaps *srccaps;
   GstCaps *old_caps;
   GstPad *srcpad;
@@ -447,36 +447,8 @@ gst_rtp_h264_set_src_caps (GstRtpH264Depay * rtph264depay)
 
   old_caps = gst_pad_get_current_caps (srcpad);
 
-  if (old_caps != NULL) {
-    /* Only update the caps if they are not equal. For
-     * AVC we don't update caps if only the codec_data
-     * changes. This is the same behaviour as in h264parse
-     */
-    if (rtph264depay->byte_stream) {
-      update_caps = !gst_caps_is_equal (srccaps, old_caps);
-    } else {
-      GstCaps *tmp_caps = gst_caps_copy (srccaps);
-      GstStructure *old_s, *tmp_s;
-
-      old_s = gst_caps_get_structure (old_caps, 0);
-      tmp_s = gst_caps_get_structure (tmp_caps, 0);
-      if (gst_structure_has_field (old_s, "codec_data"))
-        gst_structure_set_value (tmp_s, "codec_data",
-            gst_structure_get_value (old_s, "codec_data"));
-
-      update_caps = !gst_caps_is_equal (old_caps, tmp_caps);
-
-      gst_caps_unref (tmp_caps);
-    }
-    gst_caps_unref (old_caps);
-  } else {
-    update_caps = TRUE;
-  }
-
-  if (update_caps) {
+  if (old_caps == NULL || !gst_caps_is_equal (srccaps, old_caps)) {
     res = gst_rtp_h264_depay_set_output_caps (rtph264depay, srccaps);
-  } else {
-    res = TRUE;
   }
 
   gst_caps_unref (srccaps);