diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c
index 99755d7d2c..7d573fe9ef 100644
--- a/gst/rtp/gstrtph265depay.c
+++ b/gst/rtp/gstrtph265depay.c
@@ -574,7 +574,7 @@ gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay)
   gst_caps_unref (srccaps);
 
   /* Insert SPS and PPS into the stream on next opportunity */
-  {
+  if (rtph265depay->sps->len > 0 || rtph265depay->pps->len > 0) {
     gint i;
     GstBuffer *codec_data;
     GstMapInfo map;