diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c
index e7c6c68ac6..9b53dfd662 100644
--- a/gst/rtp/gstrtph265depay.c
+++ b/gst/rtp/gstrtph265depay.c
@@ -611,7 +611,8 @@ gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay)
   gst_caps_unref (srccaps);
 
   /* Insert SPS and PPS into the stream on next opportunity */
-  if (rtph265depay->sps->len > 0 || rtph265depay->pps->len > 0) {
+  if (rtph265depay->output_format != GST_H265_STREAM_FORMAT_HVC1
+      && (rtph265depay->sps->len > 0 || rtph265depay->pps->len > 0)) {
     gint i;
     GstBuffer *codec_data;
     GstMapInfo map;