diff --git a/gst/rtp/gstrtpgstpay.c b/gst/rtp/gstrtpgstpay.c
index 77f19d5bbc..ffaae71a6c 100644
--- a/gst/rtp/gstrtpgstpay.c
+++ b/gst/rtp/gstrtpgstpay.c
@@ -198,6 +198,8 @@ gst_rtp_gst_pay_handle_buffer (GstRTPBasePayload * basepayload,
 
     GST_DEBUG_OBJECT (rtpgstpay, "sending inline caps");
     rtpgstpay->next_CV++;
+
+    flags |= (1 << 7);
   } else {
     capslen_prefix_len = 0;
   }
@@ -243,9 +245,6 @@ gst_rtp_gst_pay_handle_buffer (GstRTPBasePayload * basepayload,
     gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp);
     payload = gst_rtp_buffer_get_payload (&rtp);
 
-    if (capslen > 0)
-      flags |= (1 << 7);
-
     GST_DEBUG_OBJECT (basepayload, "new packet len %u, frag %u", packet_len,
         frag_offset);