rtph265: keep delta unit flag

Without this patch all buffers that pass the payloader
are marked as non-delta-unit buffers.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2969>

subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.c

@@ -922,21 +922,24 @@ gst_rtp_h265_pay_decode_nal (GstRtpH265Pay * payloader,
 }
 
 static GstFlowReturn gst_rtp_h265_pay_payload_nal (GstRTPBasePayload *
-    basepayload, GPtrArray * paybufs, GstClockTime dts, GstClockTime pts);
+    basepayload, GPtrArray * paybufs, GstClockTime dts, GstClockTime pts,
+    gboolean delta_unit);
 static GstFlowReturn gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload *
     basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts,
-    gboolean marker);
+    gboolean marker, gboolean delta_unit);
 static GstFlowReturn gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload *
     basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts,
-    gboolean marker, guint mtu, guint8 nal_type, const guint8 * nal_header,
+    gboolean marker, gboolean delta_unit, guint mtu, guint8 nal_type,
-    int size);
+    const guint8 * nal_header, int size);
 static GstFlowReturn gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload *
     basepayload, GstBuffer * paybuf, GstClockTime dts, GstClockTime pts,
-    gboolean marker, guint8 nal_type, const guint8 * nal_header, int size);
+    gboolean marker, gboolean delta_unit, guint8 nal_type,
+    const guint8 * nal_header, int size);
 
 static GstFlowReturn
 gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload,
-    GstRtpH265Pay * rtph265pay, GstClockTime dts, GstClockTime pts)
+    GstRtpH265Pay * rtph265pay, GstClockTime dts, GstClockTime pts,
+    gboolean delta_unit)
 {
   GstFlowReturn ret = GST_FLOW_OK;
   gboolean sent_all_vps_sps_pps = TRUE;
@@ -967,7 +970,7 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload,
     g_ptr_array_add (bufs, gst_buffer_ref (pps_buf));
   }
 
-  ret = gst_rtp_h265_pay_payload_nal (basepayload, bufs, dts, pts);
+  ret = gst_rtp_h265_pay_payload_nal (basepayload, bufs, dts, pts, FALSE);
   if (ret != GST_FLOW_OK) {
     /* not critical but warn */
     GST_WARNING_OBJECT (basepayload, "failed pushing VPS/SPS/PPS");
@@ -993,7 +996,8 @@ gst_rtp_h265_pay_reset_bundle (GstRtpH265Pay * rtph265pay)
 
 static GstFlowReturn
 gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload,
-    GPtrArray * paybufs, GstClockTime dts, GstClockTime pts)
+    GPtrArray * paybufs, GstClockTime dts, GstClockTime pts,
+    gboolean delta_unit)
 {
   GstRtpH265Pay *rtph265pay;
   guint mtu;
@@ -1103,7 +1107,8 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload,
       sent_ps = TRUE;
       GST_DEBUG_OBJECT (rtph265pay, "sending VPS/SPS/PPS before current frame");
       ret =
-          gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts);
+          gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts,
+          delta_unit);
       if (ret != GST_FLOW_OK) {
         gst_buffer_unref (paybuf);
         continue;
@@ -1112,10 +1117,10 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload,
 
     if (rtph265pay->aggregate_mode != GST_RTP_H265_AGGREGATE_NONE)
       ret = gst_rtp_h265_pay_payload_nal_bundle (basepayload, paybuf, dts, pts,
-          marker, nal_type, nal_header, size);
+          marker, delta_unit, nal_type, nal_header, size);
     else
       ret = gst_rtp_h265_pay_payload_nal_fragment (basepayload, paybuf, dts,
-          pts, marker, mtu, nal_type, nal_header, size);
+          pts, marker, delta_unit, mtu, nal_type, nal_header, size);
   }
 
   g_ptr_array_free (paybufs, TRUE);
@@ -1125,7 +1130,8 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload,
 
 static GstFlowReturn
 gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload,
-    GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker)
+    GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker,
+    gboolean delta_unit)
 {
   GstBufferList *outlist;
   GstBuffer *outbuf;
@@ -1142,6 +1148,9 @@ gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload,
   gst_rtp_buffer_set_marker (&rtp, marker);
   GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_MARKER);
 
+  if (delta_unit)
+    GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT);
+
   /* timestamp the outbuffer */
   GST_BUFFER_PTS (outbuf) = pts;
   GST_BUFFER_DTS (outbuf) = dts;
@@ -1164,6 +1173,7 @@ gst_rtp_h265_pay_payload_nal_single (GstRTPBasePayload * basepayload,
 static GstFlowReturn
 gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload,
     GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean marker,
+    gboolean delta_unit,
     guint mtu, guint8 nal_type, const guint8 * nal_header, int size)
 {
   GstRtpH265Pay *rtph265pay = (GstRtpH265Pay *) basepayload;
@@ -1179,7 +1189,7 @@ gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload,
         "NAL Unit fit in one packet datasize=%d mtu=%d", size, mtu);
     /* will fit in one packet */
     return gst_rtp_h265_pay_payload_nal_single (basepayload, paybuf, dts, pts,
-        marker);
+        marker, delta_unit);
   }
 
   GST_DEBUG_OBJECT (basepayload,
@@ -1238,6 +1248,12 @@ gst_rtp_h265_pay_payload_nal_fragment (GstRTPBasePayload * basepayload,
     gst_rtp_copy_video_meta (rtph265pay, outbuf, paybuf);
     gst_buffer_copy_into (outbuf, paybuf, GST_BUFFER_COPY_MEMORY, pos,
         fragment_size);
+    if (!delta_unit)
+      /* only the first packet sent should not have the flag */
+      delta_unit = TRUE;
+    else
+      GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT);
+
     /* add the buffer to the buffer list */
     gst_buffer_list_add (outlist, outbuf);
   }
@@ -1256,6 +1272,7 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker)
   guint length, bundle_size;
   GstBuffer *first, *outbuf;
   GstClockTime dts, pts;
+  gboolean delta_unit;
 
   bundle_size = rtph265pay->bundle_size;
 
@@ -1271,6 +1288,7 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker)
   first = gst_buffer_list_get (bundle, 0);
   dts = GST_BUFFER_DTS (first);
   pts = GST_BUFFER_PTS (first);
+  delta_unit = GST_BUFFER_FLAG_IS_SET (first, GST_BUFFER_FLAG_DELTA_UNIT);
 
   if (length == 1) {
     /* Push unaggregated NALU */
@@ -1330,13 +1348,14 @@ gst_rtp_h265_pay_send_bundle (GstRtpH265Pay * rtph265pay, gboolean marker)
 
   gst_rtp_h265_pay_reset_bundle (rtph265pay);
   return gst_rtp_h265_pay_payload_nal_single (basepayload, outbuf, dts, pts,
-      marker);
+      marker, delta_unit);
 }
 
 static gboolean
 gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload,
     GstBuffer * paybuf, GstClockTime dts, GstClockTime pts,
-    gboolean marker, guint8 nal_type, const guint8 * nal_header, int size)
+    gboolean marker, gboolean delta_unit, guint8 nal_type,
+    const guint8 * nal_header, int size)
 {
   GstRtpH265Pay *rtph265pay;
   GstFlowReturn ret;
@@ -1386,7 +1405,7 @@ gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload,
       goto out;
 
     return gst_rtp_h265_pay_payload_nal_fragment (basepayload, paybuf, dts, pts,
-        marker, mtu, nal_type, nal_header, size);
+        marker, delta_unit, mtu, nal_type, nal_header, size);
   }
 
   bundle_size = rtph265pay->bundle_size + pay_size;
@@ -1418,6 +1437,11 @@ gst_rtp_h265_pay_payload_nal_bundle (GstRTPBasePayload * basepayload,
   GST_BUFFER_PTS (paybuf) = pts;
   GST_BUFFER_DTS (paybuf) = dts;
 
+  if (delta_unit)
+    GST_BUFFER_FLAG_SET (paybuf, GST_BUFFER_FLAG_DELTA_UNIT);
+  else
+    GST_BUFFER_FLAG_UNSET (paybuf, GST_BUFFER_FLAG_DELTA_UNIT);
+
   gst_buffer_list_add (bundle, gst_buffer_ref (paybuf));
   rtph265pay->bundle_size += pay_size;
   ret = GST_FLOW_OK;
@@ -1452,6 +1476,7 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
   gboolean hevc;
   GstBuffer *paybuf = NULL;
   gsize skip;
+  gboolean delayed_not_delta_unit = FALSE;
   gboolean marker = FALSE;
   gboolean discont = FALSE;
   gboolean draining = (buffer == NULL);
@@ -1469,7 +1494,13 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
       return GST_FLOW_OK;
   } else {
     if (buffer) {
+      if (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT)) {
         if (gst_adapter_available (rtph265pay->adapter) == 0)
+          rtph265pay->delta_unit = FALSE;
+        else
+          delayed_not_delta_unit = TRUE;
+      }
+
       discont = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DISCONT);
       marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
       gst_adapter_push (rtph265pay->adapter, buffer);
@@ -1506,6 +1537,8 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
 
     pts = GST_BUFFER_PTS (buffer);
     dts = GST_BUFFER_DTS (buffer);
+    rtph265pay->delta_unit = GST_BUFFER_FLAG_IS_SET (buffer,
+        GST_BUFFER_FLAG_DELTA_UNIT);
     marker = GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_MARKER);
     GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes",
         remaining_buffer_size);
@@ -1559,7 +1592,13 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
       offset += nal_len;
       remaining_buffer_size -= nal_len;
     }
-    ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts);
+    ret =
+        gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts,
+        rtph265pay->delta_unit);
+
+    if (!rtph265pay->delta_unit)
+      /* only the first outgoing packet doesn't have the DELTA_UNIT flag */
+      rtph265pay->delta_unit = TRUE;
 
     gst_buffer_memory_unmap (&memory);
     gst_buffer_unref (buffer);
@@ -1674,12 +1713,22 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload,
         discont = FALSE;
       }
 
+      if (delayed_not_delta_unit) {
+        rtph265pay->delta_unit = FALSE;
+        delayed_not_delta_unit = FALSE;
+      } else {
+        /* only the first outgoing packet doesn't have the DELTA_UNIT flag */
+        rtph265pay->delta_unit = TRUE;
+      }
+
       /* move to next NAL packet */
       /* Skips the trailing zeros */
       gst_adapter_flush (rtph265pay->adapter, nal_len - size);
     }
     /* put the data in one or more RTP packets */
-    ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts);
+    ret =
+        gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts,
+        rtph265pay->delta_unit);
     g_array_set_size (nal_queue, 0);
   }
 

subprojects/gst-plugins-good/gst/rtp/gstrtph265pay.h

@@ -73,6 +73,9 @@ struct _GstRtpH265Pay
   gboolean send_vps_sps_pps;
   GstClockTime last_vps_sps_pps;
 
+  /* TRUE if the next NALU processed should have the DELTA_UNIT flag */
+  gboolean delta_unit;
+
   /* aggregate buffers with AP */
   GstBufferList *bundle;
   guint bundle_size;

subprojects/gst-plugins-good/tests/check/elements/rtph265.c

@@ -1128,6 +1128,156 @@ GST_START_TEST (test_rtph265pay_aggregate_verify_nalu_hdr)
 }
 
 GST_END_TEST;
+
+static guint8 h265_hvc1_idr_data[] = {
+  0x00, 0x00, 0x00, 0x1a, 0x28, 0x01, 0xaf, 0x05, 0x38, 0x4a, 0x03, 0x06, 0x7c,
+  0x7a, 0xb1, 0x8b, 0xff, 0xfe, 0xfd, 0xb7, 0xff, 0xff, 0xd1, 0xff, 0x40, 0x06,
+  0xd8, 0xd3, 0xb2, 0xf8
+};
+
+static guint8 h265_hvc1_non_idr_data[] = {
+  0x00, 0x00, 0x00, 0x0d, 0x02, 0x01, 0xd0, 0x09, 0x7e, 0x10, 0xc2, 0x02, 0xbc,
+  0x38, 0x6d, 0xcf, 0x80
+};
+
+GST_START_TEST (test_rtph265pay_delta_unit_flag)
+{
+  GstHarness *h = gst_harness_new_parse ("rtph265pay timestamp-offset=123"
+      " name=p");
+  GstFlowReturn ret;
+  GstBuffer *buffer;
+
+  gst_harness_set_src_caps_str (h,
+      "video/x-h265,alignment=au,stream-format=hvc1,"
+      "codec_data=(buffer)0104080000009e28000000003ff000fcfff8f800000f032000"
+      "01001740010c01ffff0408000003009e2800000300003fba0240210001002f4201010"
+      "408000003009e2800000300003f90041020b2dd492657ff80008000b5060606040000"
+      "03000400000300782022000100074401c172b02240");
+
+  /* key frame */
+  buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data,
+      sizeof (h265_hvc1_idr_data), 0);
+  ret = gst_harness_push (h, buffer);
+  fail_unless_equals_int (ret, GST_FLOW_OK);
+
+  /* delta unit frame */
+  buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY,
+      h265_hvc1_non_idr_data, sizeof (h265_hvc1_non_idr_data), 0,
+      sizeof (h265_hvc1_non_idr_data), NULL, NULL);
+  GST_BUFFER_PTS (buffer) = 0;
+  GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
+  ret = gst_harness_push (h, buffer);
+  fail_unless_equals_int (ret, GST_FLOW_OK);
+
+  buffer = gst_harness_pull (h);
+  fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT));
+  gst_buffer_unref (buffer);
+  buffer = gst_harness_pull (h);
+  fail_unless (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT));
+  gst_buffer_unref (buffer);
+
+  gst_harness_teardown (h);
+}
+
+GST_END_TEST;
+
+GST_START_TEST (test_rtph265pay_delta_unit_flag_config_interval)
+{
+  GstHarness *h = gst_harness_new_parse ("rtph265pay timestamp-offset=123"
+      " name=p config-interval=-1");
+  GstFlowReturn ret;
+  GstBuffer *buffer;
+  guint num_buffers;
+  guint8 *payload = NULL;
+  guint8 nal_type;
+  GstRTPBuffer rtp = GST_RTP_BUFFER_INIT;
+
+  gst_harness_set_src_caps_str (h,
+      "video/x-h265,alignment=au,stream-format=hvc1,"
+      "codec_data=(buffer)0104080000009e28000000003ff000fcfff8f800000f032000"
+      "01001740010c01ffff0408000003009e2800000300003fba0240210001002f4201010"
+      "408000003009e2800000300003f90041020b2dd492657ff80008000b5060606040000"
+      "03000400000300782022000100074401c172b02240");
+
+  /* key frame */
+  buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data,
+      sizeof (h265_hvc1_idr_data), 0);
+  ret = gst_harness_push (h, buffer);
+  fail_unless_equals_int (ret, GST_FLOW_OK);
+
+  /* delta unit */
+  buffer = gst_buffer_new_wrapped_full (GST_MEMORY_FLAG_READONLY,
+      h265_hvc1_non_idr_data, sizeof (h265_hvc1_non_idr_data), 0,
+      sizeof (h265_hvc1_non_idr_data), NULL, NULL);
+  GST_BUFFER_PTS (buffer) = 0;
+  GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT);
+  ret = gst_harness_push (h, buffer);
+  fail_unless_equals_int (ret, GST_FLOW_OK);
+
+  /* another key frame */
+  buffer = wrap_static_buffer_with_pts (h265_hvc1_idr_data,
+      sizeof (h265_hvc1_idr_data), 0);
+  ret = gst_harness_push (h, buffer);
+  fail_unless_equals_int (ret, GST_FLOW_OK);
+
+  /* VSP SPS PPS I P VSP SPS PPS I */
+  num_buffers = gst_harness_buffers_in_queue (h);
+  fail_unless_equals_int (num_buffers, 9);
+
+  for (guint i = 0; i < num_buffers; i++) {
+    buffer = gst_harness_pull (h);
+    fail_unless (gst_rtp_buffer_map (buffer, GST_MAP_READ, &rtp));
+    payload = gst_rtp_buffer_get_payload (&rtp);
+    nal_type = (GST_READ_UINT8 (payload) >> 1) & 0x3f;
+    GST_INFO ("nal_type=%d,delta_unit=%d", nal_type,
+        GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT));
+    if (i == 0) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_VPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 1) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_SPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 2) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_PPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 3) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_IDR_N_LP);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 4) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_TRAIL_R);
+      fail_unless (GST_BUFFER_FLAG_IS_SET (buffer, GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 5) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_VPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 6) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_SPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 7) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_PPS);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    } else if (i == 8) {
+      fail_unless_equals_int (nal_type, GST_H265_NAL_SLICE_IDR_N_LP);
+      fail_unless (!GST_BUFFER_FLAG_IS_SET (buffer,
+              GST_BUFFER_FLAG_DELTA_UNIT));
+    }
+
+    gst_rtp_buffer_unmap (&rtp);
+    gst_buffer_unref (buffer);
+  }
+
+  gst_harness_teardown (h);
+}
+
+GST_END_TEST;
+
+
 static Suite *
 rtph265_suite (void)
 {
@@ -1154,6 +1304,8 @@ rtph265_suite (void)
   tcase_add_test (tc_chain, test_rtph265pay_aggregate_with_discont);
   tcase_add_test (tc_chain, test_rtph265pay_aggregate_until_vcl);
   tcase_add_test (tc_chain, test_rtph265pay_aggregate_verify_nalu_hdr);
+  tcase_add_test (tc_chain, test_rtph265pay_delta_unit_flag);
+  tcase_add_test (tc_chain, test_rtph265pay_delta_unit_flag_config_interval);
 
   return s;
 }