From 494246f75cfc739cf677c10ad872c843138effb0 Mon Sep 17 00:00:00 2001 From: Jonas Holmberg Date: Thu, 8 Dec 2016 15:59:33 +0100 Subject: [PATCH] rtph265pay: Fix handling of config-interval Insert VPS/SPS/PPS before the first NAL unit containing an I-frame in an access unit only. If an access unit consists of several such NAL units (tiles) VPS/SPS/PPS should only be inserted before the first of them so that parameters are only updated between frames. Do not insert VPS/SPS/PPS before P-frames when config-interval is -1. https://bugzilla.gnome.org/show_bug.cgi?id=775817 --- gst/rtp/gstrtph265pay.c | 430 ++++++++++++++++++++-------------------- 1 file changed, 216 insertions(+), 214 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 8e9a887f5c..6eaf115788 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -808,7 +808,7 @@ gst_rtp_h265_pay_decode_nal (GstRtpH265Pay * payloader, static GstFlowReturn gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, - GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au); + GPtrArray * paybufs, GstClockTime dts, GstClockTime pts); static GstFlowReturn gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, @@ -817,48 +817,38 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, GstFlowReturn ret = GST_FLOW_OK; gboolean sent_all_vps_sps_pps = TRUE; guint i; + GPtrArray *bufs; + + bufs = g_ptr_array_new (); for (i = 0; i < rtph265pay->vps->len; i++) { GstBuffer *vps_buf = GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->vps, i)); GST_DEBUG_OBJECT (rtph265pay, "inserting VPS in the stream"); - /* resend VPS */ - ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (vps_buf), - dts, pts, FALSE); - /* Not critical here; but throw a warning */ - if (ret != GST_FLOW_OK) { - sent_all_vps_sps_pps = FALSE; - GST_WARNING_OBJECT (basepayload, "Problem pushing VPS"); - } + g_ptr_array_add (bufs, gst_buffer_ref (vps_buf)); } for (i = 0; i < rtph265pay->sps->len; i++) { GstBuffer *sps_buf = GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->sps, i)); GST_DEBUG_OBJECT (rtph265pay, "inserting SPS in the stream"); - /* resend SPS */ - ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (sps_buf), - dts, pts, FALSE); - /* Not critical here; but throw a warning */ - if (ret != GST_FLOW_OK) { - sent_all_vps_sps_pps = FALSE; - GST_WARNING_OBJECT (basepayload, "Problem pushing SPS"); - } + g_ptr_array_add (bufs, gst_buffer_ref (sps_buf)); } for (i = 0; i < rtph265pay->pps->len; i++) { GstBuffer *pps_buf = GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->pps, i)); GST_DEBUG_OBJECT (rtph265pay, "inserting PPS in the stream"); - /* resend PPS */ - ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (pps_buf), - dts, pts, FALSE); - /* Not critical here; but throw a warning */ - if (ret != GST_FLOW_OK) { - sent_all_vps_sps_pps = FALSE; - GST_WARNING ("Problem pushing PPS"); - } + g_ptr_array_add (bufs, gst_buffer_ref (pps_buf)); + } + + ret = gst_rtp_h265_pay_payload_nal (basepayload, bufs, dts, pts); + if (ret != GST_FLOW_OK) { + /* not critical but warn */ + GST_WARNING_OBJECT (basepayload, "failed pushing VPS/SPS/PPS"); + + sent_all_vps_sps_pps = FALSE; } if (pts != -1 && sent_all_vps_sps_pps) @@ -869,207 +859,239 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, static GstFlowReturn gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, - GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au) + GPtrArray * paybufs, GstClockTime dts, GstClockTime pts) { GstRtpH265Pay *rtph265pay; + guint mtu; GstFlowReturn ret; - guint8 nalHeader[2]; - guint8 nalType; - guint packet_len, payload_len, mtu; - GstBuffer *outbuf; - guint8 *payload; - GstBufferList *list = NULL; - gboolean send_vps_sps_pps; - GstRTPBuffer rtp = { NULL }; - guint size = gst_buffer_get_size (paybuf); + gint i; + gboolean sent_ps; rtph265pay = GST_RTP_H265_PAY (basepayload); mtu = GST_RTP_BASE_PAYLOAD_MTU (rtph265pay); - gst_buffer_extract (paybuf, 0, nalHeader, 2); - nalType = (nalHeader[0] >> 1) & 0x3f; - - GST_DEBUG_OBJECT (rtph265pay, "Processing Buffer with NAL TYPE=%d", nalType); - /* should set src caps before pushing stuff, * and if we did not see enough VPS/SPS/PPS, that may not be the case */ if (G_UNLIKELY (!gst_pad_has_current_caps (GST_RTP_BASE_PAYLOAD_SRCPAD (basepayload)))) gst_rtp_h265_pay_set_vps_sps_pps (basepayload); - send_vps_sps_pps = FALSE; + ret = GST_FLOW_OK; + sent_ps = FALSE; + for (i = 0; i < paybufs->len; i++) { + guint8 nalHeader[2]; + guint8 nalType; + guint packet_len, payload_len; + GstBuffer *paybuf; + GstBuffer *outbuf; + guint8 *payload; + GstBufferList *outlist = NULL; + gboolean send_ps; + GstRTPBuffer rtp = { NULL }; + guint size; - /* check if we need to emit an VPS/SPS/PPS now */ - if ((nalType == GST_H265_NAL_SLICE_TRAIL_N) - || (nalType == GST_H265_NAL_SLICE_TRAIL_R) - || (nalType == GST_H265_NAL_SLICE_TSA_N) - || (nalType == GST_H265_NAL_SLICE_TSA_R) - || (nalType == GST_H265_NAL_SLICE_STSA_N) - || (nalType == GST_H265_NAL_SLICE_STSA_R) - || (nalType == GST_H265_NAL_SLICE_RASL_N) - || (nalType == GST_H265_NAL_SLICE_RASL_R) - || (nalType == GST_H265_NAL_SLICE_BLA_W_LP) - || (nalType == GST_H265_NAL_SLICE_BLA_W_RADL) - || (nalType == GST_H265_NAL_SLICE_BLA_N_LP) - || (nalType == GST_H265_NAL_SLICE_IDR_W_RADL) - || (nalType == GST_H265_NAL_SLICE_IDR_N_LP) - || (nalType == GST_H265_NAL_SLICE_CRA_NUT)) { - if (rtph265pay->vps_sps_pps_interval > 0) { - if (rtph265pay->last_vps_sps_pps != -1) { - guint64 diff; + paybuf = g_ptr_array_index (paybufs, i); - GST_LOG_OBJECT (rtph265pay, - "now %" GST_TIME_FORMAT ", last VPS/SPS/PPS %" GST_TIME_FORMAT, - GST_TIME_ARGS (pts), GST_TIME_ARGS (rtph265pay->last_vps_sps_pps)); - - /* calculate diff between last SPS/PPS in milliseconds */ - if (pts > rtph265pay->last_vps_sps_pps) - diff = pts - rtph265pay->last_vps_sps_pps; - else - diff = 0; - - GST_DEBUG_OBJECT (rtph265pay, - "interval since last VPS/SPS/PPS %" GST_TIME_FORMAT, - GST_TIME_ARGS (diff)); - - /* bigger than interval, queue SPS/PPS */ - if (GST_TIME_AS_SECONDS (diff) >= rtph265pay->vps_sps_pps_interval) { - GST_DEBUG_OBJECT (rtph265pay, "time to send VPS/SPS/PPS"); - send_vps_sps_pps = TRUE; - } - } else { - /* no known previous SPS/PPS time, send now */ - GST_DEBUG_OBJECT (rtph265pay, "no previous VPS/SPS/PPS time, send now"); - send_vps_sps_pps = TRUE; - } - } else if (rtph265pay->vps_sps_pps_interval == -1) { - GST_DEBUG_OBJECT (rtph265pay, - "sending VPS/SPS/PPS before current IDR frame"); - /* send VPS/SPS/PPS before every IDR frame */ - send_vps_sps_pps = TRUE; - } - } - - if (send_vps_sps_pps || rtph265pay->send_vps_sps_pps) { - /* we need to send SPS/PPS now first. FIXME, don't use the pts for - * checking when we need to send SPS/PPS but convert to running_time first. */ - rtph265pay->send_vps_sps_pps = FALSE; - ret = gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts); if (ret != GST_FLOW_OK) { + /* unref buffers that will not be payloaded after a flow error */ gst_buffer_unref (paybuf); - return ret; - } - } - - packet_len = gst_rtp_buffer_calc_packet_len (size, 0, 0); - - if (packet_len < mtu) { - GST_DEBUG_OBJECT (rtph265pay, - "NAL Unit fit in one packet datasize=%d mtu=%d", size, mtu); - /* will fit in one packet */ - - /* use buffer lists - * create buffer without payload containing only the RTP header - * (memory block at index 0) */ - outbuf = gst_rtp_buffer_new_allocate (0, 0, 0); - - gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp); - - /* only set the marker bit on packets containing access units */ - if (IS_ACCESS_UNIT (nalType) && end_of_au) { - gst_rtp_buffer_set_marker (&rtp, 1); + continue; } - /* timestamp the outbuffer */ - GST_BUFFER_PTS (outbuf) = pts; - GST_BUFFER_DTS (outbuf) = dts; + size = gst_buffer_get_size (paybuf); + gst_buffer_extract (paybuf, 0, nalHeader, 2); + nalType = (nalHeader[0] >> 1) & 0x3f; - /* insert payload memory block */ - gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265pay), outbuf, paybuf, - g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); - outbuf = gst_buffer_append (outbuf, paybuf); + GST_DEBUG_OBJECT (rtph265pay, "Processing Buffer with NAL TYPE=%d", + nalType); - list = gst_buffer_list_new (); + send_ps = FALSE; - /* add the buffer to the buffer list */ - gst_buffer_list_add (list, outbuf); + /* check if we need to emit an VPS/SPS/PPS now */ + if ((nalType == GST_H265_NAL_SLICE_TRAIL_N) + || (nalType == GST_H265_NAL_SLICE_TRAIL_R) + || (nalType == GST_H265_NAL_SLICE_TSA_N) + || (nalType == GST_H265_NAL_SLICE_TSA_R) + || (nalType == GST_H265_NAL_SLICE_STSA_N) + || (nalType == GST_H265_NAL_SLICE_STSA_R) + || (nalType == GST_H265_NAL_SLICE_RASL_N) + || (nalType == GST_H265_NAL_SLICE_RASL_R) + || (nalType == GST_H265_NAL_SLICE_BLA_W_LP) + || (nalType == GST_H265_NAL_SLICE_BLA_W_RADL) + || (nalType == GST_H265_NAL_SLICE_BLA_N_LP) + || (nalType == GST_H265_NAL_SLICE_IDR_W_RADL) + || (nalType == GST_H265_NAL_SLICE_IDR_N_LP) + || (nalType == GST_H265_NAL_SLICE_CRA_NUT)) { + if (rtph265pay->vps_sps_pps_interval > 0) { + if (rtph265pay->last_vps_sps_pps != -1) { + guint64 diff; - gst_rtp_buffer_unmap (&rtp); + GST_LOG_OBJECT (rtph265pay, + "now %" GST_TIME_FORMAT ", last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (pts), + GST_TIME_ARGS (rtph265pay->last_vps_sps_pps)); - /* push the list to the next element in the pipe */ - ret = gst_rtp_base_payload_push_list (basepayload, list); - } else { - /* fragmentation Units */ - guint limitedSize; - int ii = 0, start = 1, end = 0, pos = 0; + /* calculate diff between last SPS/PPS in milliseconds */ + if (pts > rtph265pay->last_vps_sps_pps) + diff = pts - rtph265pay->last_vps_sps_pps; + else + diff = 0; - GST_DEBUG_OBJECT (basepayload, - "NAL Unit DOES NOT fit in one packet datasize=%d mtu=%d", size, mtu); + GST_DEBUG_OBJECT (rtph265pay, + "interval since last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (diff)); - pos += 2; - size -= 2; + /* bigger than interval, queue SPS/PPS */ + if (GST_TIME_AS_SECONDS (diff) >= rtph265pay->vps_sps_pps_interval) { + GST_DEBUG_OBJECT (rtph265pay, "time to send VPS/SPS/PPS"); + send_ps = TRUE; + } + } else { + /* no known previous SPS/PPS time, send now */ + GST_DEBUG_OBJECT (rtph265pay, + "no previous VPS/SPS/PPS time, send now"); + send_ps = TRUE; + } + } else if (rtph265pay->vps_sps_pps_interval == -1 + && (nalType == GST_H265_NAL_SLICE_IDR_W_RADL + || nalType == GST_H265_NAL_SLICE_IDR_N_LP)) { + /* send VPS/SPS/PPS before every IDR frame */ + send_ps = TRUE; + } + } - GST_DEBUG_OBJECT (basepayload, "Using FU fragmentation for data size=%d", - size); + if (!sent_ps && (send_ps || rtph265pay->send_vps_sps_pps)) { + /* we need to send SPS/PPS now first. FIXME, don't use the pts for + * checking when we need to send SPS/PPS but convert to running_time + * first */ + rtph265pay->send_vps_sps_pps = FALSE; + sent_ps = TRUE; + GST_DEBUG_OBJECT (rtph265pay, "sending VPS/SPS/PPS before current frame"); + ret = + gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts); + if (ret != GST_FLOW_OK) { + gst_buffer_unref (paybuf); + continue; + } + } - /* We keep 3 bytes for PayloadHdr and FU Header */ - payload_len = gst_rtp_buffer_calc_payload_len (mtu - 3, 0, 0); + packet_len = gst_rtp_buffer_calc_packet_len (size, 0, 0); - list = gst_buffer_list_new (); - - while (end == 0) { - limitedSize = size < payload_len ? size : payload_len; - GST_DEBUG_OBJECT (basepayload, - "Inside FU fragmentation limitedSize=%d iteration=%d", limitedSize, - ii); + if (packet_len < mtu) { + GST_DEBUG_OBJECT (rtph265pay, + "NAL Unit fit in one packet datasize=%d mtu=%d", size, mtu); + /* will fit in one packet */ /* use buffer lists * create buffer without payload containing only the RTP header - * (memory block at index 0), and with space for PayloadHdr and FU header */ - outbuf = gst_rtp_buffer_new_allocate (3, 0, 0); + * (memory block at index 0) */ + outbuf = gst_rtp_buffer_new_allocate (0, 0, 0); gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp); - GST_BUFFER_DTS (outbuf) = dts; + /* only set the marker bit on packets containing access units */ + if (i == paybufs->len - 1 + && rtph265pay->alignment == GST_H265_ALIGNMENT_AU + && IS_ACCESS_UNIT (nalType)) { + gst_rtp_buffer_set_marker (&rtp, 1); + } + + /* timestamp the outbuffer */ GST_BUFFER_PTS (outbuf) = pts; - payload = gst_rtp_buffer_get_payload (&rtp); - - if (limitedSize == size) { - GST_DEBUG_OBJECT (basepayload, "end size=%d iteration=%d", size, ii); - end = 1; - } - - /* PayloadHdr (type = 49) */ - payload[0] = (nalHeader[0] & 0x81) | (49 << 1); - payload[1] = nalHeader[1]; - - /* set the marker bit on the last packet of an access unit */ - if (IS_ACCESS_UNIT (nalType)) { - gst_rtp_buffer_set_marker (&rtp, end && end_of_au); - } - - /* FU Header */ - payload[2] = (start << 7) | (end << 6) | (nalType & 0x3f); - - gst_rtp_buffer_unmap (&rtp); + GST_BUFFER_DTS (outbuf) = dts; /* insert payload memory block */ gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265pay), outbuf, paybuf, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); - gst_buffer_copy_into (outbuf, paybuf, GST_BUFFER_COPY_MEMORY, pos, - limitedSize); + outbuf = gst_buffer_append (outbuf, paybuf); + + outlist = gst_buffer_list_new (); + /* add the buffer to the buffer list */ - gst_buffer_list_add (list, outbuf); + gst_buffer_list_add (outlist, outbuf); - size -= limitedSize; - pos += limitedSize; - ii++; - start = 0; + gst_rtp_buffer_unmap (&rtp); + + /* push the list to the next element in the pipe */ + ret = gst_rtp_base_payload_push_list (basepayload, outlist); + } else { + /* fragmentation Units */ + guint limitedSize; + int ii = 0, start = 1, end = 0, pos = 0; + + GST_DEBUG_OBJECT (basepayload, + "NAL Unit DOES NOT fit in one packet datasize=%d mtu=%d", size, mtu); + + pos += 2; + size -= 2; + + GST_DEBUG_OBJECT (basepayload, "Using FU fragmentation for data size=%d", + size); + + /* We keep 3 bytes for PayloadHdr and FU Header */ + payload_len = gst_rtp_buffer_calc_payload_len (mtu - 3, 0, 0); + + outlist = gst_buffer_list_new (); + + while (end == 0) { + limitedSize = size < payload_len ? size : payload_len; + GST_DEBUG_OBJECT (basepayload, + "Inside FU fragmentation limitedSize=%d iteration=%d", limitedSize, + ii); + + /* use buffer lists + * create buffer without payload containing only the RTP header + * (memory block at index 0), and with space for PayloadHdr and FU header */ + outbuf = gst_rtp_buffer_new_allocate (3, 0, 0); + + gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp); + + GST_BUFFER_DTS (outbuf) = dts; + GST_BUFFER_PTS (outbuf) = pts; + payload = gst_rtp_buffer_get_payload (&rtp); + + if (limitedSize == size) { + GST_DEBUG_OBJECT (basepayload, "end size=%d iteration=%d", size, ii); + end = 1; + } + + /* PayloadHdr (type = 49) */ + payload[0] = (nalHeader[0] & 0x81) | (49 << 1); + payload[1] = nalHeader[1]; + + /* set the marker bit on the last packet of an access unit */ + if (IS_ACCESS_UNIT (nalType)) { + gst_rtp_buffer_set_marker (&rtp, + end && i == paybufs->len - 1 + && rtph265pay->alignment == GST_H265_ALIGNMENT_AU); + } + + /* FU Header */ + payload[2] = (start << 7) | (end << 6) | (nalType & 0x3f); + + gst_rtp_buffer_unmap (&rtp); + + /* insert payload memory block */ + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265pay), outbuf, paybuf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_buffer_copy_into (outbuf, paybuf, GST_BUFFER_COPY_MEMORY, pos, + limitedSize); + /* add the buffer to the buffer list */ + gst_buffer_list_add (outlist, outbuf); + + size -= limitedSize; + pos += limitedSize; + ii++; + start = 0; + } + + ret = gst_rtp_base_payload_push_list (basepayload, outlist); + gst_buffer_unref (paybuf); } - - ret = gst_rtp_base_payload_push_list (basepayload, list); - gst_buffer_unref (paybuf); } + + g_ptr_array_free (paybufs, TRUE); + return ret; } @@ -1136,12 +1158,13 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, if (hevc) { guint nal_length_size; gsize offset = 0; + GPtrArray *paybufs; + paybufs = g_ptr_array_new (); nal_length_size = rtph265pay->nal_length_size; while (size > nal_length_size) { gint i; - gboolean end_of_au = FALSE; nal_len = 0; for (i = 0; i < nal_length_size; i++) { @@ -1161,30 +1184,19 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, nal_len); } - /* If we're at the end of the buffer, then we're at the end of the - * access unit - */ - if (rtph265pay->alignment == GST_H265_ALIGNMENT_AU - && size - nal_len <= nal_length_size) { - end_of_au = TRUE; - } - paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL, offset, nal_len); - - ret = - gst_rtp_h265_pay_payload_nal (basepayload, paybuf, dts, pts, - end_of_au); - if (ret != GST_FLOW_OK) - break; + g_ptr_array_add (paybufs, paybuf); data += nal_len; offset += nal_len; size -= nal_len; } + ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts); } else { guint next; gboolean update = FALSE; + GPtrArray *paybufs; /* get offset of first start code */ next = next_start_code (data, size); @@ -1202,6 +1214,8 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, GST_DEBUG_OBJECT (basepayload, "found first start at %u, bytes left %" G_GSIZE_FORMAT, next, size); + paybufs = g_ptr_array_new (); + /* first pass to locate NALs and parse VPS/SPS/PPS */ while (size > 4) { /* skip start code */ @@ -1254,7 +1268,6 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, for (i = 0; i < nal_queue->len; i++) { guint size; - gboolean end_of_au = FALSE; nal_len = g_array_index (nal_queue, guint, i); /* skip start code */ @@ -1270,32 +1283,21 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, for (; size > 1 && data[size - 1] == 0x0; size--) /* skip */ ; - - /* If it's the last nal unit we have in non-bytestream mode, we can - * assume it's the end of an access-unit - * - * FIXME: We need to wait until the next packet or EOS to + /* FIXME: We need to wait until the next packet or EOS to * actually payload the NAL so we can know if the current NAL is * the last one of an access unit or not if we are in bytestream mode */ - if ((rtph265pay->alignment == GST_H265_ALIGNMENT_AU || buffer == NULL) && - i == nal_queue->len - 1) - end_of_au = TRUE; + paybuf = gst_adapter_take_buffer (rtph265pay->adapter, size); g_assert (paybuf); - - /* put the data in one or more RTP packets */ - ret = - gst_rtp_h265_pay_payload_nal (basepayload, paybuf, dts, pts, - end_of_au); - if (ret != GST_FLOW_OK) { - break; - } + g_ptr_array_add (paybufs, paybuf); /* move to next NAL packet */ /* Skips the trailing zeros */ gst_adapter_flush (rtph265pay->adapter, nal_len - size); } + /* put the data in one or more RTP packets */ + ret = gst_rtp_h265_pay_payload_nal (basepayload, paybufs, dts, pts); g_array_set_size (nal_queue, 0); }