From 6729a3b79ccbc108ea27641e460caec090703f4a Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Mon, 6 Dec 2010 12:16:12 +0100 Subject: [PATCH] j2kdepay: make the depayloader more resilient Use 3 adapters, one to accumulate paketization units, another on to accumulate tiles and a last one to accumulate the final frame. Don't just blindly flush the adapter on DISCONT but only discard the current packetization unit. When we dropped jpeg2000 packets between SOP markers, adjust the SOT header with the new lenght. --- gst/rtp/gstrtpj2kdepay.c | 421 ++++++++++++++++++++++++++++++++++----- gst/rtp/gstrtpj2kdepay.h | 15 +- 2 files changed, 383 insertions(+), 53 deletions(-) diff --git a/gst/rtp/gstrtpj2kdepay.c b/gst/rtp/gstrtpj2kdepay.c index 83ccd9c4a7..82411aa987 100644 --- a/gst/rtp/gstrtpj2kdepay.c +++ b/gst/rtp/gstrtpj2kdepay.c @@ -46,6 +46,16 @@ GST_STATIC_PAD_TEMPLATE ("sink", "clock-rate = (int) 90000, " "encoding-name = (string) \"JPEG2000\"") ); +typedef enum +{ + J2K_MARKER = 0xFF, + J2K_MARKER_SOC = 0x4F, + J2K_MARKER_SOT = 0x90, + J2K_MARKER_SOP = 0x91, + J2K_MARKER_SOD = 0x93, + J2K_MARKER_EOC = 0xD9 +} RtpJ2KMarker; + GST_BOILERPLATE (GstRtpJ2KDepay, gst_rtp_j2k_depay, GstBaseRTPDepayload, GST_TYPE_BASE_RTP_DEPAYLOAD); @@ -102,7 +112,40 @@ static void gst_rtp_j2k_depay_init (GstRtpJ2KDepay * rtpj2kdepay, GstRtpJ2KDepayClass * klass) { - rtpj2kdepay->adapter = gst_adapter_new (); + rtpj2kdepay->pu_adapter = gst_adapter_new (); + rtpj2kdepay->t_adapter = gst_adapter_new (); + rtpj2kdepay->f_adapter = gst_adapter_new (); +} + +static void +store_mheader (GstRtpJ2KDepay * rtpj2kdepay, guint idx, GstBuffer * buf) +{ + GstBuffer *old; + + GST_DEBUG_OBJECT (rtpj2kdepay, "storing main header %p at index %u", buf, + idx); + if ((old = rtpj2kdepay->MH[idx])) + gst_buffer_unref (old); + rtpj2kdepay->MH[idx] = buf; +} + +static void +clear_mheaders (GstRtpJ2KDepay * rtpj2kdepay) +{ + guint i; + + for (i = 0; i < 8; i++) + store_mheader (rtpj2kdepay, i, NULL); +} + +static void +gst_rtp_j2k_depay_reset (GstRtpJ2KDepay * rtpj2kdepay) +{ + clear_mheaders (rtpj2kdepay); + gst_adapter_clear (rtpj2kdepay->pu_adapter); + gst_adapter_clear (rtpj2kdepay->t_adapter); + gst_adapter_clear (rtpj2kdepay->f_adapter); + rtpj2kdepay->next_frag = 0; } static void @@ -112,8 +155,11 @@ gst_rtp_j2k_depay_finalize (GObject * object) rtpj2kdepay = GST_RTP_J2K_DEPAY (object); - g_object_unref (rtpj2kdepay->adapter); - rtpj2kdepay->adapter = NULL; + clear_mheaders (rtpj2kdepay); + + g_object_unref (rtpj2kdepay->pu_adapter); + g_object_unref (rtpj2kdepay->t_adapter); + g_object_unref (rtpj2kdepay->f_adapter); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -142,65 +188,185 @@ gst_rtp_j2k_depay_setcaps (GstBaseRTPDepayload * depayload, GstCaps * caps) return res; } -static GstBuffer * -gst_rtp_j2k_depay_process (GstBaseRTPDepayload * depayload, GstBuffer * buf) +static void +gst_rtp_j2k_depay_clear_pu (GstRtpJ2KDepay * rtpj2kdepay) +{ + gst_adapter_clear (rtpj2kdepay->pu_adapter); + rtpj2kdepay->have_sync = FALSE; +} + +static GstFlowReturn +gst_rtp_j2k_depay_flush_pu (GstBaseRTPDepayload * depayload) { GstRtpJ2KDepay *rtpj2kdepay; - GstBuffer *outbuf; - guint8 *payload; - guint frag_offset; + GstBuffer *mheader; + guint avail, MHF, mh_id; rtpj2kdepay = GST_RTP_J2K_DEPAY (depayload); - /* flush everything on discont for now */ - if (GST_BUFFER_IS_DISCONT (buf)) { - GST_DEBUG_OBJECT (rtpj2kdepay, "DISCONT, flushing data"); - gst_adapter_clear (rtpj2kdepay->adapter); - rtpj2kdepay->need_header = TRUE; + /* take all available buffers */ + avail = gst_adapter_available (rtpj2kdepay->pu_adapter); + if (avail == 0) + goto done; + + MHF = rtpj2kdepay->pu_MHF; + mh_id = rtpj2kdepay->last_mh_id; + + GST_DEBUG_OBJECT (rtpj2kdepay, "flushing PU of size %u", avail); + + if (MHF == 0) { + GList *packets, *walk; + + packets = gst_adapter_take_list (rtpj2kdepay->pu_adapter, avail); + /* append packets */ + for (walk = packets; walk; walk = g_list_next (walk)) { + GstBuffer *buf = GST_BUFFER_CAST (walk->data); + GST_DEBUG_OBJECT (rtpj2kdepay, "append pu packet of size %u", + GST_BUFFER_SIZE (buf)); + gst_adapter_push (rtpj2kdepay->t_adapter, buf); + } + g_list_free (packets); + } else { + /* we have a header */ + GST_DEBUG_OBJECT (rtpj2kdepay, "keeping header %u", mh_id); + /* we managed to see the start and end of the header, take all from + * adapter and keep in header */ + mheader = gst_adapter_take_buffer (rtpj2kdepay->pu_adapter, avail); + + store_mheader (rtpj2kdepay, mh_id, mheader); } - if (gst_rtp_buffer_get_payload_len (buf) < 8) - goto empty_packet; +done: + rtpj2kdepay->have_sync = FALSE; - payload = gst_rtp_buffer_get_payload (buf); + return GST_FLOW_OK; +} - /* - * 0 1 2 3 - * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - * |tp |MHF|mh_id|T| priority | tile number | - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - * |reserved | fragment offset | - * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - */ - frag_offset = (payload[5] << 16) | (payload[6] << 8) | payload[7]; +static GstFlowReturn +gst_rtp_j2k_depay_flush_tile (GstBaseRTPDepayload * depayload) +{ + GstRtpJ2KDepay *rtpj2kdepay; + guint avail, mh_id; + GList *packets, *walk; + guint8 end[2]; + GstFlowReturn ret = GST_FLOW_OK; - GST_DEBUG_OBJECT (rtpj2kdepay, "frag %u", frag_offset); + rtpj2kdepay = GST_RTP_J2K_DEPAY (depayload); - if (rtpj2kdepay->need_header) { - if (frag_offset != 0) + /* flush pending PU */ + gst_rtp_j2k_depay_flush_pu (depayload); + + /* take all available buffers */ + avail = gst_adapter_available (rtpj2kdepay->t_adapter); + if (avail == 0) + goto done; + + mh_id = rtpj2kdepay->last_mh_id; + + GST_DEBUG_OBJECT (rtpj2kdepay, "flushing tile of size %u", avail); + + if (gst_adapter_available (rtpj2kdepay->f_adapter) == 0) { + GstBuffer *mheader; + + /* we need a header now */ + if ((mheader = rtpj2kdepay->MH[mh_id]) == NULL) goto waiting_header; - rtpj2kdepay->need_header = FALSE; + /* push header in the adapter */ + GST_DEBUG_OBJECT (rtpj2kdepay, "pushing header %u", mh_id); + gst_adapter_push (rtpj2kdepay->f_adapter, gst_buffer_ref (mheader)); } - /* take JPEG 2000 data, push in the adapter */ - outbuf = gst_rtp_buffer_get_payload_subbuffer (buf, 8, -1); - gst_adapter_push (rtpj2kdepay->adapter, outbuf); - outbuf = NULL; + /* check for last bytes */ + gst_adapter_copy (rtpj2kdepay->t_adapter, end, avail - 2, 2); - if (gst_rtp_buffer_get_marker (buf)) { - guint avail; - guint8 end[2]; - guint8 *data; + /* now append the tile packets to the frame */ + packets = gst_adapter_take_list (rtpj2kdepay->t_adapter, avail); + for (walk = packets; walk; walk = g_list_next (walk)) { + GstBuffer *buf = GST_BUFFER_CAST (walk->data); - /* last buffer take all data out of the adapter */ - avail = gst_adapter_available (rtpj2kdepay->adapter); - GST_DEBUG_OBJECT (rtpj2kdepay, "marker set, last buffer"); + if (walk == packets) { + guint8 *data; + guint size; + /* first buffer should contain the SOT */ + data = GST_BUFFER_DATA (buf); + size = GST_BUFFER_SIZE (buf); + + if (size < 12) + goto invalid_tile; + + if (data[0] == 0xff && data[1] == J2K_MARKER_SOT) { + guint Psot, nPsot; + + if (end[0] == 0xff && end[1] == J2K_MARKER_EOC) + nPsot = avail - 2; + else + nPsot = avail; + + Psot = GST_READ_UINT32_BE (&data[6]); + if (Psot != nPsot && Psot != 0) { + /* Psot must match the size of the tile */ + GST_DEBUG_OBJECT (rtpj2kdepay, "set Psot from %u to %u", Psot, nPsot); + buf = gst_buffer_make_writable (buf); + data = GST_BUFFER_DATA (buf); + GST_WRITE_UINT32_BE (&data[6], nPsot); + } + } + } + + GST_DEBUG_OBJECT (rtpj2kdepay, "append pu packet of size %u", + GST_BUFFER_SIZE (buf)); + gst_adapter_push (rtpj2kdepay->f_adapter, buf); + } + g_list_free (packets); + +done: + rtpj2kdepay->last_tile = -1; + + return ret; + + /* errors */ +waiting_header: + { + GST_DEBUG_OBJECT (rtpj2kdepay, "waiting for header %u", mh_id); + gst_adapter_clear (rtpj2kdepay->t_adapter); + rtpj2kdepay->last_tile = -1; + return ret; + } +invalid_tile: + { + GST_ELEMENT_WARNING (rtpj2kdepay, STREAM, DECODE, ("Invalid tile"), (NULL)); + gst_adapter_clear (rtpj2kdepay->t_adapter); + rtpj2kdepay->last_tile = -1; + return ret; + } +} + +static GstFlowReturn +gst_rtp_j2k_depay_flush_frame (GstBaseRTPDepayload * depayload) +{ + GstRtpJ2KDepay *rtpj2kdepay; + guint8 end[2]; + guint8 *data; + guint avail; + GstBuffer *outbuf; + GstFlowReturn ret = GST_FLOW_OK; + + rtpj2kdepay = GST_RTP_J2K_DEPAY (depayload); + + /* flush pending tile */ + gst_rtp_j2k_depay_flush_tile (depayload); + + /* last buffer take all data out of the adapter */ + avail = gst_adapter_available (rtpj2kdepay->f_adapter); + if (avail == 0) + goto done; + + if (avail > 2) { /* take the last bytes of the JPEG 2000 data to see if there is an EOC * marker */ - gst_adapter_copy (rtpj2kdepay->adapter, end, avail - 2, 2); + gst_adapter_copy (rtpj2kdepay->f_adapter, end, avail - 2, 2); if (end[0] != 0xff && end[1] != 0xd9) { GST_DEBUG_OBJECT (rtpj2kdepay, "no EOC marker, adding one"); @@ -211,15 +377,166 @@ gst_rtp_j2k_depay_process (GstBaseRTPDepayload * depayload, GstBuffer * buf) data[0] = 0xff; data[1] = 0xd9; - gst_adapter_push (rtpj2kdepay->adapter, outbuf); + gst_adapter_push (rtpj2kdepay->f_adapter, outbuf); avail += 2; } - outbuf = gst_adapter_take_buffer (rtpj2kdepay->adapter, avail); - GST_DEBUG_OBJECT (rtpj2kdepay, "returning %u bytes", avail); + GST_DEBUG_OBJECT (rtpj2kdepay, "pushing %u bytes", avail); + outbuf = gst_adapter_take_buffer (rtpj2kdepay->f_adapter, avail); + ret = gst_base_rtp_depayload_push (depayload, outbuf); + } else { + GST_WARNING_OBJECT (rtpj2kdepay, "empty packet"); + gst_adapter_clear (rtpj2kdepay->f_adapter); } - return outbuf; + /* we accept any mh_id now */ + rtpj2kdepay->last_mh_id = -1; + + /* reset state */ + rtpj2kdepay->next_frag = 0; + rtpj2kdepay->have_sync = FALSE; + +done: + /* we can't keep headers with mh_id of 0 */ + store_mheader (rtpj2kdepay, 0, NULL); + + return ret; +} + +static GstBuffer * +gst_rtp_j2k_depay_process (GstBaseRTPDepayload * depayload, GstBuffer * buf) +{ + GstRtpJ2KDepay *rtpj2kdepay; + guint8 *payload; + guint MHF, mh_id, frag_offset, tile, payload_len, j2klen; + gint gap; + guint32 rtptime; + + rtpj2kdepay = GST_RTP_J2K_DEPAY (depayload); + + payload = gst_rtp_buffer_get_payload (buf); + payload_len = gst_rtp_buffer_get_payload_len (buf); + + /* we need at least a header */ + if (payload_len < 8) + goto empty_packet; + + rtptime = gst_rtp_buffer_get_timestamp (buf); + + /* new timestamp marks new frame */ + if (rtpj2kdepay->last_rtptime != rtptime) { + rtpj2kdepay->last_rtptime = rtptime; + /* flush pending frame */ + gst_rtp_j2k_depay_flush_frame (depayload); + } + + /* + * 0 1 2 3 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |tp |MHF|mh_id|T| priority | tile number | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |reserved | fragment offset | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + MHF = (payload[0] & 0x30) >> 4; + mh_id = (payload[0] & 0xe) >> 1; + + if (rtpj2kdepay->last_mh_id == -1) + rtpj2kdepay->last_mh_id = mh_id; + else if (rtpj2kdepay->last_mh_id != mh_id) + goto wrong_mh_id; + + tile = (payload[2] << 8) | payload[3]; + frag_offset = (payload[5] << 16) | (payload[6] << 8) | payload[7]; + j2klen = payload_len - 8; + + GST_DEBUG_OBJECT (rtpj2kdepay, "MHF %u, tile %u, frag %u, expected %u", MHF, + tile, frag_offset, rtpj2kdepay->next_frag); + + /* calculate the gap between expected frag */ + gap = frag_offset - rtpj2kdepay->next_frag; + /* calculate next frag */ + rtpj2kdepay->next_frag = frag_offset + j2klen; + + if (gap != 0) { + GST_DEBUG_OBJECT (rtpj2kdepay, "discont of %d, clear PU", gap); + /* discont, clear pu adapter and resync */ + gst_rtp_j2k_depay_clear_pu (rtpj2kdepay); + } + + /* check for sync code */ + if (j2klen > 2 && payload[8] == 0xff) { + guint marker = payload[9]; + + /* packets must start with SOC, SOT or SOP */ + switch (marker) { + case J2K_MARKER_SOC: + GST_DEBUG_OBJECT (rtpj2kdepay, "found SOC packet"); + /* flush the previous frame, should have happened when the timestamp + * changed above. */ + gst_rtp_j2k_depay_flush_frame (depayload); + rtpj2kdepay->have_sync = TRUE; + break; + case J2K_MARKER_SOT: + /* flush the previous tile */ + gst_rtp_j2k_depay_flush_tile (depayload); + GST_DEBUG_OBJECT (rtpj2kdepay, "found SOT packet"); + rtpj2kdepay->have_sync = TRUE; + /* we sync on the tile now */ + rtpj2kdepay->last_tile = tile; + break; + case J2K_MARKER_SOP: + GST_DEBUG_OBJECT (rtpj2kdepay, "found SOP packet"); + /* flush the previous PU */ + gst_rtp_j2k_depay_flush_pu (depayload); + if (rtpj2kdepay->last_tile != tile) { + /* wrong tile, we lose sync and we need a new SOT or SOC to regain + * sync. First flush out the previous tile if we have one. */ + if (rtpj2kdepay->last_tile != -1) + gst_rtp_j2k_depay_flush_tile (depayload); + /* now we have no more valid tile and no sync */ + rtpj2kdepay->last_tile = -1; + rtpj2kdepay->have_sync = FALSE; + } else { + rtpj2kdepay->have_sync = TRUE; + } + break; + default: + GST_DEBUG_OBJECT (rtpj2kdepay, "no sync packet 0x%02d", marker); + break; + } + } + + if (rtpj2kdepay->have_sync) { + GstBuffer *pu_frag; + + if (gst_adapter_available (rtpj2kdepay->pu_adapter) == 0) { + /* first part of pu, record state */ + GST_DEBUG_OBJECT (rtpj2kdepay, "first PU"); + rtpj2kdepay->pu_MHF = MHF; + } + /* and push in pu adapter */ + GST_DEBUG_OBJECT (rtpj2kdepay, "push pu of size %u in adapter", j2klen); + pu_frag = gst_rtp_buffer_get_payload_subbuffer (buf, 8, -1); + gst_adapter_push (rtpj2kdepay->pu_adapter, pu_frag); + + if (MHF & 2) { + /* last part of main header received, we can flush it */ + GST_DEBUG_OBJECT (rtpj2kdepay, "header end, flush pu"); + gst_rtp_j2k_depay_flush_pu (depayload); + } + } else { + GST_DEBUG_OBJECT (rtpj2kdepay, "discard packet, no sync"); + } + + /* marker bit finishes the frame */ + if (gst_rtp_buffer_get_marker (buf)) { + GST_DEBUG_OBJECT (rtpj2kdepay, "marker set, last buffer"); + /* then flush frame */ + gst_rtp_j2k_depay_flush_frame (depayload); + } + return NULL; /* ERRORS */ empty_packet: @@ -228,9 +545,12 @@ empty_packet: ("Empty Payload."), (NULL)); return NULL; } -waiting_header: +wrong_mh_id: { - GST_DEBUG_OBJECT (rtpj2kdepay, "we are waiting for a header"); + GST_ELEMENT_WARNING (rtpj2kdepay, STREAM, DECODE, + ("Invalid mh_id %u, expected %u", mh_id, rtpj2kdepay->last_mh_id), + (NULL)); + gst_rtp_j2k_depay_clear_pu (rtpj2kdepay); return NULL; } } @@ -247,8 +567,7 @@ gst_rtp_j2k_depay_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_NULL_TO_READY: break; case GST_STATE_CHANGE_READY_TO_PAUSED: - gst_adapter_clear (rtpj2kdepay->adapter); - rtpj2kdepay->need_header = TRUE; + gst_rtp_j2k_depay_reset (rtpj2kdepay); break; default: break; @@ -258,7 +577,7 @@ gst_rtp_j2k_depay_change_state (GstElement * element, GstStateChange transition) switch (transition) { case GST_STATE_CHANGE_PAUSED_TO_READY: - gst_adapter_clear (rtpj2kdepay->adapter); + gst_rtp_j2k_depay_reset (rtpj2kdepay); break; case GST_STATE_CHANGE_READY_TO_NULL: break; diff --git a/gst/rtp/gstrtpj2kdepay.h b/gst/rtp/gstrtpj2kdepay.h index 41120af42e..e855eef940 100644 --- a/gst/rtp/gstrtpj2kdepay.h +++ b/gst/rtp/gstrtpj2kdepay.h @@ -44,8 +44,19 @@ struct _GstRtpJ2KDepay { GstBaseRTPDepayload depayload; - GstAdapter *adapter; - gboolean need_header; + guint64 last_rtptime; + guint last_mh_id; + guint last_tile; + + GstBuffer *MH[8]; + + guint pu_MHF; + GstAdapter *pu_adapter; + GstAdapter *t_adapter; + GstAdapter *f_adapter; + + guint next_frag; + gboolean have_sync; gint width, height; };