mirrors/gstreamer
mirrors/gstreamer
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gstreamer.git synced 2024-11-25 19:21:06 +00:00
Code Issues 1 Projects Releases Wiki Activity

h264parse: Improved AU boundary detection

Browse source 
- AU boundary detection reviewed to follow more closely H.264 spec. and more
  specifically clauses 7.4.1.2.3 and 7.4.1.2.4.
- The gist of the changes is a look-a-head in then next AU required identify the
  last vcl-nal of current AU and firt vcl-nal of next AU (according to
  7.4.1.2.4) followed by the identification of the first nal of next AU
  (according to 7.4.1.2.3).
- A backlog of all nals of current AU and next AU up to the point where current
  AU can identified completed is kept.
- In NAL alignement mode vcl-nal are sent immediatly but the history is kept to
  allow AU boundary detection. Non-vcl-nal can be delayed up to the reception of
  the next vcl-nal to allow a correct AUD insertion.
- Based on this improved AU boudary detection we can avoid erronous AUD
  insertion, like the one highlighted by test
  test_parse_sliced_with_prefix_and_sei_nal_au.
- Add support for MVC AU boundary detection. (H.7.4.1.2.4)
- Explicitly report SVC not supported. We don't have the SVC NAL parsing
  required to identify boundary. (missing dependency_id and quality_id fields
  from SVC, see G.7.4.1.2.4)

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5741>
This commit is contained in:
Daniel Morin 2023-01-11 11:08:58 -05:00 committed by GStreamer Marge Bot
parent 6d71605508
commit 49f200cb54
3 changed files with 667 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
subprojects/gst-plugins-bad/gst-libs/gst/codecparsers/gsth264parser.h
View file

@ -133,6 +133,8 @@ typedef enum {
* @GST_H264_NAL_PREFIX_UNIT: Prefix NAL unit
* @GST_H264_NAL_SUBSET_SPS: Subset sequence parameter set (SSPS) NAL unit
* @GST_H264_NAL_DEPTH_SPS: Depth parameter set (DPS) NAL unit
* @GST_H264_NAL_RSV_1: First reserved parameter
* @GST_H264_NAL_RSV_2: Second reserved parameter
* @GST_H264_NAL_SLICE_AUX: Auxiliary coded picture without partitioning NAL unit
* @GST_H264_NAL_SLICE_EXT: Coded slice extension NAL unit
* @GST_H264_NAL_SLICE_DEPTH: Coded slice extension for depth or 3D-AVC texture view
@ -158,6 +160,24 @@ typedef enum
GST_H264_NAL_PREFIX_UNIT = 14,
GST_H264_NAL_SUBSET_SPS = 15,
GST_H264_NAL_DEPTH_SPS = 16,
/**
* GST_H264_NAL_RSV_1:
*
* First reserved parameter
*
* Since: 1.24
*/
GST_H264_NAL_RSV_1 = 17,
/**
* GST_H264_NAL_RSV_2:
*
* Second reserved parameter
*
* Since: 1.24
*/
GST_H264_NAL_RSV_2 = 18,
GST_H264_NAL_SLICE_AUX = 19,
GST_H264_NAL_SLICE_EXT = 20,
GST_H264_NAL_SLICE_DEPTH = 21

701
subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.c
View file

@ -40,6 +40,9 @@ GST_DEBUG_CATEGORY (h264_parse_debug);
#define DEFAULT_CONFIG_INTERVAL (0)
#define DEFAULT_UPDATE_TIMECODE FALSE
#define HIST_IDX_CURR 0
#define HIST_IDX_PREV 1
enum
{
PROP_0,
@ -82,6 +85,14 @@ enum
GST_H264_PARSE_SEI_PARSED = 2,
};
typedef enum
{
GST_H264_PARSE_BACKLOG_STATUS_AU_INCOMPLETE = 0,
GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE,
GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED,
GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED
} GstH264ParseBacklogStatus;
#define GST_H264_PARSE_STATE_VALID(parse, expected_state) \
(((parse)->state & (expected_state)) == (expected_state))
@ -128,6 +139,9 @@ static gboolean gst_h264_parse_src_event (GstBaseParse * parse,
static void gst_h264_parse_update_src_caps (GstH264Parse * h264parse,
GstCaps * caps);
static GstH264ParseBacklogStatus gst_h264_parse_update_backlog (GstH264Parse *
h264parse, GstH264NalUnit * nalu);
static void
gst_h264_parse_class_init (GstH264ParseClass * klass)
{
@ -205,6 +219,14 @@ gst_h264_parse_init (GstH264Parse * h264parse)
h264parse->aud_needed = TRUE;
h264parse->aud_insert = TRUE;
h264parse->update_timecode = DEFAULT_UPDATE_TIMECODE;
h264parse->nal_backlog = g_array_new (FALSE, FALSE, sizeof (GstH264NalUnit));
h264parse->bl_curr_au_last_vcl = -1;
h264parse->bl_next_au_first_vcl = 1;
h264parse->bl_next_au_first_nal = 1;
h264parse->bl_next_nal = 0;
h264parse->history_slice[HIST_IDX_CURR].valid = FALSE;
h264parse->history_slice[HIST_IDX_PREV].valid = FALSE;
}
static void
@ -215,6 +237,7 @@ gst_h264_parse_finalize (GObject * object)
gst_video_user_data_unregistered_clear (&h264parse->user_data_unregistered);
g_object_unref (h264parse->frame_out);
g_array_unref (h264parse->nal_backlog);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
@ -224,9 +247,6 @@ gst_h264_parse_reset_frame (GstH264Parse * h264parse)
{
GST_DEBUG_OBJECT (h264parse, "reset frame");
/* done parsing; reset state */
h264parse->current_off = -1;
h264parse->update_caps = FALSE;
h264parse->idr_pos = -1;
h264parse->sei_pos = -1;
@ -311,6 +331,7 @@ gst_h264_parse_reset (GstH264Parse * h264parse)
h264parse->discard_bidirectional = FALSE;
h264parse->marker = FALSE;
g_array_set_size (h264parse->nal_backlog, 0);
gst_h264_parse_reset_stream_info (h264parse);
}
@ -332,6 +353,7 @@ gst_h264_parse_start (GstBaseParse * parse)
h264parse->field_pic_flag = 0;
h264parse->aud_needed = TRUE;
h264parse->aud_insert = FALSE;
h264parse->current_off = -1;
gst_base_parse_set_min_frame_size (parse, 4);
@ -948,6 +970,66 @@ gst_h264_parse_process_sei (GstH264Parse * h264parse, GstH264NalUnit * nalu)
g_array_free (messages, TRUE);
}
static void
gst_h264_parse_update_vcl_nal_history_sps (GstH264Parse * h264parse,
GstH264SPS * sps)
{
h264parse->history_sps[HIST_IDX_PREV] = h264parse->history_sps[HIST_IDX_CURR];
h264parse->history_sps[HIST_IDX_CURR].pic_order_cnt_type =
sps->pic_order_cnt_type;
h264parse->history_sps[HIST_IDX_CURR].profile_idc = sps->profile_idc;
}
static void
gst_h264_parse_update_vcl_nal_history_pps (GstH264Parse * h264parse,
GstH264PPS * pps)
{
gst_h264_parse_update_vcl_nal_history_sps (h264parse, pps->sequence);
h264parse->history_pps[HIST_IDX_PREV] = h264parse->history_pps[HIST_IDX_CURR];
h264parse->history_pps[HIST_IDX_CURR].id = pps->id;
}
static void
gst_h264_parse_update_vcl_nal_history_nalu (GstH264Parse * h264parse,
GstH264NalUnit * nalu)
{
h264parse->history_nalu[HIST_IDX_PREV]
= h264parse->history_nalu[HIST_IDX_CURR];
h264parse->history_nalu[HIST_IDX_CURR].ref_idc = nalu->ref_idc;
h264parse->history_nalu[HIST_IDX_CURR].idr_pic_flag = nalu->idr_pic_flag;
if (GST_H264_IS_MVC_NALU (nalu))
h264parse->history_nalu[HIST_IDX_CURR].view_id =
nalu->extension.mvc.view_id;
}
static void
gst_h264_parse_update_vcl_nal_history (GstH264Parse * h264parse,
GstH264NalUnit * nalu, GstH264SliceHdr * slice)
{
gst_h264_parse_update_vcl_nal_history_nalu (h264parse, nalu);
gst_h264_parse_update_vcl_nal_history_pps (h264parse, slice->pps);
h264parse->history_slice[HIST_IDX_PREV]
= h264parse->history_slice[HIST_IDX_CURR];
h264parse->history_slice[HIST_IDX_CURR].valid = TRUE;
h264parse->history_slice[HIST_IDX_CURR].frame_num = slice->frame_num;
h264parse->history_slice[HIST_IDX_CURR].field_pic_flag
= slice->field_pic_flag;
h264parse->history_slice[HIST_IDX_CURR].bottom_field_flag
= slice->bottom_field_flag;
h264parse->history_slice[HIST_IDX_CURR].idr_pic_id = slice->idr_pic_id;
h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt[0]
= slice->delta_pic_order_cnt[0];
h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt[1]
= slice->delta_pic_order_cnt[1];
h264parse->history_slice[HIST_IDX_CURR].pic_order_cnt_lsb
= slice->pic_order_cnt_lsb;
h264parse->history_slice[HIST_IDX_CURR].delta_pic_order_cnt_bottom
= slice->delta_pic_order_cnt_bottom;
h264parse->history_slice[HIST_IDX_CURR].first_mb_in_slice
= slice->first_mb_in_slice;
}
/* caller guarantees 2 bytes of nal payload */
static gboolean
gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
@ -1179,44 +1261,6 @@ gst_h264_parse_process_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
return TRUE;
}
/* caller guarantees at least 2 bytes of nal payload for each nal
* returns TRUE if next_nal indicates that nal terminates an AU */
static inline gboolean
gst_h264_parse_collect_nal (GstH264Parse * h264parse, GstH264NalUnit * nalu)
{
GstH264NalUnitType nal_type = nalu->type;
gboolean complete;
/* determine if AU complete */
GST_LOG_OBJECT (h264parse, "next nal type: %d %s (picture started %i)",
nal_type, _nal_name (nal_type), h264parse->picture_start);
/* consider a coded slices (IDR or not) to start a picture,
* (so ending the previous one) if first_mb_in_slice == 0
* (non-0 is part of previous one) */
/* NOTE this is not entirely according to Access Unit specs in 7.4.1.2.4,
* but in practice it works in sane cases, needs not much parsing,
* and also works with broken frame_num in NAL
* (where spec-wise would fail) */
complete = h264parse->picture_start && ((nal_type >= GST_H264_NAL_SEI &&
nal_type <= GST_H264_NAL_AU_DELIMITER) ||
(nal_type >= 14 && nal_type <= 18));
/* first_mb_in_slice == 0 considered start of frame */
if (nalu->size > nalu->header_bytes)
complete |= h264parse->picture_start && (nal_type == GST_H264_NAL_SLICE
|| nal_type == GST_H264_NAL_SLICE_DPA
|| nal_type == GST_H264_NAL_SLICE_IDR) &&
(nalu->data[nalu->offset + nalu->header_bytes] & 0x80);
GST_LOG_OBJECT (h264parse, "au complete: %d", complete);
if (complete)
h264parse->picture_start = FALSE;
return complete;
}
static guint8 au_delim[6] = {
0x00, 0x00, 0x00, 0x01, /* nal prefix */
0x09, /* nal unit type = access unit delimiter */
@ -1333,6 +1377,426 @@ gst_h264_parse_handle_frame_packetized (GstBaseParse * parse,
return ret;
}
static gboolean
gst_h264_parse_received_first_vcl_nal_base (GstH264Parse * h264parse,
GstH264ParseHistorySlice * slice_hdr_prev)
{
/* Ref. ITU-T H.264, 7.4.1.2.4 */
GstH264ParseHistorySlice *slice_hdr_curr
= &h264parse->history_slice[HIST_IDX_CURR];
GstH264ParseHistoryPPS *pps_hist_curr
= &h264parse->history_pps[HIST_IDX_CURR];
GstH264ParseHistoryPPS *pps_hist_prev
= &h264parse->history_pps[HIST_IDX_PREV];
GstH264ParseHistorySPS *sps_hist_curr
= &h264parse->history_sps[HIST_IDX_CURR];
GstH264ParseHistorySPS *sps_hist_prev
= &h264parse->history_sps[HIST_IDX_PREV];
GstH264ParseHistoryNalUnit *nalu_hist_curr
= &h264parse->history_nalu[HIST_IDX_CURR];
GstH264ParseHistoryNalUnit *nalu_hist_prev
= &h264parse->history_nalu[HIST_IDX_PREV];
if (slice_hdr_curr->frame_num != slice_hdr_prev->frame_num) {
return TRUE;
} else if (pps_hist_curr->id != pps_hist_prev->id) {
return TRUE;
} else if (slice_hdr_curr->field_pic_flag != slice_hdr_prev->field_pic_flag) {
return TRUE;
} else if (slice_hdr_curr->field_pic_flag
&& (slice_hdr_curr->bottom_field_flag
!= slice_hdr_prev->bottom_field_flag)) {
return TRUE;
} else if ((nalu_hist_curr->ref_idc == 0 || nalu_hist_prev->ref_idc == 0)
&& nalu_hist_curr->ref_idc != nalu_hist_prev->ref_idc) {
return TRUE;
} else if (sps_hist_curr->pic_order_cnt_type == 0
&& sps_hist_prev->pic_order_cnt_type == 0
&& (slice_hdr_curr->pic_order_cnt_lsb
!= slice_hdr_prev->pic_order_cnt_lsb
|| slice_hdr_curr->delta_pic_order_cnt_bottom
!= slice_hdr_prev->delta_pic_order_cnt_bottom)) {
return TRUE;
} else if (sps_hist_curr->pic_order_cnt_type == 1
&& sps_hist_prev->pic_order_cnt_type == 1
&& (slice_hdr_curr->delta_pic_order_cnt[0]
!= slice_hdr_prev->delta_pic_order_cnt[0]
|| slice_hdr_curr->delta_pic_order_cnt[1]
!= slice_hdr_prev->delta_pic_order_cnt[1])) {
return TRUE;
} else if (nalu_hist_curr->idr_pic_flag != nalu_hist_prev->idr_pic_flag) {
return TRUE;
} else if (nalu_hist_curr->idr_pic_flag == 1
&& nalu_hist_prev->idr_pic_flag == 1
&& (slice_hdr_curr->idr_pic_id != slice_hdr_prev->idr_pic_id)) {
return TRUE;
} else if (slice_hdr_curr->first_mb_in_slice
<= slice_hdr_prev->first_mb_in_slice) {
return TRUE;
}
return FALSE;
}
static gboolean
gst_h264_parse_received_first_vcl_nal_mvc (GstH264Parse * h264parse,
GstH264ParseHistorySlice * slice_hdr_prev)
{
/* Ref. ITU-T H.264, H.7.4.1.2.4 */
GstH264ParseHistoryNalUnit *nalu_hist_curr
= &h264parse->history_nalu[HIST_IDX_CURR];
GstH264ParseHistoryNalUnit *nalu_hist_prev
= &h264parse->history_nalu[HIST_IDX_PREV];
if (nalu_hist_curr->view_id != nalu_hist_prev->view_id)
return TRUE;
return gst_h264_parse_received_first_vcl_nal_base (h264parse, slice_hdr_prev);
}
static gboolean
gst_h264_parse_received_first_vcl_nal (GstH264Parse * h264parse)
{
GstH264ParseHistorySlice *slice_hdr_prev
= &h264parse->history_slice[HIST_IDX_PREV];
GstH264ParseHistorySPS *sps_hist_prev
= &h264parse->history_sps[HIST_IDX_PREV];
if (slice_hdr_prev->valid) {
switch (sps_hist_prev->profile_idc) {
case GST_H264_PROFILE_BASELINE:
case GST_H264_PROFILE_MAIN:
case GST_H264_PROFILE_EXTENDED:
case GST_H264_PROFILE_HIGH:
case GST_H264_PROFILE_HIGH10:
case GST_H264_PROFILE_HIGH_422:
case GST_H264_PROFILE_HIGH_444:
return gst_h264_parse_received_first_vcl_nal_base (h264parse,
slice_hdr_prev);
case GST_H264_PROFILE_MULTIVIEW_HIGH:
case GST_H264_PROFILE_STEREO_HIGH:
return gst_h264_parse_received_first_vcl_nal_mvc (h264parse,
slice_hdr_prev);
case GST_H264_PROFILE_SCALABLE_BASELINE:
case GST_H264_PROFILE_SCALABLE_HIGH:
/* SVC not supported, should not be reached */
g_return_val_if_reached (FALSE);
default:
return FALSE;
}
return gst_h264_parse_received_first_vcl_nal_base (h264parse,
slice_hdr_prev);
}
return FALSE;
}
static gboolean
is_potential_nonvcl_au_limit (GstH264NalUnit * nalu)
{
gboolean ret = FALSE;
switch (nalu->type) {
case GST_H264_NAL_AU_DELIMITER:
case GST_H264_NAL_SPS:
case GST_H264_NAL_PPS:
case GST_H264_NAL_SEI:
case GST_H264_NAL_PREFIX_UNIT:
case GST_H264_NAL_SUBSET_SPS:
case GST_H264_NAL_DEPTH_SPS:
case GST_H264_NAL_RSV_1:
case GST_H264_NAL_RSV_2:
ret = TRUE;
break;
default:
break;
}
return ret;
}
static GstH264ParseBacklogStatus
gst_h264_parse_update_backlog (GstH264Parse * h264parse, GstH264NalUnit * nalu)
{
gboolean is_first_vcl_nal = FALSE;
GstH264ParserResult pres;
GstH264SliceHdr slice_hdr;
gboolean nvcl_before_cau_vcl = FALSE;
g_array_append_val (h264parse->nal_backlog, *nalu);
#define LAST_IDX (h264parse->nal_backlog->len -1)
/* Update nal_backlog_potential_au_first_idx following
* ref. ITU-T H.264 7.4.1.2.3 */
switch (nalu->type) {
case GST_H264_NAL_SLICE:
case GST_H264_NAL_SLICE_DPA:
case GST_H264_NAL_SLICE_DPB:
case GST_H264_NAL_SLICE_DPC:
case GST_H264_NAL_SLICE_IDR:
case GST_H264_NAL_SLICE_EXT:
GST_DEBUG_OBJECT (h264parse, "vcl nal (%u) added to backlog", nalu->type);
pres = gst_h264_parser_parse_slice_hdr (h264parse->nalparser, nalu,
&slice_hdr, FALSE, FALSE);
if (pres == GST_H264_PARSER_OK) {
gst_h264_parse_update_vcl_nal_history (h264parse, nalu, &slice_hdr);
is_first_vcl_nal = gst_h264_parse_received_first_vcl_nal (h264parse);
} else {
/* Reset vcl nal history */
h264parse->history_slice[HIST_IDX_PREV].valid = FALSE;
/* Reset backlog */
h264parse->bl_curr_au_last_vcl = -1;
h264parse->bl_next_au_first_vcl = 1;
h264parse->bl_next_au_first_nal = 1;
h264parse->bl_next_nal = 0;
g_array_set_size (h264parse->nal_backlog, 0);
GST_DEBUG_OBJECT (h264parse, "Failed to parse slice header");
return GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED;
}
if (h264parse->bl_curr_au_last_vcl == -1) {
/* initialization, first vcl nal reception */
h264parse->bl_curr_au_last_vcl = LAST_IDX;
/* set index above backlog, meaning not received */
h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
} else {
if (is_first_vcl_nal) {
h264parse->bl_next_au_first_vcl = LAST_IDX;
/* First AUD, SPS, PPS, SEI, PREFIX_UNIT, SUBSET_SPS, DEPTH_SPS,
* RSV1, RSV2 between last vcl nal of current AU and first vcl nal
* of next AU define the first nal of the next AU, otherwise
* first vcl nal of next AU is the first nal on next AU.*/
if (h264parse->bl_next_au_first_nal <= h264parse->bl_curr_au_last_vcl)
h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
else
g_assert (h264parse->bl_next_au_first_nal <=
h264parse->bl_next_au_first_vcl);
} else {
h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
/*if previous vcl nal was not the last, non vcl nal can't be last,
* therefore we move index of last nal to the last received vcl
* nal.*/
h264parse->bl_next_au_first_nal = h264parse->bl_next_au_first_vcl;
}
}
break;
default:
if (h264parse->bl_curr_au_last_vcl == -1) {
/* if we didn't receive any vcl, any nal from next au hasn't been
* received yet. In this state all nal from backlog belong to
* current AU. */
h264parse->bl_next_au_first_nal = h264parse->nal_backlog->len;
h264parse->bl_next_au_first_vcl = h264parse->nal_backlog->len;
}
nvcl_before_cau_vcl =
h264parse->bl_next_au_first_nal <= h264parse->bl_curr_au_last_vcl;
if (is_potential_nonvcl_au_limit (nalu)) {
/* these nal define the the first nal of a new AU if they are between
* the last vcl nal (of current AU) and first vcl (of next AU).*/
if (nvcl_before_cau_vcl)
h264parse->bl_next_au_first_nal = LAST_IDX;
/* Not the most efficient way has this will done again in _process_nal
* but sps and pps must be parsed before parsing s slice hdr. */
if (nalu->type == GST_H264_NAL_SPS) {
GstH264SPS sps;
gst_h264_parser_parse_sps (h264parse->nalparser, nalu, &sps);
g_return_val_if_fail (sps.profile_idc !=
GST_H264_PROFILE_SCALABLE_BASELINE
&& sps.profile_idc != GST_H264_PROFILE_SCALABLE_HIGH,
GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED);
} else if (nalu->type == GST_H264_NAL_PPS) {
GstH264PPS pps;
gst_h264_parser_parse_pps (h264parse->nalparser, nalu, &pps);
} else if (nalu->type == GST_H264_NAL_SUBSET_SPS) {
GstH264SPS sps;
gst_h264_parser_parse_subset_sps (h264parse->nalparser, nalu, &sps);
g_return_val_if_fail (sps.profile_idc !=
GST_H264_PROFILE_SCALABLE_BASELINE
&& sps.profile_idc != GST_H264_PROFILE_SCALABLE_HIGH,
GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED);
}
}
GST_DEBUG_OBJECT (h264parse, "Non-vcl nal (%u) added to backlog",
nalu->type);
break;
}
return is_first_vcl_nal ? GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE :
GST_H264_PARSE_BACKLOG_STATUS_AU_INCOMPLETE;
}
static void
gst_h264_parse_trim_backlog (GstH264Parse * h264parse)
{
g_array_remove_range (h264parse->nal_backlog, 0,
h264parse->bl_next_au_first_nal);
h264parse->bl_next_nal = 0;
h264parse->bl_curr_au_last_vcl =
h264parse->bl_next_au_first_vcl - h264parse->bl_next_au_first_nal;
h264parse->bl_next_au_first_nal = h264parse->bl_curr_au_last_vcl + 1;
h264parse->bl_next_au_first_vcl = h264parse->bl_next_au_first_nal;
}
static void
gst_h264_parse_clear_backlog (GstH264Parse * h264parse)
{
h264parse->bl_next_nal = 0;
g_array_remove_range (h264parse->nal_backlog, 0, h264parse->nal_backlog->len);
h264parse->bl_curr_au_last_vcl = -1;
h264parse->bl_next_au_first_nal = 1;
h264parse->bl_next_au_first_vcl = 1;
}
static gboolean
gst_h264_parse_process_backlog_loop (GstH264Parse * h264parse,
gint curr_next_thresh, gboolean * aud_insert, guint8 * data,
gint * framesize)
{
GstH264NalUnit *bnalu;
gint i, size = 0;
for (i = h264parse->bl_next_nal; i < h264parse->nal_backlog->len; i++) {
bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit, i);
if (i < curr_next_thresh) {
if (aud_insert != NULL && i == 0 &&
bnalu->type != GST_H264_NAL_AU_DELIMITER)
*aud_insert = TRUE;
bnalu->data = (guint8 *) data;
if (gst_h264_parse_process_nal (h264parse, bnalu) == FALSE)
return FALSE;
size = bnalu->offset + bnalu->size;
h264parse->bl_next_nal = i + 1;
} else {
/* section of backlog that belong to next AU */
bnalu->offset -= size;
bnalu->sc_offset -= size;
}
}
*framesize += size;
return TRUE;
}
static gboolean
gst_h264_parse_process_backlog_nal (GstH264Parse * h264parse, gint * proc_size,
gboolean * aud_insert, guint8 * data, gboolean clear_bl,
gboolean au_completed)
{
GstH264NalUnit *bnalu;
gint framesize = 0;
g_assert (h264parse->nal_backlog != NULL);
g_assert (h264parse->nal_backlog->len > 0);
bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit,
h264parse->nal_backlog->len - 1);
h264parse->current_off = bnalu->offset + bnalu->size;
/* If the index of the first NAL from next AU is after the current AU vcl
* and the AU is not completed, we can't send the this nal downstream since
* we might need to insert a AUD before and we will only know this when we've
* received a new vcl nal. In this scenario even if we are in NAL alignment
* mode we have to keep non vcl NAL, that can start a AU, and only send them
* down stream when we know if the belong to current AU, in which case we
* just send them or belong if it belong to next AU where we might need to
* insert a AUD. If the first nal from next AU is a AUD we don't need to wait
* the completion the first vcl from next AU, AUD is the start of next AU.
*/
if (gst_h264_parse_process_backlog_loop (h264parse,
h264parse->bl_next_au_first_nal, aud_insert, data,
&framesize) == FALSE)
goto fail;
/* We've processed a complete AU */
if (au_completed) {
gst_h264_parse_trim_backlog (h264parse);
}
/* Process all backlog. Used when draining or output in NAL mode. */
if (gst_h264_parse_process_backlog_loop (h264parse,
h264parse->nal_backlog->len, aud_insert, data, &framesize) == FALSE)
goto fail;
if (clear_bl) {
gst_h264_parse_clear_backlog (h264parse);
}
/* Backlog content doesn't need to parsed again, adjust offset accordingly. */
h264parse->current_off -= framesize;
if (proc_size)
*proc_size = framesize;
return TRUE;
fail:
gst_h264_parse_clear_backlog (h264parse);
return FALSE;
}
static gboolean
gst_h264_parse_process_backlog (GstH264Parse * h264parse, gint * proc_size,
gboolean * aud_insert, guint8 * data, gboolean proc_nau, gboolean clear_bl)
{
GstH264NalUnit *bnalu;
gint framesize = 0;
g_assert (h264parse->nal_backlog != NULL);
g_assert (h264parse->nal_backlog->len > 0);
bnalu = &g_array_index (h264parse->nal_backlog, GstH264NalUnit,
h264parse->nal_backlog->len - 1);
h264parse->current_off = bnalu->offset + bnalu->size;
if (gst_h264_parse_process_backlog_loop (h264parse,
h264parse->bl_next_au_first_nal, aud_insert, data,
&framesize) == FALSE)
goto fail;
/* We've processed a complete AU */
if (h264parse->bl_next_au_first_nal < h264parse->nal_backlog->len) {
gst_h264_parse_trim_backlog (h264parse);
}
/* Process all backlog. Used when draining or output in NAL mode. */
if (proc_nau) {
if (gst_h264_parse_process_backlog_loop (h264parse,
h264parse->nal_backlog->len, aud_insert, data, &framesize) == FALSE)
goto fail;
}
if (clear_bl) {
gst_h264_parse_clear_backlog (h264parse);
}
/* What is in the backlog doesn't need to parsed again, adjust offset
* accordingly.*/
h264parse->current_off -= framesize;
if (proc_size)
*proc_size = framesize;
return TRUE;
fail:
gst_h264_parse_clear_backlog (h264parse);
return FALSE;
}
static GstFlowReturn
gst_h264_parse_handle_frame (GstBaseParse * parse,
GstBaseParseFrame * frame, gint * skipsize)
@ -1348,9 +1812,12 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
GstH264NalUnit nalu;
GstH264ParserResult pres;
gint framesize;
GstH264ParseBacklogStatus blstatus = FALSE;
if (G_UNLIKELY (GST_BUFFER_FLAG_IS_SET (frame->buffer,
GST_BUFFER_FLAG_DISCONT))) {
// If any input buffer is marked discont we propagate discont
// to parsed output buffer.
h264parse->discont = TRUE;
}
@ -1394,14 +1861,28 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
/* The parser is being drain, but no new data was added, just prentend this
* AU is complete */
if (drain && current_off == size) {
GST_DEBUG_OBJECT (h264parse, "draining with no new data");
nalu.size = 0;
nalu.offset = current_off;
goto end;
if (current_off == size) {
if (drain) {
GST_DEBUG_OBJECT (h264parse, "draining with no new data");
framesize = current_off;
if (!gst_h264_parse_process_backlog (h264parse, &framesize,
&h264parse->aud_insert, data, TRUE, FALSE)) {
*skipsize = current_off;
goto skip;
}
goto end;
} else {
/* All data already parsed, we need more data. */
goto more;
}
}
g_assert (current_off < size);
/* In some case the base class can reduce the amount of data it gave us on
* previous call. When this happen we just ask for more data. */
if (current_off > size) {
goto more;
}
GST_DEBUG_OBJECT (h264parse, "last parse position %d", current_off);
/* check for initial skip */
@ -1441,6 +1922,9 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
case GST_H264_PARSER_OK:
GST_DEBUG_OBJECT (h264parse, "complete nal (offset, size): (%u, %u) ",
nalu.offset, nalu.size);
if ((nalu.offset + nalu.size) == size)
nonext = TRUE;
break;
case GST_H264_PARSER_NO_NAL:
/* In NAL alignment, assume the NAL is broken */
@ -1494,11 +1978,16 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
if (current_off == 0) {
GST_DEBUG_OBJECT (h264parse, "skipping broken nal");
*skipsize = nalu.offset;
h264parse->current_off = -1;
goto skip;
} else {
GST_DEBUG_OBJECT (h264parse, "terminating au");
nalu.size = 0;
nalu.offset = nalu.sc_offset;
framesize = nalu.sc_offset;
if (!gst_h264_parse_process_backlog (h264parse, &framesize,
&h264parse->aud_insert, data, FALSE, TRUE)) {
*skipsize = current_off;
goto skip;
}
goto end;
}
break;
@ -1507,79 +1996,73 @@ gst_h264_parse_handle_frame (GstBaseParse * parse,
break;
}
GST_DEBUG_OBJECT (h264parse, "%p complete nal found. Off: %u, Size: %u",
data, nalu.offset, nalu.size);
if (gst_h264_parse_collect_nal (h264parse, &nalu)) {
h264parse->aud_needed = TRUE;
/* complete current frame, if it exist */
if (current_off > 0) {
nalu.size = 0;
nalu.offset = nalu.sc_offset;
h264parse->marker = TRUE;
break;
}
}
if (!gst_h264_parse_process_nal (h264parse, &nalu)) {
GST_WARNING_OBJECT (h264parse,
"broken/invalid nal Type: %d %s, Size: %u will be dropped",
nalu.type, _nal_name (nalu.type), nalu.size);
*skipsize = nalu.size;
blstatus = gst_h264_parse_update_backlog (h264parse, &nalu);
if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_UPD_FAILED) {
*skipsize = current_off;
GST_ERROR_OBJECT (h264parse, "Failed to update backlog");
goto skip;
} else if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_NOT_SUPPORTED) {
/* SVC is not supported */
GST_ELEMENT_ERROR (h264parse, STREAM, FORMAT,
("Error parsing H.264 stream"), ("Not supported H.264 stream"));
goto invalid_stream;
}
/* Make sure the next buffer will contain an AUD */
if (h264parse->aud_needed) {
h264parse->aud_insert = TRUE;
h264parse->aud_needed = FALSE;
}
/* Do not push immediately if we don't have all headers. This ensure that
* our caps are complete, avoiding a renegotiation */
if (h264parse->align == GST_H264_PARSE_ALIGN_NAL &&
!GST_H264_PARSE_STATE_VALID (h264parse,
GST_H264_PARSE_STATE_VALID_PICTURE_HEADERS))
frame->flags |= GST_BASE_PARSE_FRAME_FLAG_QUEUE;
/* if no next nal, we reached the end of this buffer */
if (nonext) {
/* If there is a marker flag, or input is AU, we know this is complete */
if (GST_BUFFER_FLAG_IS_SET (frame->buffer, GST_BUFFER_FLAG_MARKER) ||
h264parse->in_align == GST_H264_PARSE_ALIGN_AU) {
h264parse->marker = TRUE;
break;
if (h264parse->align == GST_H264_PARSE_ALIGN_NAL) {
if (!gst_h264_parse_process_backlog_nal (h264parse, &framesize,
&h264parse->aud_insert, data, FALSE,
blstatus == GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE)) {
*skipsize = current_off;
}
/* or if we are draining */
if (drain || h264parse->align == GST_H264_PARSE_ALIGN_NAL)
break;
if (framesize > 0)
goto end;
} else if (h264parse->align == GST_H264_PARSE_ALIGN_AU) {
if (h264parse->in_align != GST_H264_PARSE_ALIGN_AU) {
if (blstatus == GST_H264_PARSE_BACKLOG_STATUS_AU_COMPLETE) {
if (!gst_h264_parse_process_backlog (h264parse, &framesize,
&h264parse->aud_insert, data, FALSE, FALSE)) {
*skipsize = current_off;
goto skip;
}
if (framesize > 0)
goto end;
} else if (drain && nonext) {
if (!gst_h264_parse_process_backlog (h264parse, &framesize,
&h264parse->aud_insert, data, TRUE, FALSE)) {
*skipsize = current_off;
goto skip;
}
goto end;
}
} else {
/* Accumulate all NALs from current AU in backlog */
if (nonext) {
/* input and output alignment are AU, there's nothing to do more than
* inserting a AUD if it's missing. */
if (!gst_h264_parse_process_backlog (h264parse, &framesize,
&h264parse->aud_insert, data, TRUE, TRUE)) {
*skipsize = current_off;
goto skip;
}
goto end;
}
}
}
if (nonext) {
current_off = nalu.offset + nalu.size;
goto more;
}
/* If the output is NAL, we are done */
if (h264parse->align == GST_H264_PARSE_ALIGN_NAL)
break;
GST_DEBUG_OBJECT (h264parse, "Looking for more");
current_off = nalu.offset + nalu.size;
/* expect at least 3 bytes start_code, and 1 bytes NALU header.
* the length of the NALU payload can be zero.
* (e.g. EOS/EOB placed at the end of an AU.) */
if (size - current_off < 4) {
/* Finish the frame if there is no more data in the stream */
if (drain)
break;
goto more;
}
}
} /* while end */
end:
framesize = nalu.offset + nalu.size;
gst_buffer_unmap (buffer, &map);

55
subprojects/gst-plugins-bad/gst/videoparsers/gsth264parse.h
View file

@ -48,6 +48,40 @@ typedef struct _H264Params H264Params;
GType gst_h264_parse_get_type (void);
typedef struct
{
gboolean valid;
guint16 frame_num;
guint8 field_pic_flag;
guint8 bottom_field_flag;
guint16 idr_pic_id;
gint32 delta_pic_order_cnt[2];
guint16 pic_order_cnt_lsb;
guint32 delta_pic_order_cnt_bottom;
guint32 first_mb_in_slice;
} GstH264ParseHistorySlice;
typedef struct
{
guint16 ref_idc;
guint8 idr_pic_flag;
/* For MVC Extension */
guint16 view_id;
} GstH264ParseHistoryNalUnit;
typedef struct
{
guint8 pic_order_cnt_type;
guint8 profile_idc;
} GstH264ParseHistorySPS;
typedef struct
{
gint id;
} GstH264ParseHistoryPPS;
typedef struct _GstH264Parse GstH264Parse;
typedef struct _GstH264ParseClass GstH264ParseClass;
@ -157,6 +191,21 @@ struct _GstH264Parse
GstVideoMultiviewFlags multiview_flags;
gboolean first_in_bundle;
/* For insertion of AU Delimiter */
GArray *nal_backlog;
/* Index of last vcl nal of current AU in backlog */
gint bl_curr_au_last_vcl;
/* Index of first vcl nal of next AU in backlog */
gint bl_next_au_first_vcl;
/* Index of first nal of next AU in backlog */
gint bl_next_au_first_nal;
/* Index of next nal to be processed in backlog */
gint bl_next_nal;
GstVideoParseUserData user_data;
GstVideoParseUserDataUnregistered user_data_unregistered;
@ -168,6 +217,12 @@ struct _GstH264Parse
/* For forward predicted trickmode */
gboolean discard_bidirectional;
/* First VCL NAL unit of primary code picuture detection context */
GstH264ParseHistorySlice history_slice[2];
GstH264ParseHistoryNalUnit history_nalu[2];
GstH264ParseHistorySPS history_sps[2];
GstH264ParseHistoryPPS history_pps[2];
};
struct _GstH264ParseClass