codecs: h264dec: Improve the algorithm for low latency mode.

In low_latency mode, try to bump the picture as soon as possible
without the frames disorder:
1. We can directly output the continuous non-reference frame.
2. Consider max_num_reorder_frames, which is special useful for
   I-P mode.
3. Consider the leading pictures with negative POC.
4  Output small POC pictures when non-reference frame comes.
4. Output the POC increment<=2 pictures. This is not 100% safe,
   but in practice this condition can be used.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2373>
This commit is contained in:
He Junyan 2021-07-21 00:04:18 +08:00 committed by GStreamer Marge Bot
parent 573d3f5ba5
commit 130205629f
3 changed files with 151 additions and 15 deletions

View file

@ -1846,7 +1846,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
gst_h264_decoder_drain_internal (self);
}
while (gst_h264_dpb_needs_bump (priv->dpb, picture, FALSE)) {
while (gst_h264_dpb_needs_bump (priv->dpb, picture, priv->is_live)) {
GstH264Picture *to_output;
to_output = gst_h264_dpb_bump (priv->dpb, FALSE);
@ -1897,6 +1897,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
}
} else {
gst_h264_decoder_do_output_picture (self, picture);
gst_h264_dpb_set_last_output (priv->dpb, picture);
}
GST_LOG_OBJECT (self,

View file

@ -110,6 +110,7 @@ struct _GstH264Dpb
gint num_output_needed;
guint32 max_num_reorder_frames;
gint32 last_output_poc;
gboolean last_output_non_ref;
gboolean interlaced;
};
@ -119,6 +120,7 @@ gst_h264_dpb_init (GstH264Dpb * dpb)
{
dpb->num_output_needed = 0;
dpb->last_output_poc = G_MININT32;
dpb->last_output_non_ref = FALSE;
}
/**
@ -300,6 +302,13 @@ gst_h264_dpb_add (GstH264Dpb * dpb, GstH264Picture * picture)
if (dpb->pic_list->len > dpb->max_num_frames * (dpb->interlaced + 1))
GST_ERROR ("DPB size is %d, exceed the max size %d",
dpb->pic_list->len, dpb->max_num_frames);
/* The IDR frame or mem_mgmt_5 */
if (picture->pic_order_cnt == 0) {
GST_TRACE ("last_output_poc reset because of IDR or mem_mgmt_5");
dpb->last_output_poc = G_MININT32;
dpb->last_output_non_ref = FALSE;
}
}
/**
@ -701,16 +710,124 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert,
{
GstH264Picture *picture = NULL;
gint32 lowest_poc;
gboolean is_ref_picture;
gint lowest_index;
g_return_val_if_fail (dpb != NULL, FALSE);
g_assert (dpb->num_output_needed >= 0);
/* FIXME: Need to revisit for intelaced decoding */
if (low_latency) {
/* TODO: */
lowest_poc = G_MAXINT32;
is_ref_picture = FALSE;
lowest_index = gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
if (lowest_index >= 0) {
lowest_poc = picture->pic_order_cnt;
is_ref_picture = picture->ref_pic;
gst_h264_picture_unref (picture);
} else {
goto normal_bump;
}
if (low_latency) {
/* If low latency, we should not wait for the DPB becoming full.
We try to bump the picture as soon as possible without the
frames disorder. The policy is from the safe to some risk. */
/* Do not support interlaced mode. */
if (gst_h264_dpb_get_interlaced (dpb))
goto normal_bump;
/* Equal to normal bump. */
if (!gst_h264_dpb_has_empty_frame_buffer (dpb))
goto normal_bump;
/* 7.4.1.2.2: The values of picture order count for the coded pictures
in consecutive access units in decoding order containing non-reference
pictures shall be non-decreasing. Safe. */
if (dpb->last_output_non_ref && !is_ref_picture) {
g_assert (dpb->last_output_poc < G_MAXINT32);
GST_TRACE ("Continuous non-reference frame poc: %d -> %d,"
" bumping for low-latency.", dpb->last_output_poc, lowest_poc);
return TRUE;
}
/* num_reorder_frames indicates the maximum number of frames, that
precede any frame in the coded video sequence in decoding order
and follow it in output order. Safe. */
if (lowest_index >= dpb->max_num_reorder_frames) {
guint i, need_output;
need_output = 0;
for (i = 0; i < lowest_index; i++) {
GstH264Picture *p = g_array_index (dpb->pic_list, GstH264Picture *, i);
if (p->needed_for_output)
need_output++;
}
if (need_output >= dpb->max_num_reorder_frames) {
GST_TRACE ("frame with lowest poc %d has %d precede frame, already"
" satisfy num_reorder_frames %d, bumping for low-latency.",
dpb->last_output_poc, lowest_index, dpb->max_num_reorder_frames);
return TRUE;
}
}
/* Bump leading picture with the negative POC if already found positive
POC. It's even impossible to insert another negative POC after the
positive POCs. Almost safe. */
if (lowest_poc < 0 && to_insert->pic_order_cnt > 0) {
GST_TRACE ("The negative poc %d, bumping for low-latency.", lowest_poc);
return TRUE;
}
/* There may be leading frames with negative POC following the IDR
frame in decoder order, so when IDR comes, we need to check the
following pictures. In most cases, leading pictures are in increasing
POC order. Bump and should be safe. */
if (lowest_poc == 0 && gst_h264_dpb_get_size (dpb) <= 1) {
if (to_insert->pic_order_cnt > lowest_poc) {
GST_TRACE ("The IDR or mem_mgmt_5 frame, bumping for low-latency.");
return TRUE;
}
GST_TRACE ("The IDR or mem_mgmt_5 frame is not the first frame.");
goto normal_bump;
}
/* When non-ref frame has the lowest POC, it's unlike to insert another
ref frame with very small POC. Bump and should be safe. */
if (!is_ref_picture) {
GST_TRACE ("non ref with lowest-poc: %d bumping for low-latency",
lowest_poc);
return TRUE;
}
/* When insert non-ref frame with bigger POC, it's unlike to insert
another ref frame with very small POC. Bump and should be safe. */
if (!to_insert->ref_pic && lowest_poc < to_insert->pic_order_cnt) {
GST_TRACE ("lowest-poc: %d < to insert non ref pic: %d, bumping "
"for low-latency", lowest_poc, to_insert->pic_order_cnt);
return TRUE;
}
/* PicOrderCnt increment by <=2. Not all streams meet this, but in
practice this condition can be used.
For stream with 2 poc increment like:
0(IDR), 2(P), 4(P), 6(P), 12(P), 8(B), 10(B)....
This can work well, but for streams with 1 poc increment like:
0(IDR), 2(P), 4(P), 1(B), 3(B) ...
This can cause picture disorder. Most stream in practice has the
2 poc increment, but this may have risk and be careful. */
#if 0
if (lowest_poc > dpb->last_output_poc
&& lowest_poc - dpb->last_output_poc <= 2) {
GST_TRACE ("lowest-poc: %d, last-output-poc: %d, bumping for"
" low-latency", lowest_poc, dpb->last_output_poc);
return TRUE;
}
#endif
}
normal_bump:
/* C.4.5.3: The "bumping" process is invoked in the following cases.
- There is no empty frame buffer and a empty frame buffer is needed
for storage of an inferred "non-existing" frame.
@ -731,13 +848,6 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert,
return TRUE;
}
lowest_poc = G_MAXINT32;
gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
if (picture) {
lowest_poc = picture->pic_order_cnt;
gst_h264_picture_unref (picture);
}
if (to_insert->pic_order_cnt > lowest_poc) {
GST_TRACE ("No empty frame buffer, lowest poc %d < current poc %d,"
" need bumping.", lowest_poc, to_insert->pic_order_cnt);
@ -816,10 +926,31 @@ gst_h264_dpb_bump (GstH264Dpb * dpb, gboolean drain)
}
dpb->last_output_poc = picture->pic_order_cnt;
dpb->last_output_non_ref = !picture->ref_pic;
return picture;
}
/**
* gst_h264_dpb_set_last_output:
* @dpb: a #GstH264Dpb
* @picture: a #GstH264Picture of the last output.
*
* Notify the DPB that @picture is output directly without storing
* in the DPB.
*
* Since: 1.20
*/
void
gst_h264_dpb_set_last_output (GstH264Dpb * dpb, GstH264Picture * picture)
{
g_return_if_fail (dpb != NULL);
g_return_if_fail (GST_IS_H264_PICTURE (picture));
dpb->last_output_poc = picture->pic_order_cnt;
dpb->last_output_non_ref = !picture->ref_pic;
}
static gint
get_picNumX (GstH264Picture * picture, GstH264RefPicMarking * ref_pic_marking)
{

View file

@ -226,13 +226,13 @@ GST_CODECS_API
void gst_h264_dpb_set_interlaced (GstH264Dpb * dpb,
gboolean interlaced);
GST_CODECS_API
gboolean gst_h264_dpb_get_interlaced (GstH264Dpb * dpb);
GST_CODECS_API
void gst_h264_dpb_set_max_num_reorder_frames (GstH264Dpb * dpb,
guint32 max_num_reorder_frames);
GST_CODECS_API
gboolean gst_h264_dpb_get_interlaced (GstH264Dpb * dpb);
GST_CODECS_API
void gst_h264_dpb_free (GstH264Dpb * dpb);
@ -296,6 +296,10 @@ GST_CODECS_API
GstH264Picture * gst_h264_dpb_bump (GstH264Dpb * dpb,
gboolean drain);
GST_CODECS_API
void gst_h264_dpb_set_last_output (GstH264Dpb * dpb,
GstH264Picture * picture);
GST_CODECS_API
gboolean gst_h264_dpb_perform_memory_management_control_operation (GstH264Dpb * dpb,
GstH264RefPicMarking *ref_pic_marking,