From 130205629f94c26ebb155f50c643bdb9664e2051 Mon Sep 17 00:00:00 2001 From: He Junyan Date: Wed, 21 Jul 2021 00:04:18 +0800 Subject: [PATCH] codecs: h264dec: Improve the algorithm for low latency mode. In low_latency mode, try to bump the picture as soon as possible without the frames disorder: 1. We can directly output the continuous non-reference frame. 2. Consider max_num_reorder_frames, which is special useful for I-P mode. 3. Consider the leading pictures with negative POC. 4 Output small POC pictures when non-reference frame comes. 4. Output the POC increment<=2 pictures. This is not 100% safe, but in practice this condition can be used. Part-of: --- gst-libs/gst/codecs/gsth264decoder.c | 3 +- gst-libs/gst/codecs/gsth264picture.c | 153 +++++++++++++++++++++++++-- gst-libs/gst/codecs/gsth264picture.h | 10 +- 3 files changed, 151 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/codecs/gsth264decoder.c b/gst-libs/gst/codecs/gsth264decoder.c index 1442498540..3baf4118b6 100644 --- a/gst-libs/gst/codecs/gsth264decoder.c +++ b/gst-libs/gst/codecs/gsth264decoder.c @@ -1846,7 +1846,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self, gst_h264_decoder_drain_internal (self); } - while (gst_h264_dpb_needs_bump (priv->dpb, picture, FALSE)) { + while (gst_h264_dpb_needs_bump (priv->dpb, picture, priv->is_live)) { GstH264Picture *to_output; to_output = gst_h264_dpb_bump (priv->dpb, FALSE); @@ -1897,6 +1897,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self, } } else { gst_h264_decoder_do_output_picture (self, picture); + gst_h264_dpb_set_last_output (priv->dpb, picture); } GST_LOG_OBJECT (self, diff --git a/gst-libs/gst/codecs/gsth264picture.c b/gst-libs/gst/codecs/gsth264picture.c index a824b785a1..14e7bfd844 100644 --- a/gst-libs/gst/codecs/gsth264picture.c +++ b/gst-libs/gst/codecs/gsth264picture.c @@ -110,6 +110,7 @@ struct _GstH264Dpb gint num_output_needed; guint32 max_num_reorder_frames; gint32 last_output_poc; + gboolean last_output_non_ref; gboolean interlaced; }; @@ -119,6 +120,7 @@ gst_h264_dpb_init (GstH264Dpb * dpb) { dpb->num_output_needed = 0; dpb->last_output_poc = G_MININT32; + dpb->last_output_non_ref = FALSE; } /** @@ -300,6 +302,13 @@ gst_h264_dpb_add (GstH264Dpb * dpb, GstH264Picture * picture) if (dpb->pic_list->len > dpb->max_num_frames * (dpb->interlaced + 1)) GST_ERROR ("DPB size is %d, exceed the max size %d", dpb->pic_list->len, dpb->max_num_frames); + + /* The IDR frame or mem_mgmt_5 */ + if (picture->pic_order_cnt == 0) { + GST_TRACE ("last_output_poc reset because of IDR or mem_mgmt_5"); + dpb->last_output_poc = G_MININT32; + dpb->last_output_non_ref = FALSE; + } } /** @@ -701,16 +710,124 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert, { GstH264Picture *picture = NULL; gint32 lowest_poc; + gboolean is_ref_picture; + gint lowest_index; g_return_val_if_fail (dpb != NULL, FALSE); g_assert (dpb->num_output_needed >= 0); - /* FIXME: Need to revisit for intelaced decoding */ - - if (low_latency) { - /* TODO: */ + lowest_poc = G_MAXINT32; + is_ref_picture = FALSE; + lowest_index = gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture); + if (lowest_index >= 0) { + lowest_poc = picture->pic_order_cnt; + is_ref_picture = picture->ref_pic; + gst_h264_picture_unref (picture); + } else { + goto normal_bump; } + if (low_latency) { + /* If low latency, we should not wait for the DPB becoming full. + We try to bump the picture as soon as possible without the + frames disorder. The policy is from the safe to some risk. */ + + /* Do not support interlaced mode. */ + if (gst_h264_dpb_get_interlaced (dpb)) + goto normal_bump; + + /* Equal to normal bump. */ + if (!gst_h264_dpb_has_empty_frame_buffer (dpb)) + goto normal_bump; + + /* 7.4.1.2.2: The values of picture order count for the coded pictures + in consecutive access units in decoding order containing non-reference + pictures shall be non-decreasing. Safe. */ + if (dpb->last_output_non_ref && !is_ref_picture) { + g_assert (dpb->last_output_poc < G_MAXINT32); + GST_TRACE ("Continuous non-reference frame poc: %d -> %d," + " bumping for low-latency.", dpb->last_output_poc, lowest_poc); + return TRUE; + } + + /* num_reorder_frames indicates the maximum number of frames, that + precede any frame in the coded video sequence in decoding order + and follow it in output order. Safe. */ + if (lowest_index >= dpb->max_num_reorder_frames) { + guint i, need_output; + + need_output = 0; + for (i = 0; i < lowest_index; i++) { + GstH264Picture *p = g_array_index (dpb->pic_list, GstH264Picture *, i); + if (p->needed_for_output) + need_output++; + } + + if (need_output >= dpb->max_num_reorder_frames) { + GST_TRACE ("frame with lowest poc %d has %d precede frame, already" + " satisfy num_reorder_frames %d, bumping for low-latency.", + dpb->last_output_poc, lowest_index, dpb->max_num_reorder_frames); + return TRUE; + } + } + + /* Bump leading picture with the negative POC if already found positive + POC. It's even impossible to insert another negative POC after the + positive POCs. Almost safe. */ + if (lowest_poc < 0 && to_insert->pic_order_cnt > 0) { + GST_TRACE ("The negative poc %d, bumping for low-latency.", lowest_poc); + return TRUE; + } + + /* There may be leading frames with negative POC following the IDR + frame in decoder order, so when IDR comes, we need to check the + following pictures. In most cases, leading pictures are in increasing + POC order. Bump and should be safe. */ + if (lowest_poc == 0 && gst_h264_dpb_get_size (dpb) <= 1) { + if (to_insert->pic_order_cnt > lowest_poc) { + GST_TRACE ("The IDR or mem_mgmt_5 frame, bumping for low-latency."); + return TRUE; + } + + GST_TRACE ("The IDR or mem_mgmt_5 frame is not the first frame."); + goto normal_bump; + } + + /* When non-ref frame has the lowest POC, it's unlike to insert another + ref frame with very small POC. Bump and should be safe. */ + if (!is_ref_picture) { + GST_TRACE ("non ref with lowest-poc: %d bumping for low-latency", + lowest_poc); + return TRUE; + } + + /* When insert non-ref frame with bigger POC, it's unlike to insert + another ref frame with very small POC. Bump and should be safe. */ + if (!to_insert->ref_pic && lowest_poc < to_insert->pic_order_cnt) { + GST_TRACE ("lowest-poc: %d < to insert non ref pic: %d, bumping " + "for low-latency", lowest_poc, to_insert->pic_order_cnt); + return TRUE; + } + + /* PicOrderCnt increment by <=2. Not all streams meet this, but in + practice this condition can be used. + For stream with 2 poc increment like: + 0(IDR), 2(P), 4(P), 6(P), 12(P), 8(B), 10(B).... + This can work well, but for streams with 1 poc increment like: + 0(IDR), 2(P), 4(P), 1(B), 3(B) ... + This can cause picture disorder. Most stream in practice has the + 2 poc increment, but this may have risk and be careful. */ +#if 0 + if (lowest_poc > dpb->last_output_poc + && lowest_poc - dpb->last_output_poc <= 2) { + GST_TRACE ("lowest-poc: %d, last-output-poc: %d, bumping for" + " low-latency", lowest_poc, dpb->last_output_poc); + return TRUE; + } +#endif + } + +normal_bump: /* C.4.5.3: The "bumping" process is invoked in the following cases. - There is no empty frame buffer and a empty frame buffer is needed for storage of an inferred "non-existing" frame. @@ -731,13 +848,6 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert, return TRUE; } - lowest_poc = G_MAXINT32; - gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture); - if (picture) { - lowest_poc = picture->pic_order_cnt; - gst_h264_picture_unref (picture); - } - if (to_insert->pic_order_cnt > lowest_poc) { GST_TRACE ("No empty frame buffer, lowest poc %d < current poc %d," " need bumping.", lowest_poc, to_insert->pic_order_cnt); @@ -816,10 +926,31 @@ gst_h264_dpb_bump (GstH264Dpb * dpb, gboolean drain) } dpb->last_output_poc = picture->pic_order_cnt; + dpb->last_output_non_ref = !picture->ref_pic; return picture; } +/** + * gst_h264_dpb_set_last_output: + * @dpb: a #GstH264Dpb + * @picture: a #GstH264Picture of the last output. + * + * Notify the DPB that @picture is output directly without storing + * in the DPB. + * + * Since: 1.20 + */ +void +gst_h264_dpb_set_last_output (GstH264Dpb * dpb, GstH264Picture * picture) +{ + g_return_if_fail (dpb != NULL); + g_return_if_fail (GST_IS_H264_PICTURE (picture)); + + dpb->last_output_poc = picture->pic_order_cnt; + dpb->last_output_non_ref = !picture->ref_pic; +} + static gint get_picNumX (GstH264Picture * picture, GstH264RefPicMarking * ref_pic_marking) { diff --git a/gst-libs/gst/codecs/gsth264picture.h b/gst-libs/gst/codecs/gsth264picture.h index 3ac505c032..d0b914f2ce 100644 --- a/gst-libs/gst/codecs/gsth264picture.h +++ b/gst-libs/gst/codecs/gsth264picture.h @@ -226,13 +226,13 @@ GST_CODECS_API void gst_h264_dpb_set_interlaced (GstH264Dpb * dpb, gboolean interlaced); -GST_CODECS_API -gboolean gst_h264_dpb_get_interlaced (GstH264Dpb * dpb); - GST_CODECS_API void gst_h264_dpb_set_max_num_reorder_frames (GstH264Dpb * dpb, guint32 max_num_reorder_frames); +GST_CODECS_API +gboolean gst_h264_dpb_get_interlaced (GstH264Dpb * dpb); + GST_CODECS_API void gst_h264_dpb_free (GstH264Dpb * dpb); @@ -296,6 +296,10 @@ GST_CODECS_API GstH264Picture * gst_h264_dpb_bump (GstH264Dpb * dpb, gboolean drain); +GST_CODECS_API +void gst_h264_dpb_set_last_output (GstH264Dpb * dpb, + GstH264Picture * picture); + GST_CODECS_API gboolean gst_h264_dpb_perform_memory_management_control_operation (GstH264Dpb * dpb, GstH264RefPicMarking *ref_pic_marking,