codecs: h264dec: Improve the algorithm for low latency mode.

In low_latency mode, try to bump the picture as soon as possible
without the frames disorder:
1. We can directly output the continuous non-reference frame.
2. Consider max_num_reorder_frames, which is special useful for
   I-P mode.
3. Consider the leading pictures with negative POC.
4  Output small POC pictures when non-reference frame comes.
4. Output the POC increment<=2 pictures. This is not 100% safe,
   but in practice this condition can be used.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2373>

gst-libs/gst/codecs/gsth264decoder.c

@@ -1846,7 +1846,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
     gst_h264_decoder_drain_internal (self);
   }
 
-  while (gst_h264_dpb_needs_bump (priv->dpb, picture, FALSE)) {
+  while (gst_h264_dpb_needs_bump (priv->dpb, picture, priv->is_live)) {
     GstH264Picture *to_output;
 
     to_output = gst_h264_dpb_bump (priv->dpb, FALSE);
@@ -1897,6 +1897,7 @@ gst_h264_decoder_finish_picture (GstH264Decoder * self,
     }
   } else {
     gst_h264_decoder_do_output_picture (self, picture);
+    gst_h264_dpb_set_last_output (priv->dpb, picture);
   }
 
   GST_LOG_OBJECT (self,

gst-libs/gst/codecs/gsth264picture.c

@@ -110,6 +110,7 @@ struct _GstH264Dpb
   gint num_output_needed;
   guint32 max_num_reorder_frames;
   gint32 last_output_poc;
+  gboolean last_output_non_ref;
 
   gboolean interlaced;
 };
@@ -119,6 +120,7 @@ gst_h264_dpb_init (GstH264Dpb * dpb)
 {
   dpb->num_output_needed = 0;
   dpb->last_output_poc = G_MININT32;
+  dpb->last_output_non_ref = FALSE;
 }
 
 /**
@@ -300,6 +302,13 @@ gst_h264_dpb_add (GstH264Dpb * dpb, GstH264Picture * picture)
   if (dpb->pic_list->len > dpb->max_num_frames * (dpb->interlaced + 1))
     GST_ERROR ("DPB size is %d, exceed the max size %d",
         dpb->pic_list->len, dpb->max_num_frames);
+
+  /* The IDR frame or mem_mgmt_5 */
+  if (picture->pic_order_cnt == 0) {
+    GST_TRACE ("last_output_poc reset because of IDR or mem_mgmt_5");
+    dpb->last_output_poc = G_MININT32;
+    dpb->last_output_non_ref = FALSE;
+  }
 }
 
 /**
@@ -701,16 +710,124 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert,
 {
   GstH264Picture *picture = NULL;
   gint32 lowest_poc;
+  gboolean is_ref_picture;
+  gint lowest_index;
 
   g_return_val_if_fail (dpb != NULL, FALSE);
   g_assert (dpb->num_output_needed >= 0);
 
-  /* FIXME: Need to revisit for intelaced decoding */
+  lowest_poc = G_MAXINT32;
-
+  is_ref_picture = FALSE;
-  if (low_latency) {
+  lowest_index = gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
-    /* TODO: */
+  if (lowest_index >= 0) {
+    lowest_poc = picture->pic_order_cnt;
+    is_ref_picture = picture->ref_pic;
+    gst_h264_picture_unref (picture);
+  } else {
+    goto normal_bump;
   }
 
+  if (low_latency) {
+    /* If low latency, we should not wait for the DPB becoming full.
+       We try to bump the picture as soon as possible without the
+       frames disorder. The policy is from the safe to some risk. */
+
+    /* Do not support interlaced mode. */
+    if (gst_h264_dpb_get_interlaced (dpb))
+      goto normal_bump;
+
+    /* Equal to normal bump. */
+    if (!gst_h264_dpb_has_empty_frame_buffer (dpb))
+      goto normal_bump;
+
+    /* 7.4.1.2.2: The values of picture order count for the coded pictures
+       in consecutive access units in decoding order containing non-reference
+       pictures shall be non-decreasing. Safe. */
+    if (dpb->last_output_non_ref && !is_ref_picture) {
+      g_assert (dpb->last_output_poc < G_MAXINT32);
+      GST_TRACE ("Continuous non-reference frame poc: %d -> %d,"
+          " bumping for low-latency.", dpb->last_output_poc, lowest_poc);
+      return TRUE;
+    }
+
+    /* num_reorder_frames indicates the maximum number of frames, that
+       precede any frame in the coded video sequence in decoding order
+       and follow it in output order. Safe. */
+    if (lowest_index >= dpb->max_num_reorder_frames) {
+      guint i, need_output;
+
+      need_output = 0;
+      for (i = 0; i < lowest_index; i++) {
+        GstH264Picture *p = g_array_index (dpb->pic_list, GstH264Picture *, i);
+        if (p->needed_for_output)
+          need_output++;
+      }
+
+      if (need_output >= dpb->max_num_reorder_frames) {
+        GST_TRACE ("frame with lowest poc %d has %d precede frame, already"
+            " satisfy num_reorder_frames %d, bumping for low-latency.",
+            dpb->last_output_poc, lowest_index, dpb->max_num_reorder_frames);
+        return TRUE;
+      }
+    }
+
+    /* Bump leading picture with the negative POC if already found positive
+       POC. It's even impossible to insert another negative POC after the
+       positive POCs. Almost safe. */
+    if (lowest_poc < 0 && to_insert->pic_order_cnt > 0) {
+      GST_TRACE ("The negative poc %d, bumping for low-latency.", lowest_poc);
+      return TRUE;
+    }
+
+    /* There may be leading frames with negative POC following the IDR
+       frame in decoder order, so when IDR comes, we need to check the
+       following pictures. In most cases, leading pictures are in increasing
+       POC order. Bump and should be safe. */
+    if (lowest_poc == 0 && gst_h264_dpb_get_size (dpb) <= 1) {
+      if (to_insert->pic_order_cnt > lowest_poc) {
+        GST_TRACE ("The IDR or mem_mgmt_5 frame, bumping for low-latency.");
+        return TRUE;
+      }
+
+      GST_TRACE ("The IDR or mem_mgmt_5 frame is not the first frame.");
+      goto normal_bump;
+    }
+
+    /* When non-ref frame has the lowest POC, it's unlike to insert another
+       ref frame with very small POC. Bump and should be safe. */
+    if (!is_ref_picture) {
+      GST_TRACE ("non ref with lowest-poc: %d bumping for low-latency",
+          lowest_poc);
+      return TRUE;
+    }
+
+    /* When insert non-ref frame with bigger POC, it's unlike to insert
+       another ref frame with very small POC. Bump and should be safe. */
+    if (!to_insert->ref_pic && lowest_poc < to_insert->pic_order_cnt) {
+      GST_TRACE ("lowest-poc: %d < to insert non ref pic: %d, bumping "
+          "for low-latency", lowest_poc, to_insert->pic_order_cnt);
+      return TRUE;
+    }
+
+    /* PicOrderCnt increment by <=2. Not all streams meet this, but in
+       practice this condition can be used.
+       For stream with 2 poc increment like:
+       0(IDR), 2(P), 4(P), 6(P), 12(P), 8(B), 10(B)....
+       This can work well, but for streams with 1 poc increment like:
+       0(IDR), 2(P), 4(P), 1(B), 3(B) ...
+       This can cause picture disorder. Most stream in practice has the
+       2 poc increment, but this may have risk and be careful. */
+#if 0
+    if (lowest_poc > dpb->last_output_poc
+        && lowest_poc - dpb->last_output_poc <= 2) {
+      GST_TRACE ("lowest-poc: %d, last-output-poc: %d, bumping for"
+          " low-latency", lowest_poc, dpb->last_output_poc);
+      return TRUE;
+    }
+#endif
+  }
+
+normal_bump:
   /* C.4.5.3: The "bumping" process is invoked in the following cases.
      - There is no empty frame buffer and a empty frame buffer is needed
      for storage of an inferred "non-existing" frame.
@@ -731,13 +848,6 @@ gst_h264_dpb_needs_bump (GstH264Dpb * dpb, GstH264Picture * to_insert,
     return TRUE;
   }
 
-  lowest_poc = G_MAXINT32;
-  gst_h264_dpb_get_lowest_output_needed_picture (dpb, &picture);
-  if (picture) {
-    lowest_poc = picture->pic_order_cnt;
-    gst_h264_picture_unref (picture);
-  }
-
   if (to_insert->pic_order_cnt > lowest_poc) {
     GST_TRACE ("No empty frame buffer, lowest poc %d < current poc %d,"
         " need bumping.", lowest_poc, to_insert->pic_order_cnt);
@@ -816,10 +926,31 @@ gst_h264_dpb_bump (GstH264Dpb * dpb, gboolean drain)
   }
 
   dpb->last_output_poc = picture->pic_order_cnt;
+  dpb->last_output_non_ref = !picture->ref_pic;
 
   return picture;
 }
 
+/**
+ * gst_h264_dpb_set_last_output:
+ * @dpb: a #GstH264Dpb
+ * @picture: a #GstH264Picture of the last output.
+ *
+ * Notify the DPB that @picture is output directly without storing
+ * in the DPB.
+ *
+ * Since: 1.20
+ */
+void
+gst_h264_dpb_set_last_output (GstH264Dpb * dpb, GstH264Picture * picture)
+{
+  g_return_if_fail (dpb != NULL);
+  g_return_if_fail (GST_IS_H264_PICTURE (picture));
+
+  dpb->last_output_poc = picture->pic_order_cnt;
+  dpb->last_output_non_ref = !picture->ref_pic;
+}
+
 static gint
 get_picNumX (GstH264Picture * picture, GstH264RefPicMarking * ref_pic_marking)
 {

gst-libs/gst/codecs/gsth264picture.h

@@ -226,13 +226,13 @@ GST_CODECS_API
 void gst_h264_dpb_set_interlaced      (GstH264Dpb * dpb,
                                        gboolean interlaced);
 
-GST_CODECS_API
-gboolean gst_h264_dpb_get_interlaced  (GstH264Dpb * dpb);
-
 GST_CODECS_API
 void gst_h264_dpb_set_max_num_reorder_frames (GstH264Dpb * dpb,
                                               guint32 max_num_reorder_frames);
 
+GST_CODECS_API
+gboolean gst_h264_dpb_get_interlaced  (GstH264Dpb * dpb);
+
 GST_CODECS_API
 void  gst_h264_dpb_free             (GstH264Dpb * dpb);
 
@@ -296,6 +296,10 @@ GST_CODECS_API
 GstH264Picture * gst_h264_dpb_bump (GstH264Dpb * dpb,
                                     gboolean drain);
 
+GST_CODECS_API
+void gst_h264_dpb_set_last_output (GstH264Dpb * dpb,
+                                   GstH264Picture * picture);
+
 GST_CODECS_API
 gboolean         gst_h264_dpb_perform_memory_management_control_operation (GstH264Dpb * dpb,
                                                                            GstH264RefPicMarking *ref_pic_marking,